You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2017/05/11 16:58:42 UTC
[01/50] [abbrv] hadoop git commit: HDFS-11448. JN log segment syncing
should support HA upgrade. Contributed by Hanisha Koneru. [Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/HDFS-10467 f242d255c -> 6632c5a45 (forced update)
HDFS-11448. JN log segment syncing should support HA upgrade. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/07761af3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/07761af3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/07761af3
Branch: refs/heads/HDFS-10467
Commit: 07761af357ef4da791df2972d7d3f049d6011c8d
Parents: 54e2b9e
Author: Arpit Agarwal <ar...@apache.org>
Authored: Thu May 4 15:57:44 2017 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu May 4 15:57:44 2017 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/qjournal/server/JNStorage.java | 25 ++++-
.../hadoop/hdfs/qjournal/server/Journal.java | 21 +++-
.../hdfs/qjournal/server/JournalNodeSyncer.java | 104 ++++++++++++-------
.../hdfs/qjournal/TestJournalNodeSync.java | 1 +
4 files changed, 105 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07761af3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java
index 8f40f6b..7226cae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java
@@ -58,6 +58,8 @@ class JNStorage extends Storage {
private static final List<Pattern> PAXOS_DIR_PURGE_REGEXES =
ImmutableList.of(Pattern.compile("(\\d+)"));
+ private static final String STORAGE_EDITS_SYNC = "edits.sync";
+
/**
* @param conf Configuration object
* @param logDir the path to the directory in which data will be stored
@@ -120,12 +122,29 @@ class JNStorage extends Storage {
return new File(sd.getCurrentDir(), name);
}
- File getTemporaryEditsFile(long startTxId, long endTxId, long timestamp) {
- return NNStorage.getTemporaryEditsFile(sd, startTxId, endTxId, timestamp);
+ File getCurrentDir() {
+ return sd.getCurrentDir();
+ }
+
+ /**
+ * Directory {@code edits.sync} temporarily holds the log segments
+ * downloaded through {@link JournalNodeSyncer} before they are moved to
+ * {@code current} directory.
+ *
+ * @return the directory path
+ */
+ File getEditsSyncDir() {
+ return new File(sd.getRoot(), STORAGE_EDITS_SYNC);
+ }
+
+ File getTemporaryEditsFile(long startTxId, long endTxId) {
+ return new File(getEditsSyncDir(), String.format("%s_%019d-%019d",
+ NNStorage.NameNodeFile.EDITS.getName(), startTxId, endTxId));
}
File getFinalizedEditsFile(long startTxId, long endTxId) {
- return NNStorage.getFinalizedEditsFile(sd, startTxId, endTxId);
+ return new File(sd.getCurrentDir(), String.format("%s_%019d-%019d",
+ NNStorage.NameNodeFile.EDITS.getName(), startTxId, endTxId));
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07761af3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
index ca21373..0041d5e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
@@ -24,6 +24,8 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStreamWriter;
import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
import java.security.PrivilegedExceptionAction;
import java.util.Iterator;
import java.util.List;
@@ -1092,19 +1094,28 @@ public class Journal implements Closeable {
committedTxnId.set(startTxId - 1);
}
- synchronized boolean renameTmpSegment(File tmpFile, File finalFile,
+ synchronized boolean moveTmpSegmentToCurrent(File tmpFile, File finalFile,
long endTxId) throws IOException {
final boolean success;
if (endTxId <= committedTxnId.get()) {
- success = tmpFile.renameTo(finalFile);
- if (!success) {
- LOG.warn("Unable to rename edits file from " + tmpFile + " to " +
+ if (!finalFile.getParentFile().exists()) {
+ LOG.error(finalFile.getParentFile() + " doesn't exist. Aborting tmp " +
+ "segment move to current directory");
+ return false;
+ }
+ Files.move(tmpFile.toPath(), finalFile.toPath(),
+ StandardCopyOption.ATOMIC_MOVE);
+ if (finalFile.exists() && FileUtil.canRead(finalFile)) {
+ success = true;
+ } else {
+ success = false;
+ LOG.warn("Unable to move edits file from " + tmpFile + " to " +
finalFile);
}
} else {
success = false;
LOG.error("The endTxId of the temporary file is not less than the " +
- "last committed transaction id. Aborting renaming to final file" +
+ "last committed transaction id. Aborting move to final file" +
finalFile);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07761af3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java
index f195c00..788c5de 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java
@@ -41,7 +41,6 @@ import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
import org.apache.hadoop.hdfs.util.DataTransferThrottler;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.util.Daemon;
-import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -98,6 +97,11 @@ public class JournalNodeSyncer {
void stopSync() {
shouldSync = false;
+ // Delete the edits.sync directory
+ File editsSyncDir = journal.getStorage().getEditsSyncDir();
+ if (editsSyncDir.exists()) {
+ FileUtil.fullyDelete(editsSyncDir);
+ }
if (syncJournalDaemon != null) {
syncJournalDaemon.interrupt();
}
@@ -112,6 +116,15 @@ public class JournalNodeSyncer {
}
}
+ private boolean createEditsSyncDir() {
+ File editsSyncDir = journal.getStorage().getEditsSyncDir();
+ if (editsSyncDir.exists()) {
+ LOG.info(editsSyncDir + " directory already exists.");
+ return true;
+ }
+ return editsSyncDir.mkdir();
+ }
+
private boolean getOtherJournalNodeProxies() {
List<InetSocketAddress> otherJournalNodes = getOtherJournalNodeAddrs();
if (otherJournalNodes == null || otherJournalNodes.isEmpty()) {
@@ -135,35 +148,51 @@ public class JournalNodeSyncer {
}
private void startSyncJournalsDaemon() {
- syncJournalDaemon = new Daemon(new Runnable() {
- @Override
- public void run() {
- while(shouldSync) {
- try {
- if (!journal.isFormatted()) {
- LOG.warn("Journal not formatted. Cannot sync.");
+ syncJournalDaemon = new Daemon(() -> {
+ // Wait for journal to be formatted to create edits.sync directory
+ while(!journal.isFormatted()) {
+ try {
+ Thread.sleep(journalSyncInterval);
+ } catch (InterruptedException e) {
+ LOG.error("JournalNodeSyncer daemon received Runtime exception.", e);
+ Thread.currentThread().interrupt();
+ return;
+ }
+ }
+ if (!createEditsSyncDir()) {
+ LOG.error("Failed to create directory for downloading log " +
+ "segments: %s. Stopping Journal Node Sync.",
+ journal.getStorage().getEditsSyncDir());
+ return;
+ }
+ while(shouldSync) {
+ try {
+ if (!journal.isFormatted()) {
+ LOG.warn("Journal cannot sync. Not formatted.");
+ } else {
+ syncJournals();
+ }
+ Thread.sleep(journalSyncInterval);
+ } catch (Throwable t) {
+ if (!shouldSync) {
+ if (t instanceof InterruptedException) {
+ LOG.info("Stopping JournalNode Sync.");
+ Thread.currentThread().interrupt();
+ return;
} else {
- syncJournals();
+ LOG.warn("JournalNodeSyncer received an exception while " +
+ "shutting down.", t);
}
- Thread.sleep(journalSyncInterval);
- } catch (Throwable t) {
- if (!shouldSync) {
- if (t instanceof InterruptedException) {
- LOG.info("Stopping JournalNode Sync.");
- } else {
- LOG.warn("JournalNodeSyncer received an exception while " +
- "shutting down.", t);
- }
- break;
- } else {
- if (t instanceof InterruptedException) {
- LOG.warn("JournalNodeSyncer interrupted", t);
- break;
- }
+ break;
+ } else {
+ if (t instanceof InterruptedException) {
+ LOG.warn("JournalNodeSyncer interrupted", t);
+ Thread.currentThread().interrupt();
+ return;
}
- LOG.error(
- "JournalNodeSyncer daemon received Runtime exception. ", t);
}
+ LOG.error(
+ "JournalNodeSyncer daemon received Runtime exception. ", t);
}
}
});
@@ -335,8 +364,8 @@ public class JournalNodeSyncer {
/**
* Transfer an edit log from one journal node to another for sync-up.
*/
- private boolean downloadMissingLogSegment(URL url, RemoteEditLog log) throws
- IOException {
+ private boolean downloadMissingLogSegment(URL url, RemoteEditLog log)
+ throws IOException {
LOG.info("Downloading missing Edit Log from " + url + " to " + jnStorage
.getRoot());
@@ -350,9 +379,10 @@ public class JournalNodeSyncer {
return true;
}
- final long milliTime = Time.monotonicNow();
- File tmpEditsFile = jnStorage.getTemporaryEditsFile(log.getStartTxId(), log
- .getEndTxId(), milliTime);
+ // Download the log segment to current.tmp directory first.
+ File tmpEditsFile = jnStorage.getTemporaryEditsFile(
+ log.getStartTxId(), log.getEndTxId());
+
try {
Util.doGetUrl(url, ImmutableList.of(tmpEditsFile), jnStorage, false,
logSegmentTransferTimeout, throttler);
@@ -367,14 +397,12 @@ public class JournalNodeSyncer {
LOG.info("Downloaded file " + tmpEditsFile.getName() + " of size " +
tmpEditsFile.length() + " bytes.");
- LOG.debug("Renaming " + tmpEditsFile.getName() + " to "
- + finalEditsFile.getName());
- boolean renameSuccess = journal.renameTmpSegment(tmpEditsFile,
+ final boolean moveSuccess = journal.moveTmpSegmentToCurrent(tmpEditsFile,
finalEditsFile, log.getEndTxId());
- if (!renameSuccess) {
- //If rename is not successful, delete the tmpFile
- LOG.debug("Renaming unsuccessful. Deleting temporary file: "
- + tmpEditsFile);
+ if (!moveSuccess) {
+ // If move is not successful, delete the tmpFile
+ LOG.debug("Move to current directory unsuccessful. Deleting temporary " +
+ "file: " + tmpEditsFile);
if (!tmpEditsFile.delete()) {
LOG.warn("Deleting " + tmpEditsFile + " has failed");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07761af3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestJournalNodeSync.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestJournalNodeSync.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestJournalNodeSync.java
index 5375b02..8415a6f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestJournalNodeSync.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestJournalNodeSync.java
@@ -57,6 +57,7 @@ public class TestJournalNodeSync {
@Before
public void setUpMiniCluster() throws IOException {
final Configuration conf = new HdfsConfiguration();
+ conf.setBoolean(DFSConfigKeys.DFS_JOURNALNODE_ENABLE_SYNC_KEY, true);
conf.setLong(DFSConfigKeys.DFS_JOURNALNODE_SYNC_INTERVAL_KEY, 1000L);
qjmhaCluster = new MiniQJMHACluster.Builder(conf).setNumNameNodes(2)
.build();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: HDFS-11644. Support for querying
outputstream capabilities. Contributed by Manoj Govindassamy.
Posted by in...@apache.org.
HDFS-11644. Support for querying outputstream capabilities. Contributed by Manoj Govindassamy.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/54fd0e44
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/54fd0e44
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/54fd0e44
Branch: refs/heads/HDFS-10467
Commit: 54fd0e44b76c4b982dcfb47932b6159851f14136
Parents: 749e5c0
Author: Andrew Wang <wa...@apache.org>
Authored: Mon May 8 21:59:49 2017 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Mon May 8 21:59:49 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/fs/FSDataOutputStream.java | 10 ++-
.../apache/hadoop/fs/StreamCapabilities.java | 67 ++++++++++++++++++++
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 15 ++++-
.../hadoop/hdfs/DFSStripedOutputStream.java | 11 +++-
.../apache/hadoop/hdfs/TestDFSOutputStream.java | 25 ++++++++
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 24 ++++---
.../fs/azure/SyncableDataOutputStream.java | 12 +++-
7 files changed, 150 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54fd0e44/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java
index 3f97ea8..1d95cd3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.classification.InterfaceStability;
@InterfaceAudience.Public
@InterfaceStability.Stable
public class FSDataOutputStream extends DataOutputStream
- implements Syncable, CanSetDropBehind {
+ implements Syncable, CanSetDropBehind, StreamCapabilities {
private final OutputStream wrappedStream;
private static class PositionCache extends FilterOutputStream {
@@ -111,6 +111,14 @@ public class FSDataOutputStream extends DataOutputStream
return wrappedStream;
}
+ @Override
+ public boolean hasCapability(String capability) {
+ if (wrappedStream instanceof StreamCapabilities) {
+ return ((StreamCapabilities) wrappedStream).hasCapability(capability);
+ }
+ return false;
+ }
+
@Override // Syncable
public void hflush() throws IOException {
if (wrappedStream instanceof Syncable) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54fd0e44/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilities.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilities.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilities.java
new file mode 100644
index 0000000..65aa679
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilities.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Interface to query streams for supported capabilities.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public interface StreamCapabilities {
+ /**
+ * Capabilities that a stream can support and be queried for.
+ */
+ enum StreamCapability {
+ /**
+ * Stream hflush capability to flush out the data in client's buffer.
+ * Streams with this capability implement {@link Syncable} and support
+ * {@link Syncable#hflush()}.
+ */
+ HFLUSH("hflush"),
+
+ /**
+ * Stream hsync capability to flush out the data in client's buffer and
+ * the disk device. Streams with this capability implement {@link Syncable}
+ * and support {@link Syncable#hsync()}.
+ */
+ HSYNC("hsync");
+
+ private final String capability;
+
+ StreamCapability(String value) {
+ this.capability = value;
+ }
+
+ public final String getValue() {
+ return capability;
+ }
+ }
+
+ /**
+ * Query the stream for a specific capability.
+ *
+ * @param capability string to query the stream support for.
+ * @return True if the stream supports capability.
+ */
+ boolean hasCapability(String capability);
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54fd0e44/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index ceaefd8..83f1425 100755
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.hdfs;
+import static org.apache.hadoop.fs.StreamCapabilities.StreamCapability.HFLUSH;
+import static org.apache.hadoop.fs.StreamCapabilities.StreamCapability.HSYNC;
+
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InterruptedIOException;
@@ -34,6 +37,7 @@ import org.apache.hadoop.fs.FSOutputSummer;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.fs.ParentNotDirectoryException;
+import org.apache.hadoop.fs.StreamCapabilities;
import org.apache.hadoop.fs.Syncable;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
@@ -90,7 +94,7 @@ import com.google.common.base.Preconditions;
****************************************************************/
@InterfaceAudience.Private
public class DFSOutputStream extends FSOutputSummer
- implements Syncable, CanSetDropBehind {
+ implements Syncable, CanSetDropBehind, StreamCapabilities {
static final Logger LOG = LoggerFactory.getLogger(DFSOutputStream.class);
/**
* Number of times to retry creating a file when there are transient
@@ -546,6 +550,15 @@ public class DFSOutputStream extends FSOutputSummer
}
}
+ @Override
+ public boolean hasCapability(String capability) {
+ if (capability.equalsIgnoreCase(HSYNC.getValue()) ||
+ capability.equalsIgnoreCase((HFLUSH.getValue()))) {
+ return true;
+ }
+ return false;
+ }
+
/**
* Flushes out to all replicas of the block. The data is in the buffers
* of the DNs but not necessarily in the DN's OS buffers.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54fd0e44/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
index 0fdae8c..2aa9e98 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
@@ -46,6 +46,7 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.StreamCapabilities;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
@@ -77,8 +78,8 @@ import org.apache.htrace.core.TraceScope;
* Each stripe contains a sequence of cells.
*/
@InterfaceAudience.Private
-public class DFSStripedOutputStream extends DFSOutputStream {
-
+public class DFSStripedOutputStream extends DFSOutputStream
+ implements StreamCapabilities {
private static final ByteBufferPool BUFFER_POOL = new ElasticByteBufferPool();
static class MultipleBlockingQueue<T> {
@@ -810,6 +811,12 @@ public class DFSStripedOutputStream extends DFSOutputStream {
}
@Override
+ public boolean hasCapability(String capability) {
+ // StreamCapabilities like hsync / hflush are not supported yet.
+ return false;
+ }
+
+ @Override
public void hflush() {
// not supported yet
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54fd0e44/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java
index 52e3bb4..f281a3b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java
@@ -17,9 +17,11 @@
*/
package org.apache.hadoop.hdfs;
+import java.io.ByteArrayInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
+import java.io.InputStream;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
@@ -32,8 +34,10 @@ import java.util.Random;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsTracer;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StreamCapabilities.StreamCapability;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DataStreamer.LastExceptionInStreamer;
import org.apache.hadoop.hdfs.client.impl.DfsClientConf;
@@ -48,6 +52,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.PathUtils;
import org.apache.htrace.core.SpanId;
@@ -55,6 +60,8 @@ import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyLong;
import org.mockito.Mockito;
@@ -346,6 +353,24 @@ public class TestDFSOutputStream {
verify(spyClient, times(1)).endFileLease(anyLong());
}
+ @Test
+ public void testStreamFlush() throws Exception {
+ FileSystem fs = cluster.getFileSystem();
+ FSDataOutputStream os = fs.create(new Path("/normal-file"));
+ // Verify output stream supports hsync() and hflush().
+ assertTrue("DFSOutputStream should support hflush()!",
+ os.hasCapability(StreamCapability.HFLUSH.getValue()));
+ assertTrue("DFSOutputStream should support hsync()!",
+ os.hasCapability(StreamCapability.HSYNC.getValue()));
+ byte[] bytes = new byte[1024];
+ InputStream is = new ByteArrayInputStream(bytes);
+ IOUtils.copyBytes(is, os, bytes.length);
+ os.hflush();
+ IOUtils.copyBytes(is, os, bytes.length);
+ os.hsync();
+ os.close();
+ }
+
@AfterClass
public static void tearDown() {
if (cluster != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54fd0e44/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
index 70309c9..c0cfea2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.hdfs;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.fail;
+import static org.junit.Assert.assertFalse;
import java.io.ByteArrayInputStream;
import java.io.IOException;
@@ -29,6 +29,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StreamCapabilities.StreamCapability;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.io.IOUtils;
@@ -195,14 +196,19 @@ public class TestDFSStripedOutputStream {
public void testStreamFlush() throws Exception {
final byte[] bytes = StripedFileTestUtil.generateBytes(blockSize *
dataBlocks * 3 + cellSize * dataBlocks + cellSize + 123);
- try (FSDataOutputStream os = fs.create(new Path("/ec-file-1"))) {
- InputStream is = new ByteArrayInputStream(bytes);
- IOUtils.copyBytes(is, os, bytes.length);
- os.hflush();
- os.hsync();
- } catch (Exception e) {
- fail("hflush()/hsync() on striped file output stream failed!", e);
- }
+ FSDataOutputStream os = fs.create(new Path("/ec-file-1"));
+ assertFalse("DFSStripedOutputStream should not have hflush() " +
+ "capability yet!", os.hasCapability(
+ StreamCapability.HFLUSH.getValue()));
+ assertFalse("DFSStripedOutputStream should not have hsync() " +
+ "capability yet!", os.hasCapability(
+ StreamCapability.HSYNC.getValue()));
+ InputStream is = new ByteArrayInputStream(bytes);
+ IOUtils.copyBytes(is, os, bytes.length);
+ os.hflush();
+ IOUtils.copyBytes(is, os, bytes.length);
+ os.hsync();
+ os.close();
}
private void testOneFile(String src, int writeBytes) throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54fd0e44/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SyncableDataOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SyncableDataOutputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SyncableDataOutputStream.java
index 9bec7a5..a52fdb7 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SyncableDataOutputStream.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SyncableDataOutputStream.java
@@ -22,6 +22,7 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.io.OutputStream;
+import org.apache.hadoop.fs.StreamCapabilities;
import org.apache.hadoop.fs.Syncable;
/**
@@ -30,13 +31,22 @@ import org.apache.hadoop.fs.Syncable;
* wrapped stream passed in to the constructor. This is required
* for HBase when wrapping a PageBlobOutputStream used as a write-ahead log.
*/
-public class SyncableDataOutputStream extends DataOutputStream implements Syncable {
+public class SyncableDataOutputStream extends DataOutputStream
+ implements Syncable, StreamCapabilities {
public SyncableDataOutputStream(OutputStream out) {
super(out);
}
@Override
+ public boolean hasCapability(String capability) {
+ if (out instanceof StreamCapabilities) {
+ return ((StreamCapabilities) out).hasCapability(capability);
+ }
+ return false;
+ }
+
+ @Override
public void hflush() throws IOException {
if (out instanceof Syncable) {
((Syncable) out).hflush();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: HDFS-11546. Federation Router RPC
server. Contributed by Jason Kace and Inigo Goiri.
Posted by in...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6632c5a4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
index 24792bb..4bae71e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
@@ -17,16 +17,109 @@
*/
package org.apache.hadoop.hdfs.server.federation.router;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HANDLER_COUNT_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HANDLER_COUNT_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HANDLER_QUEUE_SIZE_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HANDLER_QUEUE_SIZE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_READER_COUNT_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_READER_COUNT_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_READER_QUEUE_SIZE_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_READER_QUEUE_SIZE_KEY;
+
+import java.io.FileNotFoundException;
import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.CryptoProtocolVersion;
+import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries;
+import org.apache.hadoop.fs.CacheFlag;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FsServerDefaults;
+import org.apache.hadoop.fs.Options;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.QuotaUsage;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.fs.XAttr;
+import org.apache.hadoop.fs.XAttrSetFlag;
+import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.AddBlockFlag;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.inotify.EventBatchList;
+import org.apache.hadoop.hdfs.protocol.AddingECPolicyResponse;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
+import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
+import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
+import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.EncryptionZone;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ClientNamenodeProtocol;
+import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB;
+import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB;
+import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException;
+import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
+import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException;
+import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
+import org.apache.hadoop.io.EnumSetWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RPC.Server;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.ipc.StandbyException;
+import org.apache.hadoop.net.NodeBase;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.service.AbstractService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
+import com.google.protobuf.BlockingService;
/**
* This class is responsible for handling all of the RPC calls to the It is
@@ -36,10 +129,42 @@ import com.google.common.annotations.VisibleForTesting;
* the requests to the active
* {@link org.apache.hadoop.hdfs.server.namenode.NameNode NameNode}.
*/
-public class RouterRpcServer extends AbstractService {
+public class RouterRpcServer extends AbstractService implements ClientProtocol {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RouterRpcServer.class);
+
+
+ /** Configuration for the RPC server. */
+ private Configuration conf;
+
+ /** Identifier for the super user. */
+ private final String superUser;
+ /** Identifier for the super group. */
+ private final String superGroup;
+
+ /** Router using this RPC server. */
+ private final Router router;
/** The RPC server that listens to requests from clients. */
private final Server rpcServer;
+ /** The address for this RPC server. */
+ private final InetSocketAddress rpcAddress;
+
+ /** RPC clients to connect to the Namenodes. */
+ private final RouterRpcClient rpcClient;
+
+
+ /** Interface to identify the active NN for a nameservice or blockpool ID. */
+ private final ActiveNamenodeResolver namenodeResolver;
+
+ /** Interface to map global name space to HDFS subcluster name spaces. */
+ private final FileSubclusterResolver subclusterResolver;
+
+
+ /** Category of the operation that a thread is executing. */
+ private final ThreadLocal<OperationCategory> opCategory = new ThreadLocal<>();
+
/**
* Construct a router RPC server.
@@ -54,31 +179,94 @@ public class RouterRpcServer extends AbstractService {
throws IOException {
super(RouterRpcServer.class.getName());
- this.rpcServer = null;
- }
+ this.conf = configuration;
+ this.router = router;
+ this.namenodeResolver = nnResolver;
+ this.subclusterResolver = fileResolver;
- /**
- * Allow access to the client RPC server for testing.
- *
- * @return The RPC server.
- */
- @VisibleForTesting
- public Server getServer() {
- return this.rpcServer;
+ // User and group for reporting
+ this.superUser = System.getProperty("user.name");
+ this.superGroup = this.conf.get(
+ DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_KEY,
+ DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT);
+
+ // RPC server settings
+ int handlerCount = this.conf.getInt(DFS_ROUTER_HANDLER_COUNT_KEY,
+ DFS_ROUTER_HANDLER_COUNT_DEFAULT);
+
+ int readerCount = this.conf.getInt(DFS_ROUTER_READER_COUNT_KEY,
+ DFS_ROUTER_READER_COUNT_DEFAULT);
+
+ int handlerQueueSize = this.conf.getInt(DFS_ROUTER_HANDLER_QUEUE_SIZE_KEY,
+ DFS_ROUTER_HANDLER_QUEUE_SIZE_DEFAULT);
+
+ // Override Hadoop Common IPC setting
+ int readerQueueSize = this.conf.getInt(DFS_ROUTER_READER_QUEUE_SIZE_KEY,
+ DFS_ROUTER_READER_QUEUE_SIZE_DEFAULT);
+ this.conf.setInt(
+ CommonConfigurationKeys.IPC_SERVER_RPC_READ_CONNECTION_QUEUE_SIZE_KEY,
+ readerQueueSize);
+
+ RPC.setProtocolEngine(this.conf, ClientNamenodeProtocolPB.class,
+ ProtobufRpcEngine.class);
+
+ ClientNamenodeProtocolServerSideTranslatorPB
+ clientProtocolServerTranslator =
+ new ClientNamenodeProtocolServerSideTranslatorPB(this);
+ BlockingService clientNNPbService = ClientNamenodeProtocol
+ .newReflectiveBlockingService(clientProtocolServerTranslator);
+
+ InetSocketAddress confRpcAddress = conf.getSocketAddr(
+ DFSConfigKeys.DFS_ROUTER_RPC_BIND_HOST_KEY,
+ DFSConfigKeys.DFS_ROUTER_RPC_ADDRESS_KEY,
+ DFSConfigKeys.DFS_ROUTER_RPC_ADDRESS_DEFAULT,
+ DFSConfigKeys.DFS_ROUTER_RPC_PORT_DEFAULT);
+ LOG.info("RPC server binding to {} with {} handlers for Router {}",
+ confRpcAddress, handlerCount, this.router.getRouterId());
+
+ this.rpcServer = new RPC.Builder(this.conf)
+ .setProtocol(ClientNamenodeProtocolPB.class)
+ .setInstance(clientNNPbService)
+ .setBindAddress(confRpcAddress.getHostName())
+ .setPort(confRpcAddress.getPort())
+ .setNumHandlers(handlerCount)
+ .setnumReaders(readerCount)
+ .setQueueSizePerHandler(handlerQueueSize)
+ .setVerbose(false)
+ .build();
+ // We don't want the server to log the full stack trace for some exceptions
+ this.rpcServer.addTerseExceptions(
+ RemoteException.class,
+ StandbyException.class,
+ SafeModeException.class,
+ FileNotFoundException.class,
+ FileAlreadyExistsException.class,
+ AccessControlException.class,
+ LeaseExpiredException.class,
+ NotReplicatedYetException.class,
+ IOException.class);
+
+ // The RPC-server port can be ephemeral... ensure we have the correct info
+ InetSocketAddress listenAddress = this.rpcServer.getListenerAddress();
+ this.rpcAddress = new InetSocketAddress(
+ confRpcAddress.getHostName(), listenAddress.getPort());
+
+ // Create the client
+ this.rpcClient = new RouterRpcClient(this.conf, this.router.getRouterId(),
+ this.namenodeResolver);
}
@Override
protected void serviceInit(Configuration configuration) throws Exception {
+ this.conf = configuration;
super.serviceInit(configuration);
}
- /**
- * Start client and service RPC servers.
- */
@Override
protected void serviceStart() throws Exception {
if (this.rpcServer != null) {
this.rpcServer.start();
+ LOG.info("Router RPC up at: {}", this.getRpcAddress());
}
super.serviceStart();
}
@@ -92,11 +280,1652 @@ public class RouterRpcServer extends AbstractService {
}
/**
- * Wait until the RPC servers have shutdown.
+ * Get the RPC client to the Namenode.
+ *
+ * @return RPC clients to the Namenodes.
*/
- void join() throws InterruptedException {
- if (this.rpcServer != null) {
- this.rpcServer.join();
+ public RouterRpcClient getRPCClient() {
+ return rpcClient;
+ }
+
+ /**
+ * Allow access to the client RPC server for testing.
+ *
+ * @return The RPC server.
+ */
+ @VisibleForTesting
+ public Server getServer() {
+ return rpcServer;
+ }
+
+ /**
+ * Get the RPC address of the service.
+ *
+ * @return RPC service address.
+ */
+ public InetSocketAddress getRpcAddress() {
+ return rpcAddress;
+ }
+
+ /**
+ * Check if the Router is in safe mode. We should only see READ, WRITE, and
+ * UNCHECKED. It includes a default handler when we haven't implemented an
+ * operation. If not supported, it always throws an exception reporting the
+ * operation.
+ *
+ * @param op Category of the operation to check.
+ * @param supported If the operation is supported or not. If not, it will
+ * throw an UnsupportedOperationException.
+ * @throws StandbyException If the Router is in safe mode and cannot serve
+ * client requests.
+ * @throws UnsupportedOperationException If the operation is not supported.
+ */
+ private void checkOperation(OperationCategory op, boolean supported)
+ throws StandbyException, UnsupportedOperationException {
+ checkOperation(op);
+
+ if (!supported) {
+ String methodName = getMethodName();
+ throw new UnsupportedOperationException(
+ "Operation \"" + methodName + "\" is not supported");
+ }
+ }
+
+ /**
+ * Check if the Router is in safe mode. We should only see READ, WRITE, and
+ * UNCHECKED. This function should be called by all ClientProtocol functions.
+ *
+ * @param op Category of the operation to check.
+ * @throws StandbyException If the Router is in safe mode and cannot serve
+ * client requests.
+ */
+ private void checkOperation(OperationCategory op) throws StandbyException {
+ // Log the function we are currently calling.
+ if (LOG.isDebugEnabled()) {
+ String methodName = getMethodName();
+ LOG.debug("Proxying operation: {}", methodName);
+ }
+
+ // Store the category of the operation category for this thread
+ opCategory.set(op);
+
+ // We allow unchecked and read operations
+ if (op == OperationCategory.UNCHECKED || op == OperationCategory.READ) {
+ return;
+ }
+
+ // TODO check Router safe mode and return Standby exception
+ }
+
+ @Override // ClientProtocol
+ public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ return null;
+ }
+
+ /**
+ * The the delegation token from each name service.
+ * @param renewer
+ * @return Name service -> Token.
+ * @throws IOException
+ */
+ public Map<FederationNamespaceInfo, Token<DelegationTokenIdentifier>>
+ getDelegationTokens(Text renewer) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ return null;
+ }
+
+ @Override // ClientProtocol
+ public long renewDelegationToken(Token<DelegationTokenIdentifier> token)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ return 0;
+ }
+
+ @Override // ClientProtocol
+ public void cancelDelegationToken(Token<DelegationTokenIdentifier> token)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // ClientProtocol
+ public LocatedBlocks getBlockLocations(String src, final long offset,
+ final long length) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod remoteMethod = new RemoteMethod("getBlockLocations",
+ new Class<?>[] {String.class, long.class, long.class},
+ new RemoteParam(), offset, length);
+ return (LocatedBlocks) rpcClient.invokeSequential(locations, remoteMethod,
+ LocatedBlocks.class, null);
+ }
+
+ @Override // ClientProtocol
+ public FsServerDefaults getServerDefaults() throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ RemoteMethod method = new RemoteMethod("getServerDefaults");
+ String ns = subclusterResolver.getDefaultNamespace();
+ return (FsServerDefaults) rpcClient.invokeSingle(ns, method);
+ }
+
+ @Override // ClientProtocol
+ public HdfsFileStatus create(String src, FsPermission masked,
+ String clientName, EnumSetWritable<CreateFlag> flag,
+ boolean createParent, short replication, long blockSize,
+ CryptoProtocolVersion[] supportedVersions, String ecPolicyName)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ RemoteLocation createLocation = getCreateLocation(src);
+ RemoteMethod method = new RemoteMethod("create",
+ new Class<?>[] {String.class, FsPermission.class, String.class,
+ EnumSetWritable.class, boolean.class, short.class,
+ long.class, CryptoProtocolVersion[].class,
+ String.class},
+ createLocation.getDest(), masked, clientName, flag, createParent,
+ replication, blockSize, supportedVersions, ecPolicyName);
+ return (HdfsFileStatus) rpcClient.invokeSingle(createLocation, method);
+ }
+
+ /**
+ * Get the location to create a file. It checks if the file already existed
+ * in one of the locations.
+ *
+ * @param src Path of the file to check.
+ * @return The remote location for this file.
+ * @throws IOException If the file has no creation location.
+ */
+ private RemoteLocation getCreateLocation(final String src)
+ throws IOException {
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ if (locations == null || locations.isEmpty()) {
+ throw new IOException("Cannot get locations to create " + src);
+ }
+
+ RemoteLocation createLocation = locations.get(0);
+ if (locations.size() > 1) {
+ try {
+ // Check if this file already exists in other subclusters
+ LocatedBlocks existingLocation = getBlockLocations(src, 0, 1);
+ if (existingLocation != null) {
+ // Forward to the existing location and let the NN handle the error
+ LocatedBlock existingLocationLastLocatedBlock =
+ existingLocation.getLastLocatedBlock();
+ if (existingLocationLastLocatedBlock == null) {
+ // The block has no blocks yet, check for the meta data
+ for (RemoteLocation location : locations) {
+ RemoteMethod method = new RemoteMethod("getFileInfo",
+ new Class<?>[] {String.class}, new RemoteParam());
+ if (rpcClient.invokeSingle(location, method) != null) {
+ createLocation = location;
+ break;
+ }
+ }
+ } else {
+ ExtendedBlock existingLocationLastBlock =
+ existingLocationLastLocatedBlock.getBlock();
+ String blockPoolId = existingLocationLastBlock.getBlockPoolId();
+ createLocation = getLocationForPath(src, true, blockPoolId);
+ }
+ }
+ } catch (FileNotFoundException fne) {
+ // Ignore if the file is not found
+ }
+ }
+ return createLocation;
+ }
+
+ // Medium
+ @Override // ClientProtocol
+ public LastBlockWithStatus append(String src, final String clientName,
+ final EnumSetWritable<CreateFlag> flag) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("append",
+ new Class<?>[] {String.class, String.class, EnumSetWritable.class},
+ new RemoteParam(), clientName, flag);
+ return (LastBlockWithStatus) rpcClient.invokeSequential(
+ locations, method, LastBlockWithStatus.class, null);
+ }
+
+ // Low
+ @Override // ClientProtocol
+ public boolean recoverLease(String src, String clientName)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("recoverLease",
+ new Class<?>[] {String.class, String.class}, new RemoteParam(),
+ clientName);
+ Object result = rpcClient.invokeSequential(
+ locations, method, Boolean.class, Boolean.TRUE);
+ return (boolean) result;
+ }
+
+ @Override // ClientProtocol
+ public boolean setReplication(String src, short replication)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("setReplication",
+ new Class<?>[] {String.class, short.class}, new RemoteParam(),
+ replication);
+ Object result = rpcClient.invokeSequential(
+ locations, method, Boolean.class, Boolean.TRUE);
+ return (boolean) result;
+ }
+
+ @Override
+ public void setStoragePolicy(String src, String policyName)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("setStoragePolicy",
+ new Class<?>[] {String.class, String.class},
+ new RemoteParam(), policyName);
+ rpcClient.invokeSequential(locations, method, null, null);
+ }
+
+ @Override
+ public BlockStoragePolicy[] getStoragePolicies() throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ RemoteMethod method = new RemoteMethod("getStoragePolicies");
+ String ns = subclusterResolver.getDefaultNamespace();
+ return (BlockStoragePolicy[]) rpcClient.invokeSingle(ns, method);
+ }
+
+ @Override // ClientProtocol
+ public void setPermission(String src, FsPermission permissions)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("setPermission",
+ new Class<?>[] {String.class, FsPermission.class},
+ new RemoteParam(), permissions);
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @Override // ClientProtocol
+ public void setOwner(String src, String username, String groupname)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("setOwner",
+ new Class<?>[] {String.class, String.class, String.class},
+ new RemoteParam(), username, groupname);
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ /**
+ * Excluded and favored nodes are not verified and will be ignored by
+ * placement policy if they are not in the same nameservice as the file.
+ */
+ @Override // ClientProtocol
+ public LocatedBlock addBlock(String src, String clientName,
+ ExtendedBlock previous, DatanodeInfo[] excludedNodes, long fileId,
+ String[] favoredNodes, EnumSet<AddBlockFlag> addBlockFlags)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("addBlock",
+ new Class<?>[] {String.class, String.class, ExtendedBlock.class,
+ DatanodeInfo[].class, long.class, String[].class,
+ EnumSet.class},
+ new RemoteParam(), clientName, previous, excludedNodes, fileId,
+ favoredNodes, addBlockFlags);
+ // TODO verify the excludedNodes and favoredNodes are acceptable to this NN
+ return (LocatedBlock) rpcClient.invokeSequential(
+ locations, method, LocatedBlock.class, null);
+ }
+
+ /**
+ * Excluded nodes are not verified and will be ignored by placement if they
+ * are not in the same nameservice as the file.
+ */
+ @Override // ClientProtocol
+ public LocatedBlock getAdditionalDatanode(final String src, final long fileId,
+ final ExtendedBlock blk, final DatanodeInfo[] existings,
+ final String[] existingStorageIDs, final DatanodeInfo[] excludes,
+ final int numAdditionalNodes, final String clientName)
+ throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod method = new RemoteMethod("getAdditionalDatanode",
+ new Class<?>[] {String.class, long.class, ExtendedBlock.class,
+ DatanodeInfo[].class, String[].class,
+ DatanodeInfo[].class, int.class, String.class},
+ new RemoteParam(), fileId, blk, existings, existingStorageIDs, excludes,
+ numAdditionalNodes, clientName);
+ return (LocatedBlock) rpcClient.invokeSequential(
+ locations, method, LocatedBlock.class, null);
+ }
+
+ @Override // ClientProtocol
+ public void abandonBlock(ExtendedBlock b, long fileId, String src,
+ String holder) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ RemoteMethod method = new RemoteMethod("abandonBlock",
+ new Class<?>[] {ExtendedBlock.class, long.class, String.class,
+ String.class},
+ b, fileId, new RemoteParam(), holder);
+ rpcClient.invokeSingle(b, method);
+ }
+
+ @Override // ClientProtocol
+ public boolean complete(String src, String clientName, ExtendedBlock last,
+ long fileId) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("complete",
+ new Class<?>[] {String.class, String.class, ExtendedBlock.class,
+ long.class},
+ new RemoteParam(), clientName, last, fileId);
+ // Complete can return true/false, so don't expect a result
+ return ((Boolean) rpcClient.invokeSequential(
+ locations, method, Boolean.class, null)).booleanValue();
+ }
+
+ @Override // ClientProtocol
+ public LocatedBlock updateBlockForPipeline(
+ ExtendedBlock block, String clientName) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ RemoteMethod method = new RemoteMethod("updateBlockForPipeline",
+ new Class<?>[] {ExtendedBlock.class, String.class},
+ block, clientName);
+ return (LocatedBlock) rpcClient.invokeSingle(block, method);
+ }
+
+ /**
+ * Datanode are not verified to be in the same nameservice as the old block.
+ * TODO This may require validation.
+ */
+ @Override // ClientProtocol
+ public void updatePipeline(String clientName, ExtendedBlock oldBlock,
+ ExtendedBlock newBlock, DatanodeID[] newNodes, String[] newStorageIDs)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ RemoteMethod method = new RemoteMethod("updatePipeline",
+ new Class<?>[] {String.class, ExtendedBlock.class, ExtendedBlock.class,
+ DatanodeID[].class, String[].class},
+ clientName, oldBlock, newBlock, newNodes, newStorageIDs);
+ rpcClient.invokeSingle(oldBlock, method);
+ }
+
+ @Override // ClientProtocol
+ public long getPreferredBlockSize(String src) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("getPreferredBlockSize",
+ new Class<?>[] {String.class}, new RemoteParam());
+ return ((Long) rpcClient.invokeSequential(
+ locations, method, Long.class, null)).longValue();
+ }
+
+ /**
+ * Determines combinations of eligible src/dst locations for a rename. A
+ * rename cannot change the namespace. Renames are only allowed if there is an
+ * eligible dst location in the same namespace as the source.
+ *
+ * @param srcLocations List of all potential source destinations where the
+ * path may be located. On return this list is trimmed to include
+ * only the paths that have corresponding destinations in the same
+ * namespace.
+ * @param dst The destination path
+ * @return A map of all eligible source namespaces and their corresponding
+ * replacement value.
+ * @throws IOException If the dst paths could not be determined.
+ */
+ private RemoteParam getRenameDestinations(
+ final List<RemoteLocation> srcLocations, final String dst)
+ throws IOException {
+
+ final List<RemoteLocation> dstLocations = getLocationsForPath(dst, true);
+ final Map<RemoteLocation, String> dstMap = new HashMap<>();
+
+ Iterator<RemoteLocation> iterator = srcLocations.iterator();
+ while (iterator.hasNext()) {
+ RemoteLocation srcLocation = iterator.next();
+ RemoteLocation eligibleDst =
+ getFirstMatchingLocation(srcLocation, dstLocations);
+ if (eligibleDst != null) {
+ // Use this dst for this source location
+ dstMap.put(srcLocation, eligibleDst.getDest());
+ } else {
+ // This src destination is not valid, remove from the source list
+ iterator.remove();
+ }
+ }
+ return new RemoteParam(dstMap);
+ }
+
+ /**
+ * Get first matching location.
+ *
+ * @param location Location we are looking for.
+ * @param locations List of locations.
+ * @return The first matchin location in the list.
+ */
+ private RemoteLocation getFirstMatchingLocation(RemoteLocation location,
+ List<RemoteLocation> locations) {
+ for (RemoteLocation loc : locations) {
+ if (loc.getNameserviceId().equals(location.getNameserviceId())) {
+ // Return first matching location
+ return loc;
+ }
+ }
+ return null;
+ }
+
+ @Deprecated
+ @Override // ClientProtocol
+ public boolean rename(final String src, final String dst)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> srcLocations = getLocationsForPath(src, true);
+ // srcLocations may be trimmed by getRenameDestinations()
+ final List<RemoteLocation> locs = new LinkedList<>(srcLocations);
+ RemoteParam dstParam = getRenameDestinations(locs, dst);
+ if (locs.isEmpty()) {
+ throw new IOException(
+ "Rename of " + src + " to " + dst + " is not allowed," +
+ " no eligible destination in the same namespace was found.");
+ }
+ RemoteMethod method = new RemoteMethod("rename",
+ new Class<?>[] {String.class, String.class},
+ new RemoteParam(), dstParam);
+ return ((Boolean) rpcClient.invokeSequential(
+ locs, method, Boolean.class, Boolean.TRUE)).booleanValue();
+ }
+
+ @Override // ClientProtocol
+ public void rename2(final String src, final String dst,
+ final Options.Rename... options) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> srcLocations = getLocationsForPath(src, true);
+ // srcLocations may be trimmed by getRenameDestinations()
+ final List<RemoteLocation> locs = new LinkedList<>(srcLocations);
+ RemoteParam dstParam = getRenameDestinations(locs, dst);
+ if (locs.isEmpty()) {
+ throw new IOException(
+ "Rename of " + src + " to " + dst + " is not allowed," +
+ " no eligible destination in the same namespace was found.");
+ }
+ RemoteMethod method = new RemoteMethod("rename2",
+ new Class<?>[] {String.class, String.class, options.getClass()},
+ new RemoteParam(), dstParam, options);
+ rpcClient.invokeSequential(locs, method, null, null);
+ }
+
+ @Override // ClientProtocol
+ public void concat(String trg, String[] src) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // See if the src and target files are all in the same namespace
+ LocatedBlocks targetBlocks = getBlockLocations(trg, 0, 1);
+ if (targetBlocks == null) {
+ throw new IOException("Cannot locate blocks for target file - " + trg);
+ }
+ LocatedBlock lastLocatedBlock = targetBlocks.getLastLocatedBlock();
+ String targetBlockPoolId = lastLocatedBlock.getBlock().getBlockPoolId();
+ for (String source : src) {
+ LocatedBlocks sourceBlocks = getBlockLocations(source, 0, 1);
+ if (sourceBlocks == null) {
+ throw new IOException(
+ "Cannot located blocks for source file " + source);
+ }
+ String sourceBlockPoolId =
+ sourceBlocks.getLastLocatedBlock().getBlock().getBlockPoolId();
+ if (!sourceBlockPoolId.equals(targetBlockPoolId)) {
+ throw new IOException("Cannot concatenate source file " + source
+ + " because it is located in a different namespace"
+ + " with block pool id " + sourceBlockPoolId
+ + " from the target file with block pool id "
+ + targetBlockPoolId);
+ }
+ }
+
+ // Find locations in the matching namespace.
+ final RemoteLocation targetDestination =
+ getLocationForPath(trg, true, targetBlockPoolId);
+ String[] sourceDestinations = new String[src.length];
+ for (int i = 0; i < src.length; i++) {
+ String sourceFile = src[i];
+ RemoteLocation location =
+ getLocationForPath(sourceFile, true, targetBlockPoolId);
+ sourceDestinations[i] = location.getDest();
+ }
+ // Invoke
+ RemoteMethod method = new RemoteMethod("concat",
+ new Class<?>[] {String.class, String[].class},
+ targetDestination.getDest(), sourceDestinations);
+ rpcClient.invokeSingle(targetDestination, method);
+ }
+
+ @Override // ClientProtocol
+ public boolean truncate(String src, long newLength, String clientName)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("truncate",
+ new Class<?>[] {String.class, long.class, String.class},
+ new RemoteParam(), newLength, clientName);
+ return ((Boolean) rpcClient.invokeSequential(locations, method,
+ Boolean.class, Boolean.TRUE)).booleanValue();
+ }
+
+ @Override // ClientProtocol
+ public boolean delete(String src, boolean recursive) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("delete",
+ new Class<?>[] {String.class, boolean.class}, new RemoteParam(),
+ recursive);
+ return ((Boolean) rpcClient.invokeSequential(locations, method,
+ Boolean.class, Boolean.TRUE)).booleanValue();
+ }
+
+ @Override // ClientProtocol
+ public boolean mkdirs(String src, FsPermission masked, boolean createParent)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ if (locations.size() > 1) {
+ // Check if this directory already exists
+ try {
+ HdfsFileStatus fileStatus = getFileInfo(src);
+ if (fileStatus != null) {
+ // When existing, the NN doesn't return an exception; return true
+ return true;
+ }
+ } catch (IOException ioe) {
+ // Can't query if this file exists or not.
+ LOG.error("Error requesting file info for path {} while proxing mkdirs",
+ src, ioe);
+ }
+ }
+
+ RemoteLocation firstLocation = locations.get(0);
+ RemoteMethod method = new RemoteMethod("mkdirs",
+ new Class<?>[] {String.class, FsPermission.class, boolean.class},
+ new RemoteParam(), masked, createParent);
+ return ((Boolean) rpcClient.invokeSingle(firstLocation, method))
+ .booleanValue();
+ }
+
+ @Override // ClientProtocol
+ public void renewLease(String clientName) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ RemoteMethod method = new RemoteMethod("renewLease",
+ new Class<?>[] {String.class}, clientName);
+ Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ rpcClient.invokeConcurrent(nss, method, false, false);
+ }
+
+ @Override // ClientProtocol
+ public DirectoryListing getListing(String src, byte[] startAfter,
+ boolean needLocation) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ // Locate the dir and fetch the listing
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("getListing",
+ new Class<?>[] {String.class, startAfter.getClass(), boolean.class},
+ new RemoteParam(), startAfter, needLocation);
+ Map<RemoteLocation, Object> listings =
+ rpcClient.invokeConcurrent(locations, method, false, false);
+
+ Map<String, HdfsFileStatus> nnListing = new TreeMap<>();
+ int totalRemainingEntries = 0;
+ int remainingEntries = 0;
+ boolean namenodeListingExists = false;
+ if (listings != null) {
+ // Check the subcluster listing with the smallest name
+ String lastName = null;
+ for (Entry<RemoteLocation, Object> entry : listings.entrySet()) {
+ RemoteLocation location = entry.getKey();
+ DirectoryListing listing = (DirectoryListing) entry.getValue();
+ if (listing == null) {
+ LOG.debug("Cannot get listing from {}", location);
+ } else {
+ totalRemainingEntries += listing.getRemainingEntries();
+ HdfsFileStatus[] partialListing = listing.getPartialListing();
+ int length = partialListing.length;
+ if (length > 0) {
+ HdfsFileStatus lastLocalEntry = partialListing[length-1];
+ String lastLocalName = lastLocalEntry.getLocalName();
+ if (lastName == null || lastName.compareTo(lastLocalName) > 0) {
+ lastName = lastLocalName;
+ }
+ }
+ }
+ }
+
+ // Add existing entries
+ for (Object value : listings.values()) {
+ DirectoryListing listing = (DirectoryListing) value;
+ if (listing != null) {
+ namenodeListingExists = true;
+ for (HdfsFileStatus file : listing.getPartialListing()) {
+ String filename = file.getLocalName();
+ if (totalRemainingEntries > 0 && filename.compareTo(lastName) > 0) {
+ // Discarding entries further than the lastName
+ remainingEntries++;
+ } else {
+ nnListing.put(filename, file);
+ }
+ }
+ remainingEntries += listing.getRemainingEntries();
+ }
+ }
+ }
+
+ // Add mount points at this level in the tree
+ final List<String> children = subclusterResolver.getMountPoints(src);
+ if (children != null) {
+ // Get the dates for each mount point
+ Map<String, Long> dates = getMountPointDates(src);
+
+ // Create virtual folder with the mount name
+ for (String child : children) {
+ long date = 0;
+ if (dates != null && dates.containsKey(child)) {
+ date = dates.get(child);
+ }
+ // TODO add number of children
+ HdfsFileStatus dirStatus = getMountPointStatus(child, 0, date);
+
+ // This may overwrite existing listing entries with the mount point
+ // TODO don't add if already there?
+ nnListing.put(child, dirStatus);
+ }
+ }
+
+ if (!namenodeListingExists && nnListing.size() == 0) {
+ // NN returns a null object if the directory cannot be found and has no
+ // listing. If we didn't retrieve any NN listing data, and there are no
+ // mount points here, return null.
+ return null;
+ }
+
+ // Generate combined listing
+ HdfsFileStatus[] combinedData = new HdfsFileStatus[nnListing.size()];
+ combinedData = nnListing.values().toArray(combinedData);
+ return new DirectoryListing(combinedData, remainingEntries);
+ }
+
+ @Override // ClientProtocol
+ public HdfsFileStatus getFileInfo(String src) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod method = new RemoteMethod("getFileInfo",
+ new Class<?>[] {String.class}, new RemoteParam());
+ HdfsFileStatus ret = (HdfsFileStatus) rpcClient.invokeSequential(
+ locations, method, HdfsFileStatus.class, null);
+
+ // If there is no real path, check mount points
+ if (ret == null) {
+ List<String> children = subclusterResolver.getMountPoints(src);
+ if (children != null && !children.isEmpty()) {
+ Map<String, Long> dates = getMountPointDates(src);
+ long date = 0;
+ if (dates != null && dates.containsKey(src)) {
+ date = dates.get(src);
+ }
+ ret = getMountPointStatus(src, children.size(), date);
+ }
+ }
+
+ return ret;
+ }
+
+ @Override // ClientProtocol
+ public boolean isFileClosed(String src) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod method = new RemoteMethod("isFileClosed",
+ new Class<?>[] {String.class}, new RemoteParam());
+ return ((Boolean) rpcClient.invokeSequential(
+ locations, method, Boolean.class, Boolean.TRUE)).booleanValue();
+ }
+
+ @Override // ClientProtocol
+ public HdfsFileStatus getFileLinkInfo(String src) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod method = new RemoteMethod("getFileLinkInfo",
+ new Class<?>[] {String.class}, new RemoteParam());
+ return (HdfsFileStatus) rpcClient.invokeSequential(
+ locations, method, HdfsFileStatus.class, null);
+ }
+
+ @Override // ClientProtocol
+ public long[] getStats() throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+
+ RemoteMethod method = new RemoteMethod("getStats");
+ Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, Object> results =
+ rpcClient.invokeConcurrent(nss, method, true, false);
+ long[] combinedData = new long[STATS_ARRAY_LENGTH];
+ for (Object o : results.values()) {
+ long[] data = (long[]) o;
+ for (int i = 0; i < combinedData.length && i < data.length; i++) {
+ if (data[i] >= 0) {
+ combinedData[i] += data[i];
+ }
+ }
+ }
+ return combinedData;
+ }
+
+ @Override // ClientProtocol
+ public DatanodeInfo[] getDatanodeReport(DatanodeReportType type)
+ throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+
+ Map<String, DatanodeInfo> datanodesMap = new LinkedHashMap<>();
+ RemoteMethod method = new RemoteMethod("getDatanodeReport",
+ new Class<?>[] {DatanodeReportType.class}, type);
+
+ Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, Object> results =
+ rpcClient.invokeConcurrent(nss, method, true, false);
+ for (Entry<FederationNamespaceInfo, Object> entry : results.entrySet()) {
+ FederationNamespaceInfo ns = entry.getKey();
+ DatanodeInfo[] result = (DatanodeInfo[]) entry.getValue();
+ for (DatanodeInfo node : result) {
+ String nodeId = node.getXferAddr();
+ if (!datanodesMap.containsKey(nodeId)) {
+ // Add the subcluster as a suffix to the network location
+ node.setNetworkLocation(
+ NodeBase.PATH_SEPARATOR_STR + ns.getNameserviceId() +
+ node.getNetworkLocation());
+ datanodesMap.put(nodeId, node);
+ } else {
+ LOG.debug("{} is in multiple subclusters", nodeId);
+ }
+ }
+ }
+ // Map -> Array
+ Collection<DatanodeInfo> datanodes = datanodesMap.values();
+ DatanodeInfo[] combinedData = new DatanodeInfo[datanodes.size()];
+ combinedData = datanodes.toArray(combinedData);
+ return combinedData;
+ }
+
+ @Override // ClientProtocol
+ public DatanodeStorageReport[] getDatanodeStorageReport(
+ DatanodeReportType type) throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+
+ Map<String, DatanodeStorageReport> datanodesMap = new HashMap<>();
+ RemoteMethod method = new RemoteMethod("getDatanodeStorageReport",
+ new Class<?>[] {DatanodeReportType.class}, type);
+ Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, Object> results =
+ rpcClient.invokeConcurrent(nss, method, true, false);
+ for (Object r : results.values()) {
+ DatanodeStorageReport[] result = (DatanodeStorageReport[]) r;
+ for (DatanodeStorageReport node : result) {
+ String nodeId = node.getDatanodeInfo().getXferAddr();
+ if (!datanodesMap.containsKey(nodeId)) {
+ datanodesMap.put(nodeId, node);
+ }
+ // TODO merge somehow, right now it just takes the first one
+ }
+ }
+
+ Collection<DatanodeStorageReport> datanodes = datanodesMap.values();
+ // TODO sort somehow
+ DatanodeStorageReport[] combinedData =
+ new DatanodeStorageReport[datanodes.size()];
+ combinedData = datanodes.toArray(combinedData);
+ return combinedData;
+ }
+
+ @Override // ClientProtocol
+ public boolean setSafeMode(SafeModeAction action, boolean isChecked)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // Set safe mode in all the name spaces
+ RemoteMethod method = new RemoteMethod("setSafeMode",
+ new Class<?>[] {SafeModeAction.class, boolean.class},
+ action, isChecked);
+ Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, Object> results =
+ rpcClient.invokeConcurrent(nss, method, true, true);
+
+ // We only report true if all the name space are in safe mode
+ int numSafemode = 0;
+ for (Object result : results.values()) {
+ if (result instanceof Boolean) {
+ boolean safemode = (boolean) result;
+ if (safemode) {
+ numSafemode++;
+ }
+ }
}
+ return numSafemode == results.size();
+ }
+
+ @Override // ClientProtocol
+ public boolean restoreFailedStorage(String arg) throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+
+ RemoteMethod method = new RemoteMethod("restoreFailedStorage",
+ new Class<?>[] {String.class}, arg);
+ final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, Object> ret =
+ rpcClient.invokeConcurrent(nss, method, true, false);
+
+ boolean success = true;
+ Object obj = ret;
+ @SuppressWarnings("unchecked")
+ Map<FederationNamespaceInfo, Boolean> results =
+ (Map<FederationNamespaceInfo, Boolean>)obj;
+ Collection<Boolean> sucesses = results.values();
+ for (boolean s : sucesses) {
+ if (!s) {
+ success = false;
+ }
+ }
+ return success;
+ }
+
+ @Override // ClientProtocol
+ public boolean saveNamespace(long timeWindow, long txGap) throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+
+ RemoteMethod method = new RemoteMethod("saveNamespace",
+ new Class<?>[] {Long.class, Long.class}, timeWindow, txGap);
+ final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, Object> ret =
+ rpcClient.invokeConcurrent(nss, method, true, false);
+
+ boolean success = true;
+ Object obj = ret;
+ @SuppressWarnings("unchecked")
+ Map<FederationNamespaceInfo, Boolean> results =
+ (Map<FederationNamespaceInfo, Boolean>)obj;
+ Collection<Boolean> sucesses = results.values();
+ for (boolean s : sucesses) {
+ if (!s) {
+ success = false;
+ }
+ }
+ return success;
+ }
+
+ @Override // ClientProtocol
+ public long rollEdits() throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ RemoteMethod method = new RemoteMethod("rollEdits", new Class<?>[] {});
+ final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, Object> ret =
+ rpcClient.invokeConcurrent(nss, method, true, false);
+
+ // Return the maximum txid
+ long txid = 0;
+ Object obj = ret;
+ @SuppressWarnings("unchecked")
+ Map<FederationNamespaceInfo, Long> results =
+ (Map<FederationNamespaceInfo, Long>)obj;
+ Collection<Long> txids = results.values();
+ for (long t : txids) {
+ if (t > txid) {
+ txid = t;
+ }
+ }
+ return txid;
+ }
+
+ @Override // ClientProtocol
+ public void refreshNodes() throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+
+ RemoteMethod method = new RemoteMethod("refreshNodes", new Class<?>[] {});
+ final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ rpcClient.invokeConcurrent(nss, method, true, true);
+ }
+
+ @Override // ClientProtocol
+ public void finalizeUpgrade() throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+
+ RemoteMethod method = new RemoteMethod("finalizeUpgrade",
+ new Class<?>[] {});
+ final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ rpcClient.invokeConcurrent(nss, method, true, false);
+ }
+
+ @Override // ClientProtocol
+ public RollingUpgradeInfo rollingUpgrade(RollingUpgradeAction action)
+ throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ RemoteMethod method = new RemoteMethod("rollingUpgrade",
+ new Class<?>[] {RollingUpgradeAction.class}, action);
+ final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, Object> ret =
+ rpcClient.invokeConcurrent(nss, method, true, false);
+
+ // Return the first rolling upgrade info
+ RollingUpgradeInfo info = null;
+ Object obj = ret;
+ @SuppressWarnings("unchecked")
+ Map<FederationNamespaceInfo, RollingUpgradeInfo> results =
+ (Map<FederationNamespaceInfo, RollingUpgradeInfo>)obj;
+ Collection<RollingUpgradeInfo> infos = results.values();
+ for (RollingUpgradeInfo infoNs : infos) {
+ if (info == null && infoNs != null) {
+ info = infoNs;
+ }
+ }
+ return info;
+ }
+
+ @Override // ClientProtocol
+ public void metaSave(String filename) throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+
+ RemoteMethod method = new RemoteMethod("metaSave",
+ new Class<?>[] {String.class}, filename);
+ final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ rpcClient.invokeConcurrent(nss, method, true, false);
+ }
+
+ @Override // ClientProtocol
+ public CorruptFileBlocks listCorruptFileBlocks(String path, String cookie)
+ throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ final List<RemoteLocation> locations = getLocationsForPath(path, false);
+ RemoteMethod method = new RemoteMethod("listCorruptFileBlocks",
+ new Class<?>[] {String.class, String.class},
+ new RemoteParam(), cookie);
+ return (CorruptFileBlocks) rpcClient.invokeSequential(
+ locations, method, CorruptFileBlocks.class, null);
+ }
+
+ @Override // ClientProtocol
+ public void setBalancerBandwidth(long bandwidth) throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+
+ RemoteMethod method = new RemoteMethod("setBalancerBandwidth",
+ new Class<?>[] {Long.class}, bandwidth);
+ final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ rpcClient.invokeConcurrent(nss, method, true, false);
+ }
+
+ @Override // ClientProtocol
+ public ContentSummary getContentSummary(String path) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ // Get the summaries from regular files
+ Collection<ContentSummary> summaries = new LinkedList<>();
+ FileNotFoundException notFoundException = null;
+ try {
+ final List<RemoteLocation> locations = getLocationsForPath(path, false);
+ RemoteMethod method = new RemoteMethod("getContentSummary",
+ new Class<?>[] {String.class}, new RemoteParam());
+ @SuppressWarnings("unchecked")
+ Map<String, ContentSummary> results =
+ (Map<String, ContentSummary>) ((Object)rpcClient.invokeConcurrent(
+ locations, method, false, false));
+ summaries.addAll(results.values());
+ } catch (FileNotFoundException e) {
+ notFoundException = e;
+ }
+
+ // Add mount points at this level in the tree
+ final List<String> children = subclusterResolver.getMountPoints(path);
+ if (children != null) {
+ for (String child : children) {
+ Path childPath = new Path(path, child);
+ try {
+ ContentSummary mountSummary = getContentSummary(childPath.toString());
+ if (mountSummary != null) {
+ summaries.add(mountSummary);
+ }
+ } catch (Exception e) {
+ LOG.error("Cannot get content summary for mount {}: {}",
+ childPath, e.getMessage());
+ }
+ }
+ }
+
+ // Throw original exception if no original nor mount points
+ if (summaries.isEmpty() && notFoundException != null) {
+ throw notFoundException;
+ }
+
+ return aggregateContentSummary(summaries);
+ }
+
+ /**
+ * Aggregate content summaries for each subcluster.
+ *
+ * @param results Collection of individual summaries.
+ * @return Aggregated content summary.
+ */
+ private ContentSummary aggregateContentSummary(
+ Collection<ContentSummary> summaries) {
+ if (summaries.size() == 1) {
+ return summaries.iterator().next();
+ }
+
+ long length = 0;
+ long fileCount = 0;
+ long directoryCount = 0;
+ long quota = 0;
+ long spaceConsumed = 0;
+ long spaceQuota = 0;
+
+ for (ContentSummary summary : summaries) {
+ length += summary.getLength();
+ fileCount += summary.getFileCount();
+ directoryCount += summary.getDirectoryCount();
+ quota += summary.getQuota();
+ spaceConsumed += summary.getSpaceConsumed();
+ spaceQuota += summary.getSpaceQuota();
+ }
+
+ ContentSummary ret = new ContentSummary.Builder()
+ .length(length)
+ .fileCount(fileCount)
+ .directoryCount(directoryCount)
+ .quota(quota)
+ .spaceConsumed(spaceConsumed)
+ .spaceQuota(spaceQuota)
+ .build();
+ return ret;
+ }
+
+ @Override // ClientProtocol
+ public void fsync(String src, long fileId, String clientName,
+ long lastBlockLength) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("fsync",
+ new Class<?>[] {String.class, long.class, String.class, long.class },
+ new RemoteParam(), fileId, clientName, lastBlockLength);
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @Override // ClientProtocol
+ public void setTimes(String src, long mtime, long atime) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("setTimes",
+ new Class<?>[] {String.class, long.class, long.class},
+ new RemoteParam(), mtime, atime);
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @Override // ClientProtocol
+ public void createSymlink(String target, String link, FsPermission dirPerms,
+ boolean createParent) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO Verify that the link location is in the same NS as the targets
+ final List<RemoteLocation> targetLocations =
+ getLocationsForPath(target, true);
+ final List<RemoteLocation> linkLocations =
+ getLocationsForPath(link, true);
+ RemoteLocation linkLocation = linkLocations.get(0);
+ RemoteMethod method = new RemoteMethod("createSymlink",
+ new Class<?>[] {String.class, String.class, FsPermission.class,
+ boolean.class},
+ new RemoteParam(), linkLocation.getDest(), dirPerms, createParent);
+ rpcClient.invokeSequential(targetLocations, method);
+ }
+
+ @Override // ClientProtocol
+ public String getLinkTarget(String path) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ final List<RemoteLocation> locations = getLocationsForPath(path, true);
+ RemoteMethod method = new RemoteMethod("getLinkTarget",
+ new Class<?>[] {String.class}, new RemoteParam());
+ return (String) rpcClient.invokeSequential(
+ locations, method, String.class, null);
+ }
+
+ @Override // Client Protocol
+ public void allowSnapshot(String snapshotRoot) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // Client Protocol
+ public void disallowSnapshot(String snapshot) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // ClientProtocol
+ public void renameSnapshot(String snapshotRoot, String snapshotOldName,
+ String snapshotNewName) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // Client Protocol
+ public SnapshottableDirectoryStatus[] getSnapshottableDirListing()
+ throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override // ClientProtocol
+ public SnapshotDiffReport getSnapshotDiffReport(String snapshotRoot,
+ String earlierSnapshotName, String laterSnapshotName) throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override // ClientProtocol
+ public long addCacheDirective(CacheDirectiveInfo path,
+ EnumSet<CacheFlag> flags) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ return 0;
+ }
+
+ @Override // ClientProtocol
+ public void modifyCacheDirective(CacheDirectiveInfo directive,
+ EnumSet<CacheFlag> flags) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // ClientProtocol
+ public void removeCacheDirective(long id) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // ClientProtocol
+ public BatchedEntries<CacheDirectiveEntry> listCacheDirectives(
+ long prevId, CacheDirectiveInfo filter) throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override // ClientProtocol
+ public void addCachePool(CachePoolInfo info) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // ClientProtocol
+ public void modifyCachePool(CachePoolInfo info) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // ClientProtocol
+ public void removeCachePool(String cachePoolName) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // ClientProtocol
+ public BatchedEntries<CachePoolEntry> listCachePools(String prevKey)
+ throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override // ClientProtocol
+ public void modifyAclEntries(String src, List<AclEntry> aclSpec)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("modifyAclEntries",
+ new Class<?>[] {String.class, List.class},
+ new RemoteParam(), aclSpec);
+ rpcClient.invokeSequential(locations, method, null, null);
+ }
+
+ @Override // ClienProtocol
+ public void removeAclEntries(String src, List<AclEntry> aclSpec)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("removeAclEntries",
+ new Class<?>[] {String.class, List.class},
+ new RemoteParam(), aclSpec);
+ rpcClient.invokeSequential(locations, method, null, null);
+ }
+
+ @Override // ClientProtocol
+ public void removeDefaultAcl(String src) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("removeDefaultAcl",
+ new Class<?>[] {String.class}, new RemoteParam());
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @Override // ClientProtocol
+ public void removeAcl(String src) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("removeAcl",
+ new Class<?>[] {String.class}, new RemoteParam());
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @Override // ClientProtocol
+ public void setAcl(String src, List<AclEntry> aclSpec) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod(
+ "setAcl", new Class<?>[] {String.class, List.class},
+ new RemoteParam(), aclSpec);
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @Override // ClientProtocol
+ public AclStatus getAclStatus(String src) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod method = new RemoteMethod("getAclStatus",
+ new Class<?>[] {String.class}, new RemoteParam());
+ return (AclStatus) rpcClient.invokeSequential(
+ locations, method, AclStatus.class, null);
+ }
+
+ @Override // ClientProtocol
+ public void createEncryptionZone(String src, String keyName)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("createEncryptionZone",
+ new Class<?>[] {String.class, String.class},
+ new RemoteParam(), keyName);
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @Override // ClientProtocol
+ public EncryptionZone getEZForPath(String src) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod method = new RemoteMethod("getEZForPath",
+ new Class<?>[] {String.class}, new RemoteParam());
+ return (EncryptionZone) rpcClient.invokeSequential(
+ locations, method, EncryptionZone.class, null);
+ }
+
+ @Override // ClientProtocol
+ public BatchedEntries<EncryptionZone> listEncryptionZones(long prevId)
+ throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override // ClientProtocol
+ public void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("setXAttr",
+ new Class<?>[] {String.class, XAttr.class, EnumSet.class},
+ new RemoteParam(), xAttr, flag);
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override // ClientProtocol
+ public List<XAttr> getXAttrs(String src, List<XAttr> xAttrs)
+ throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod method = new RemoteMethod("getXAttrs",
+ new Class<?>[] {String.class, List.class}, new RemoteParam(), xAttrs);
+ return (List<XAttr>) rpcClient.invokeSequential(
+ locations, method, List.class, null);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override // ClientProtocol
+ public List<XAttr> listXAttrs(String src) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod method = new RemoteMethod("listXAttrs",
+ new Class<?>[] {String.class}, new RemoteParam());
+ return (List<XAttr>) rpcClient.invokeSequential(
+ locations, method, List.class, null);
+ }
+
+ @Override // ClientProtocol
+ public void removeXAttr(String src, XAttr xAttr) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("removeXAttr",
+ new Class<?>[] {String.class, XAttr.class}, new RemoteParam(), xAttr);
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @Override // ClientProtocol
+ public void checkAccess(String path, FsAction mode) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(path, true);
+ RemoteMethod method = new RemoteMethod("checkAccess",
+ new Class<?>[] {String.class, FsAction.class},
+ new RemoteParam(), mode);
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @Override // ClientProtocol
+ public long getCurrentEditLogTxid() throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ RemoteMethod method = new RemoteMethod(
+ "getCurrentEditLogTxid", new Class<?>[] {});
+ final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, Object> ret =
+ rpcClient.invokeConcurrent(nss, method, true, false);
+
+ // Return the maximum txid
+ long txid = 0;
+ Object obj = ret;
+ @SuppressWarnings("unchecked")
+ Map<FederationNamespaceInfo, Long> results =
+ (Map<FederationNamespaceInfo, Long>)obj;
+ Collection<Long> txids = results.values();
+ for (long t : txids) {
+ if (t > txid) {
+ txid = t;
+ }
+ }
+ return txid;
+ }
+
+ @Override // ClientProtocol
+ public EventBatchList getEditsFromTxid(long txid) throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override
+ public DataEncryptionKey getDataEncryptionKey() throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override
+ public String createSnapshot(String snapshotRoot, String snapshotName)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+ return null;
+ }
+
+ @Override
+ public void deleteSnapshot(String snapshotRoot, String snapshotName)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override
+ public ErasureCodingPolicy[] getErasureCodingPolicies() throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override // ClientProtocol
+ public ErasureCodingPolicy getErasureCodingPolicy(String src)
+ throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override // ClientProtocol
+ public void setErasureCodingPolicy(String src, String ecPolicyName)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // ClientProtocol
+ public AddingECPolicyResponse[] addErasureCodingPolicies(
+ ErasureCodingPolicy[] policies) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ return null;
+ }
+
+ @Override // ClientProtocol
+ public void unsetErasureCodingPolicy(String src) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // ClientProtocol
+ public void setQuota(String path, long namespaceQuota, long storagespaceQuota,
+ StorageType type) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO assign global replicas instead of applying them to each folder
+ final List<RemoteLocation> locations = getLocationsForPath(path, true);
+ RemoteMethod method = new RemoteMethod("setQuota",
+ new Class<?>[] {String.class, Long.class, Long.class,
+ StorageType.class},
+ new RemoteParam(), namespaceQuota, storagespaceQuota, type);
+ rpcClient.invokeConcurrent(locations, method, false, false);
+ }
+
+ @Override // ClientProtocol
+ public QuotaUsage getQuotaUsage(String path) throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override
+ public void reportBadBlocks(LocatedBlock[] blocks) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // Block pool id -> blocks
+ Map<String, List<LocatedBlock>> blockLocations = new HashMap<>();
+ for (LocatedBlock block : blocks) {
+ String bpId = block.getBlock().getBlockPoolId();
+ List<LocatedBlock> bpBlocks = blockLocations.get(bpId);
+ if (bpBlocks == null) {
+ bpBlocks = new LinkedList<>();
+ blockLocations.put(bpId, bpBlocks);
+ }
+ bpBlocks.add(block);
+ }
+
+ // Invoke each block pool
+ for (Entry<String, List<LocatedBlock>> entry : blockLocations.entrySet()) {
+ String bpId = entry.getKey();
+ List<LocatedBlock> bpBlocks = entry.getValue();
+
+ LocatedBlock[] bpBlocksArray =
+ bpBlocks.toArray(new LocatedBlock[bpBlocks.size()]);
+ RemoteMethod method = new RemoteMethod("reportBadBlocks",
+ new Class<?>[] {LocatedBlock[].class},
+ new Object[] {bpBlocksArray});
+ rpcClient.invokeSingleBlockPool(bpId, method);
+ }
+ }
+
+ @Override
+ public void unsetStoragePolicy(String src) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override
+ public BlockStoragePolicy getStoragePolicy(String path) throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ /**
+ * Locate the location with the matching block pool id.
+ *
+ * @param path Path to check.
+ * @param failIfLocked Fail the request if locked (top mount point).
+ * @param blockPoolId The block pool ID of the namespace to search for.
+ * @return Prioritized list of locations in the federated cluster.
+ * @throws IOException if the location for this path cannot be determined.
+ */
+ private RemoteLocation getLocationForPath(
+ String path, boolean failIfLocked, String blockPoolId)
+ throws IOException {
+
+ final List<RemoteLocation> locations =
+ getLocationsForPath(path, failIfLocked);
+
+ String nameserviceId = null;
+ Set<FederationNamespaceInfo> namespaces =
+ this.namenodeResolver.getNamespaces();
+ for (FederationNamespaceInfo namespace : namespaces) {
+ if (namespace.getBlockPoolId().equals(blockPoolId)) {
+ nameserviceId = namespace.getNameserviceId();
+ break;
+ }
+ }
+ if (nameserviceId != null) {
+ for (RemoteLocation location : locations) {
+ if (location.getNameserviceId().equals(nameserviceId)) {
+ return location;
+ }
+ }
+ }
+ throw new IOException(
+ "Cannot locate a nameservice for block pool " + blockPoolId);
+ }
+
+ /**
+ * Get the possible locations of a path in the federated cluster.
+ *
+ * @param path Path to check.
+ * @param failIfLocked Fail the request if locked (top mount point).
+ * @return Prioritized list of locations in the federated cluster.
+ * @throws IOException If the location for this path cannot be determined.
+ */
+ private List<RemoteLocation> getLocationsForPath(
+ String path, boolean failIfLocked) throws IOException {
+ // Check the location for this path
+ final PathLocation location =
+ this.subclusterResolver.getDestinationForPath(path);
+ if (location == null) {
+ throw new IOException("Cannot find locations for " + path + " in " +
+ this.subclusterResolver);
+ }
+
+ // Log the access to a path
+ return location.getDestinations();
+ }
+
+ /**
+ * Get the modification dates for mount points.
+ *
+ * @param path Name of the path to start checking dates from.
+ * @return Map with the modification dates for all sub-entries.
+ */
+ private Map<String, Long> getMountPointDates(String path) {
+ Map<String, Long> ret = new TreeMap<>();
+ // TODO add when we have a Mount Table
+ return ret;
+ }
+
+ /**
+ * Create a new file status for a mount point.
+ *
+ * @param name Name of the mount point.
+ * @param childrenNum Number of children.
+ * @param dates Map with the dates.
+ * @return New HDFS file status representing a mount point.
+ */
+ private HdfsFileStatus getMountPointStatus(
+ String name, int childrenNum, long date) {
+ long modTime = date;
+ long accessTime = date;
+ FsPermission permission = FsPermission.getDirDefault();
+ String owner = this.superUser;
+ String group = this.superGroup;
+ try {
+ // TODO support users, it should be the user for the pointed folder
+ UserGroupInformation ugi = getRemoteUser();
+ owner = ugi.getUserName();
+ group = ugi.getPrimaryGroupName();
+ } catch (IOException e) {
+ LOG.error("Cannot get the remote user: {}", e.getMessage());
+ }
+ long inodeId = 0;
+ return new HdfsFileStatus(0, true, 0, 0, modTime, accessTime, permission,
+ owner, group, new byte[0], DFSUtil.string2Bytes(name), inodeId,
+ childrenNum, null, (byte) 0, null);
+ }
+
+ /**
+ * Get the name of the method that is calling this function.
+ *
+ * @return Name of the method calling this function.
+ */
+ private static String getMethodName() {
+ final StackTraceElement[] stack = Thread.currentThread().getStackTrace();
+ String methodName = stack[3].getMethodName();
+ return methodName;
+ }
+
+ /**
+ * Get the user that is invoking this operation.
+ *
+ * @return Remote user group information.
+ * @throws IOException If we cannot get the user information.
+ */
+ static UserGroupInformation getRemoteUser() throws IOException {
+ UserGroupInformation ugi = Server.getRemoteUser();
+ return (ugi != null) ? ugi : UserGroupInformation.getCurrentUser();
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6632c5a4/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 12e4f8b..5f746ba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4514,6 +4514,101 @@
</property>
<property>
+ <name>dfs.federation.router.default.nameserviceId</name>
+ <value></value>
+ <description>
+ Nameservice identifier of the default subcluster to monitor.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.rpc.enable</name>
+ <value>true</value>
+ <description>
+ If the RPC service to handle client requests in the router is enabled.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.rpc-address</name>
+ <value>0.0.0.0:8888</value>
+ <description>
+ RPC address that handles all clients requests.
+ The value of this property will take the form of router-host1:rpc-port.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.rpc-bind-host</name>
+ <value></value>
+ <description>
+ The actual address the RPC server will bind to. If this optional address is
+ set, it overrides only the hostname portion of
+ dfs.federation.router.rpc-address. This is useful for making the name node
+ listen on all interfaces by setting it to 0.0.0.0.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.handler.count</name>
+ <value>10</value>
+ <description>
+ The number of server threads for the router to handle RPC requests from
+ clients.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.handler.queue.size</name>
+ <value>100</value>
+ <description>
+ The size of the queue for the number of handlers to handle RPC client requests.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.reader.count</name>
+ <value>1</value>
+ <description>
+ The number of readers for the router to handle RPC client requests.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.reader.queue.size</name>
+ <value>100</value>
+ <description>
+ The size of the queue for the number of readers for the router to handle RPC client requests.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.connection.pool-size</name>
+ <value>1</value>
+ <description>
+ Size of the pool of connections from the router to namenodes.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.connection.clean.ms</name>
+ <value>10000</value>
+ <description>
+ Time interval, in milliseconds, to check if the connection pool should
+ remove unused connections.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.connection.pool.clean.ms</name>
+ <value>60000</value>
+ <description>
+ Time interval, in milliseconds, to check if the connection manager should
+ remove unused connection pools.
+ </description>
+ </property>
+
+ <property>
<name>dfs.federation.router.file.resolver.client.class</name>
<value>org.apache.hadoop.hdfs.server.federation.MockResolver</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6632c5a4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
index 8674682..bbb548ca 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.hdfs.server.federation;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.BufferedReader;
import java.io.FileNotFoundException;
@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
@@ -51,17 +52,11 @@ import org.apache.hadoop.security.AccessControlException;
*/
public final class FederationTestUtils {
- public final static String NAMESERVICE1 = "ns0";
- public final static String NAMESERVICE2 = "ns1";
- public final static String NAMENODE1 = "nn0";
- public final static String NAMENODE2 = "nn1";
- public final static String NAMENODE3 = "nn2";
- public final static String NAMENODE4 = "nn3";
- public final static String ROUTER1 = "router0";
- public final static String ROUTER2 = "router1";
- public final static String ROUTER3 = "router2";
- public final static String ROUTER4 = "router3";
- public final static long BLOCK_SIZE_BYTES = 134217728;
+ public final static String[] NAMESERVICES = {"ns0", "ns1"};
+ public final static String[] NAMENODES = {"nn0", "nn1", "nn2", "nn3"};
+ public final static String[] ROUTERS =
+ {"router0", "router1", "router2", "router3"};
+
private FederationTestUtils() {
// Utility class
@@ -81,7 +76,7 @@ public final class FederationTestUtils {
triggeredException = e;
}
if (exceptionClass != null) {
- assertNotNull("No exception was triggered, expected exception - "
+ assertNotNull("No exception was triggered, expected exception"
+ exceptionClass.getName(), triggeredException);
assertEquals(exceptionClass, triggeredException.getClass());
} else {
@@ -101,48 +96,43 @@ public final class FederationTestUtils {
return report;
}
report.setHAServiceState(state);
- report.setNamespaceInfo(new NamespaceInfo(1, "tesclusterid", ns, 0,
- "testbuildvesion", "testsoftwareversion"));
+ NamespaceInfo nsInfo = new NamespaceInfo(
+ 1, "tesclusterid", ns, 0, "testbuildvesion", "testsoftwareversion");
+ report.setNamespaceInfo(nsInfo);
return report;
}
public static void waitNamenodeRegistered(ActiveNamenodeResolver resolver,
- String nameserviceId, String namenodeId,
- FederationNamenodeServiceState finalState)
+ String nsId, String nnId, FederationNamenodeServiceState finalState)
throws InterruptedException, IllegalStateException, IOException {
for (int loopCount = 0; loopCount < 20; loopCount++) {
-
if (loopCount > 0) {
Thread.sleep(1000);
}
- List<? extends FederationNamenodeContext> namenodes;
- namenodes =
- resolver.getNamenodesForNameserviceId(nameserviceId);
+ List<? extends FederationNamenodeContext> namenodes =
+ resolver.getNamenodesForNameserviceId(nsId);
for (FederationNamenodeContext namenode : namenodes) {
- if (namenodeId != null
- && !namenode.getNamenodeId().equals(namenodeId)) {
- // Keep looking
- continue;
+ // Check if this is the Namenode we are checking
+ if (namenode.getNamenodeId() == nnId ||
+ namenode.getNamenodeId().equals(nnId)) {
+ if (finalState != null && !namenode.getState().equals(finalState)) {
+ // Wrong state, wait a bit more
+ break;
+ } else {
+ // Found and verified
+ return;
+ }
}
- if (finalState != null && !namenode.getState().equals(finalState)) {
- // Wrong state, wait a bit more
- break;
- }
- // Found
- return;
}
}
- assertTrue("Failed to verify state store registration for state - "
- + finalState + " - " + " - " + nameserviceId + " - ", false);
+ fail("Failed to verify State Store registration of " + nsId + " " + nnId +
+ " for state " + finalState);
}
public static boolean verifyDate(Date d1, Date d2, long precision) {
- if (Math.abs(d1.getTime() - d2.getTime()) < precision) {
- return true;
- }
- return false;
+ return Math.abs(d1.getTime() - d2.getTime()) < precision;
}
public static boolean addDirectory(FileSystem context, String path)
@@ -165,15 +155,14 @@ public final class FederationTestUtils {
} catch (Exception e) {
return false;
}
-
return false;
}
- public static boolean checkForFileInDirectory(FileSystem context,
- String testPath, String targetFile) throws AccessControlException,
- FileNotFoundException,
- UnsupportedFileSystemException, IllegalArgumentException,
- IOException {
+ public static boolean checkForFileInDirectory(
+ FileSystem context, String testPath, String targetFile)
+ throws IOException, AccessControlException, FileNotFoundException,
+ UnsupportedFileSystemException, IllegalArgumentException {
+
FileStatus[] fileStatus = context.listStatus(new Path(testPath));
String file = null;
String verifyPath = testPath + "/" + targetFile;
@@ -194,7 +183,8 @@ public final class FederationTestUtils {
public static int countContents(FileSystem context, String testPath)
throws IOException {
- FileStatus[] fileStatus = context.listStatus(new Path(testPath));
+ Path path = new Path(testPath);
+ FileStatus[] fileStatus = context.listStatus(path);
return fileStatus.length;
}
@@ -202,7 +192,7 @@ public final class FederationTestUtils {
throws IOException {
FsPermission permissions = new FsPermission("700");
FSDataOutputStream writeStream = fs.create(new Path(path), permissions,
- true, 1000, (short) 1, BLOCK_SIZE_BYTES, null);
+ true, 1000, (short) 1, DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT, null);
for (int i = 0; i < length; i++) {
writeStream.write(i);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: HADOOP-14361. Azure:
NativeAzureFileSystem.getDelegationToken() call fails sometimes when invoked
concurrently. Contributed by Santhosh G Nayak
Posted by in...@apache.org.
HADOOP-14361. Azure: NativeAzureFileSystem.getDelegationToken() call fails sometimes when invoked concurrently. Contributed by Santhosh G Nayak
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eed73149
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eed73149
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eed73149
Branch: refs/heads/HDFS-10467
Commit: eed731496f63537350206274af09d2118b116a17
Parents: 5cb6e3e
Author: Mingliang Liu <li...@apache.org>
Authored: Wed May 10 15:05:11 2017 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Wed May 10 15:05:11 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eed73149/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 45c45b1..03a2782 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -2968,7 +2968,7 @@ public class NativeAzureFileSystem extends FileSystem {
* @throws IOException thrown when getting the current user.
*/
@Override
- public Token<?> getDelegationToken(final String renewer) throws IOException {
+ public synchronized Token<?> getDelegationToken(final String renewer) throws IOException {
if (kerberosSupportEnabled) {
try {
final UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: YARN-6561. Update exception
message during timeline collector aux service initialization. (Vrushali C via
Haibo Chen)
Posted by in...@apache.org.
YARN-6561. Update exception message during timeline collector aux service initialization. (Vrushali C via Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ab2bb93a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ab2bb93a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ab2bb93a
Branch: refs/heads/HDFS-10467
Commit: ab2bb93a2ab1651b73ec9ba2d1deec4deafdecaf
Parents: 461ee44
Author: Haibo Chen <ha...@cloudera.com>
Authored: Tue May 9 21:37:30 2017 -0700
Committer: Haibo Chen <ha...@cloudera.com>
Committed: Tue May 9 21:37:30 2017 -0700
----------------------------------------------------------------------
.../collector/PerNodeTimelineCollectorsAuxService.java | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab2bb93a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
index 041e7c2..266bd04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
@@ -73,7 +73,12 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
@Override
protected void serviceInit(Configuration conf) throws Exception {
if (!YarnConfiguration.timelineServiceV2Enabled(conf)) {
- throw new YarnException("Timeline service v2 is not enabled");
+ throw new YarnException(
+ "Looks like timeline_collector is set as an auxillary service in "
+ + YarnConfiguration.NM_AUX_SERVICES
+ + ". But Timeline service v2 is not enabled,"
+ + " so timeline_collector needs to be removed"
+ + " from that list of auxillary services.");
}
collectorLingerPeriod =
conf.getLong(YarnConfiguration.ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: YARN-6473. Create
ReservationInvariantChecker to validate ReservationSystem + Scheduler
operations.
Posted by in...@apache.org.
YARN-6473. Create ReservationInvariantChecker to validate ReservationSystem + Scheduler operations.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5cb6e3e0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5cb6e3e0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5cb6e3e0
Branch: refs/heads/HDFS-10467
Commit: 5cb6e3e082ed9edbdb7c46d27daa049a4712e82b
Parents: e7654c4
Author: Carlo Curino <cu...@apache.org>
Authored: Wed May 10 14:49:25 2017 -0700
Committer: Carlo Curino <cu...@apache.org>
Committed: Wed May 10 14:49:25 2017 -0700
----------------------------------------------------------------------
.../sls/TestReservationSystemInvariants.java | 73 ++++++++++++++++++++
.../ReservationInvariantsChecker.java | 63 +++++++++++++++++
2 files changed, 136 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cb6e3e0/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestReservationSystemInvariants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestReservationSystemInvariants.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestReservationSystemInvariants.java
new file mode 100644
index 0000000..b3a79cb
--- /dev/null
+++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestReservationSystemInvariants.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.sls;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.monitor.invariants.InvariantsChecker;
+import org.apache.hadoop.yarn.server.resourcemanager.monitor.invariants.ReservationInvariantsChecker;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import net.jcip.annotations.NotThreadSafe;
+
+/**
+ * This test performs an SLS run enabling a
+ * {@code ReservationInvariantsChecker}.
+ */
+@RunWith(value = Parameterized.class)
+@NotThreadSafe
+public class TestReservationSystemInvariants extends BaseSLSRunnerTest {
+
+ @Parameters(name = "Testing with: {1}, {0}, (nodeFile {3})")
+ public static Collection<Object[]> data() {
+ // Test with both schedulers, and all three trace types
+ return Arrays.asList(new Object[][] {
+ {CapacityScheduler.class.getCanonicalName(), "SYNTH",
+ "src/test/resources/syn.json", null},
+ {FairScheduler.class.getCanonicalName(), "SYNTH",
+ "src/test/resources/syn.json", null}
+ });
+ }
+
+ @Test(timeout = 120000)
+ @SuppressWarnings("all")
+ public void testSimulatorRunning() throws Exception {
+
+ Configuration conf = new Configuration(false);
+ conf.set(YarnConfiguration.RM_SCHEDULER, schedulerType);
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
+ conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
+ ReservationInvariantsChecker.class.getCanonicalName());
+ conf.setBoolean(InvariantsChecker.THROW_ON_VIOLATION, true);
+
+
+ long timeTillShutDownInSec = 90;
+ runSLS(conf, timeTillShutDownInSec);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cb6e3e0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/invariants/ReservationInvariantsChecker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/invariants/ReservationInvariantsChecker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/invariants/ReservationInvariantsChecker.java
new file mode 100644
index 0000000..2f9f03e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/invariants/ReservationInvariantsChecker.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.monitor.invariants;
+
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
+import org.apache.hadoop.yarn.util.UTCClock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collection;
+
+/**
+ * Invariant checker that checks certain reservation invariants are respected.
+ */
+public class ReservationInvariantsChecker extends InvariantsChecker {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ReservationInvariantsChecker.class);
+
+ private UTCClock clock = new UTCClock();
+
+ @Override
+ public void editSchedule() {
+ Collection<Plan> plans =
+ getContext().getReservationSystem().getAllPlans().values();
+
+ try {
+ for (Plan plan : plans) {
+ long currReservations =
+ plan.getReservationsAtTime(clock.getTime()).size();
+ long numberReservationQueues = getContext().getScheduler()
+ .getQueueInfo(plan.getQueueName(), true, false).getChildQueues()
+ .size();
+ if (currReservations != numberReservationQueues - 1) {
+ logOrThrow("Number of reservations (" + currReservations
+ + ") does NOT match the number of reservationQueues ("
+ + (numberReservationQueues - 1) + "), while it should.");
+ }
+ }
+ } catch (IOException io) {
+ throw new InvariantViolationException("Issue during invariant check: ",
+ io);
+ }
+
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: YARN-3742. YARN RM will shut down
if ZKClient creation times out. (Daniel Templeton via kasha)
Posted by in...@apache.org.
YARN-3742. YARN RM will shut down if ZKClient creation times out. (Daniel Templeton via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/166be0ee
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/166be0ee
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/166be0ee
Branch: refs/heads/HDFS-10467
Commit: 166be0ee95d5ef976f074342656b289b41a11ccd
Parents: f7faac8
Author: Karthik Kambatla <ka...@cloudera.com>
Authored: Tue May 9 14:44:16 2017 -0700
Committer: Karthik Kambatla <ka...@cloudera.com>
Committed: Tue May 9 14:44:16 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/client/TestRMFailover.java | 5 +-
...ActiveStandbyElectorBasedElectorService.java | 5 +-
.../server/resourcemanager/AdminService.java | 3 +-
...MCriticalThreadUncaughtExceptionHandler.java | 19 ++++--
.../server/resourcemanager/RMFatalEvent.java | 67 ++++++++++++++++++--
.../resourcemanager/RMFatalEventType.java | 1 +
.../server/resourcemanager/ResourceManager.java | 44 +++++++++++--
.../resourcemanager/recovery/RMStateStore.java | 12 ++--
.../recovery/TestMemoryRMStateStore.java | 2 +
9 files changed, 127 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/166be0ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
index d568d6a..5025c7c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
@@ -54,6 +54,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.HATestUtil;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.RMCriticalThreadUncaughtExceptionHandler;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.RMFatalEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.RMFatalEventType;
import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
import org.junit.After;
@@ -200,7 +202,8 @@ public class TestRMFailover extends ClientBaseWithFixes {
// so it transitions to standby.
ResourceManager rm = cluster.getResourceManager(
cluster.getActiveRMIndex());
- rm.handleTransitionToStandByInNewThread();
+ rm.getRMContext().getDispatcher().getEventHandler().handle(
+ new RMFatalEvent(RMFatalEventType.STATE_STORE_FENCED, "test"));
verifyRMTransitionToStandby(rm);
verifyConnections();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/166be0ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ActiveStandbyElectorBasedElectorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ActiveStandbyElectorBasedElectorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ActiveStandbyElectorBasedElectorService.java
index 751eedd..b59bc25 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ActiveStandbyElectorBasedElectorService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ActiveStandbyElectorBasedElectorService.java
@@ -108,8 +108,9 @@ public class ActiveStandbyElectorBasedElectorService extends AbstractService
elector.ensureParentZNode();
if (!isParentZnodeSafe(clusterId)) {
- notifyFatalError(electionZNode + " znode has invalid data! "+
- "Might need formatting!");
+ notifyFatalError(String.format("invalid data in znode, %s, " +
+ "which may require the state store to be reformatted",
+ electionZNode));
}
super.serviceInit(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/166be0ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index 74c87a2..7571765 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
@@ -304,13 +304,12 @@ public class AdminService extends CompositeService implements
// call all refresh*s for active RM to get the updated configurations.
refreshAll();
} catch (Exception e) {
- LOG.error("RefreshAll failed so firing fatal event", e);
rmContext
.getDispatcher()
.getEventHandler()
.handle(
new RMFatalEvent(RMFatalEventType.TRANSITION_TO_ACTIVE_FAILED,
- e));
+ e, "failure to refresh configuration settings"));
throw new ServiceFailedException(
"Error on refreshAll during transition to Active", e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/166be0ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMCriticalThreadUncaughtExceptionHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMCriticalThreadUncaughtExceptionHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMCriticalThreadUncaughtExceptionHandler.java
index a67f81a..c863889 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMCriticalThreadUncaughtExceptionHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMCriticalThreadUncaughtExceptionHandler.java
@@ -23,7 +23,7 @@ import java.lang.Thread.UncaughtExceptionHandler;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.yarn.conf.HAUtil;
+import org.apache.hadoop.yarn.exceptions.YarnException;
/**
* This class either shuts down {@link ResourceManager} or transitions the
@@ -45,14 +45,19 @@ public class RMCriticalThreadUncaughtExceptionHandler
@Override
public void uncaughtException(Thread t, Throwable e) {
- LOG.fatal("Critical thread " + t.getName() + " crashed!", e);
+ Exception ex;
- if (HAUtil.isHAEnabled(rmContext.getYarnConfiguration())) {
- rmContext.getResourceManager().handleTransitionToStandByInNewThread();
+ if (e instanceof Exception) {
+ ex = (Exception)e;
} else {
- rmContext.getDispatcher().getEventHandler().handle(
- new RMFatalEvent(RMFatalEventType.CRITICAL_THREAD_CRASH,
- new Exception(e)));
+ ex = new YarnException(e);
}
+
+ RMFatalEvent event =
+ new RMFatalEvent(RMFatalEventType.CRITICAL_THREAD_CRASH, ex,
+ String.format("a critical thread, %s, that exited unexpectedly",
+ t.getName()));
+
+ rmContext.getDispatcher().getEventHandler().handle(event);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/166be0ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMFatalEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMFatalEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMFatalEvent.java
index 59e6236..899377d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMFatalEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMFatalEvent.java
@@ -20,18 +20,73 @@ package org.apache.hadoop.yarn.server.resourcemanager;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.event.AbstractEvent;
+/**
+ * Event that indicates a non-recoverable error for the resource manager.
+ */
public class RMFatalEvent extends AbstractEvent<RMFatalEventType> {
- private String cause;
+ private final Exception cause;
+ private final String message;
- public RMFatalEvent(RMFatalEventType rmFatalEventType, String cause) {
- super(rmFatalEventType);
- this.cause = cause;
+ /**
+ * Create a new event of the given type with the given cause.
+ * @param rmFatalEventType The {@link RMFatalEventType} of the event
+ * @param message a text description of the reason for the event
+ */
+ public RMFatalEvent(RMFatalEventType rmFatalEventType, String message) {
+ this(rmFatalEventType, null, message);
}
+ /**
+ * Create a new event of the given type around the given source
+ * {@link Exception}.
+ * @param rmFatalEventType The {@link RMFatalEventType} of the event
+ * @param cause the source exception
+ */
public RMFatalEvent(RMFatalEventType rmFatalEventType, Exception cause) {
+ this(rmFatalEventType, cause, null);
+ }
+
+ /**
+ * Create a new event of the given type around the given source
+ * {@link Exception} with the given cause.
+ * @param rmFatalEventType The {@link RMFatalEventType} of the event
+ * @param cause the source exception
+ * @param message a text description of the reason for the event
+ */
+ public RMFatalEvent(RMFatalEventType rmFatalEventType, Exception cause,
+ String message) {
super(rmFatalEventType);
- this.cause = StringUtils.stringifyException(cause);
+ this.cause = cause;
+ this.message = message;
}
- public String getCause() {return this.cause;}
+ /**
+ * Get a text description of the reason for the event. If a cause was, that
+ * {@link Exception} will be converted to a {@link String} and included in
+ * the result.
+ * @return a text description of the reason for the event
+ */
+ public String getExplanation() {
+ StringBuilder sb = new StringBuilder();
+
+ if (message != null) {
+ sb.append(message);
+
+ if (cause != null) {
+ sb.append(": ");
+ }
+ }
+
+ if (cause != null) {
+ sb.append(StringUtils.stringifyException(cause));
+ }
+
+ return sb.toString();
+ }
+
+ @Override
+ public String toString() {
+ return String.format("RMFatalEvent of type %s, caused by %s",
+ getType().name(), getExplanation());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/166be0ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMFatalEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMFatalEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMFatalEventType.java
index b6f6b3c..69b285f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMFatalEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMFatalEventType.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
@InterfaceAudience.Private
public enum RMFatalEventType {
// Source <- Store
+ STATE_STORE_FENCED,
STATE_STORE_OP_FAILED,
// Source <- Embedded Elector
http://git-wip-us.apache.org/repos/asf/hadoop/blob/166be0ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 81c3f1b..75d6df2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -811,15 +811,45 @@ public class ResourceManager extends CompositeService implements Recoverable {
}
@Private
- public static class RMFatalEventDispatcher
- implements EventHandler<RMFatalEvent> {
-
+ private class RMFatalEventDispatcher implements EventHandler<RMFatalEvent> {
@Override
public void handle(RMFatalEvent event) {
- LOG.fatal("Received a " + RMFatalEvent.class.getName() + " of type " +
- event.getType().name() + ". Cause:\n" + event.getCause());
+ LOG.error("Received " + event);
- ExitUtil.terminate(1, event.getCause());
+ if (HAUtil.isHAEnabled(getConfig())) {
+ // If we're in an HA config, the right answer is always to go into
+ // standby.
+ LOG.warn("Transitioning the resource manager to standby.");
+ handleTransitionToStandByInNewThread();
+ } else {
+ // If we're stand-alone, we probably want to shut down, but the if and
+ // how depends on the event.
+ switch(event.getType()) {
+ case STATE_STORE_FENCED:
+ LOG.fatal("State store fenced even though the resource manager " +
+ "is not configured for high availability. Shutting down this " +
+ "resource manager to protect the integrity of the state store.");
+ ExitUtil.terminate(1, event.getExplanation());
+ break;
+ case STATE_STORE_OP_FAILED:
+ if (YarnConfiguration.shouldRMFailFast(getConfig())) {
+ LOG.fatal("Shutting down the resource manager because a state " +
+ "store operation failed, and the resource manager is " +
+ "configured to fail fast. See the yarn.fail-fast and " +
+ "yarn.resourcemanager.fail-fast properties.");
+ ExitUtil.terminate(1, event.getExplanation());
+ } else {
+ LOG.warn("Ignoring state store operation failure because the " +
+ "resource manager is not configured to fail fast. See the " +
+ "yarn.fail-fast and yarn.resourcemanager.fail-fast " +
+ "properties.");
+ }
+ break;
+ default:
+ LOG.fatal("Shutting down the resource manager.");
+ ExitUtil.terminate(1, event.getExplanation());
+ }
+ }
}
}
@@ -827,7 +857,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
* Transition to standby state in a new thread. The transition operation is
* asynchronous to avoid deadlock caused by cyclic dependency.
*/
- public void handleTransitionToStandByInNewThread() {
+ private void handleTransitionToStandByInNewThread() {
Thread standByTransitionThread =
new Thread(activeServices.standByTransitionRunnable);
standByTransitionThread.setName("StandByTransitionThread");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/166be0ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
index 5e3cf22..975847c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
@@ -1129,18 +1129,18 @@ public abstract class RMStateStore extends AbstractService {
Exception failureCause) {
boolean isFenced = false;
LOG.error("State store operation failed ", failureCause);
+
if (HAUtil.isHAEnabled(getConfig())) {
- LOG.warn("State-store fenced ! Transitioning RM to standby");
+ rmDispatcher.getEventHandler().handle(
+ new RMFatalEvent(RMFatalEventType.STATE_STORE_FENCED,
+ failureCause));
isFenced = true;
- resourceManager.handleTransitionToStandByInNewThread();
- } else if (YarnConfiguration.shouldRMFailFast(getConfig())) {
- LOG.fatal("Fail RM now due to state-store error!");
+ } else {
rmDispatcher.getEventHandler().handle(
new RMFatalEvent(RMFatalEventType.STATE_STORE_OP_FAILED,
failureCause));
- } else {
- LOG.warn("Skip the state-store error.");
}
+
return isFenced;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/166be0ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestMemoryRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestMemoryRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestMemoryRMStateStore.java
index 89b9e2b..cb278c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestMemoryRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestMemoryRMStateStore.java
@@ -43,6 +43,7 @@ public class TestMemoryRMStateStore {
store.init(conf);
ResourceManager mockRM = mock(ResourceManager.class);
store.setResourceManager(mockRM);
+ store.setRMDispatcher(new RMStateStoreTestBase.TestDispatcher());
RMDelegationTokenIdentifier mockTokenId =
mock(RMDelegationTokenIdentifier.class);
store.removeRMDelegationToken(mockTokenId);
@@ -58,6 +59,7 @@ public class TestMemoryRMStateStore {
};
store.init(conf);
store.setResourceManager(mockRM);
+ store.setRMDispatcher(new RMStateStoreTestBase.TestDispatcher());
store.removeRMDelegationToken(mockTokenId);
assertTrue("RMStateStore should have been in fenced state",
store.isFencedState());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: MAPREDUCE-6882. Increase MapReduce
test timeouts from 1 second to 10 seconds. Contributed by Eric Badger
Posted by in...@apache.org.
MAPREDUCE-6882. Increase MapReduce test timeouts from 1 second to 10 seconds. Contributed by Eric Badger
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1e71fe8c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1e71fe8c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1e71fe8c
Branch: refs/heads/HDFS-10467
Commit: 1e71fe8c42b7945170c47023e82a0412397ff57e
Parents: 6099dee
Author: Jason Lowe <jl...@yahoo-inc.com>
Authored: Wed May 10 09:57:41 2017 -0500
Committer: Jason Lowe <jl...@yahoo-inc.com>
Committed: Wed May 10 09:57:41 2017 -0500
----------------------------------------------------------------------
.../org/apache/hadoop/mapred/TestMRWithDistributedCache.java | 2 +-
.../src/test/java/org/apache/hadoop/mapred/TestClock.java | 2 +-
.../test/java/org/apache/hadoop/mapred/TestClusterStatus.java | 4 ++--
.../src/test/java/org/apache/hadoop/mapred/TestJobConf.java | 2 +-
4 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e71fe8c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java
index 02fe60b..341f38d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java
@@ -223,7 +223,7 @@ public class TestMRWithDistributedCache extends TestCase {
return p;
}
- @Test (timeout = 1000)
+ @Test (timeout = 10000)
public void testDeprecatedFunctions() throws Exception {
DistributedCache.addLocalArchives(conf, "Test Local Archives 1");
Assert.assertEquals("Test Local Archives 1",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e71fe8c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestClock.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestClock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestClock.java
index 7081cd5..70016bf 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestClock.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestClock.java
@@ -27,7 +27,7 @@ import static org.junit.Assert.*;
*/
public class TestClock {
- @Test (timeout=1000)
+ @Test (timeout=10000)
public void testClock(){
Clock clock= new Clock();
long templateTime=System.currentTimeMillis();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e71fe8c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestClusterStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestClusterStatus.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestClusterStatus.java
index 08059a0..399e972 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestClusterStatus.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestClusterStatus.java
@@ -26,14 +26,14 @@ public class TestClusterStatus {
private ClusterStatus clusterStatus = new ClusterStatus();
@SuppressWarnings("deprecation")
- @Test (timeout = 1000)
+ @Test (timeout = 10000)
public void testGraylistedTrackers() {
Assert.assertEquals(0, clusterStatus.getGraylistedTrackers());
Assert.assertTrue(clusterStatus.getGraylistedTrackerNames().isEmpty());
}
@SuppressWarnings("deprecation")
- @Test (timeout = 1000)
+ @Test (timeout = 10000)
public void testJobTrackerState() {
Assert.assertEquals(JobTracker.State.RUNNING,
clusterStatus.getJobTrackerState());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e71fe8c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java
index 58092f2..a493d16 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java
@@ -163,7 +163,7 @@ public class TestJobConf {
* old property names
*/
@SuppressWarnings("deprecation")
- @Test (timeout = 1000)
+ @Test (timeout = 10000)
public void testDeprecatedPropertyNameForTaskVmem() {
JobConf configuration = new JobConf();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: HADOOP-14383. Implement FileSystem
that reads from HTTP / HTTPS endpoints.
Posted by in...@apache.org.
HADOOP-14383. Implement FileSystem that reads from HTTP / HTTPS endpoints.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ff5ec3b8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ff5ec3b8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ff5ec3b8
Branch: refs/heads/HDFS-10467
Commit: ff5ec3b841612f7f28ad8be5bbfec4168a8ac1f0
Parents: 424887e
Author: Haohui Mai <wh...@apache.org>
Authored: Thu May 4 17:27:44 2017 -0700
Committer: Haohui Mai <wh...@apache.org>
Committed: Mon May 8 15:28:45 2017 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/pom.xml | 5 +
.../hadoop/fs/http/AbstractHttpFileSystem.java | 153 +++++++++++++++++++
.../apache/hadoop/fs/http/HttpFileSystem.java | 28 ++++
.../apache/hadoop/fs/http/HttpsFileSystem.java | 28 ++++
.../org/apache/hadoop/fs/http/package-info.java | 23 +++
.../services/org.apache.hadoop.fs.FileSystem | 2 +
.../hadoop/fs/http/TestHttpFileSystem.java | 67 ++++++++
hadoop-project/pom.xml | 6 +
8 files changed, 312 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff5ec3b8/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index f76575d..e8b5317 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -318,6 +318,11 @@
<artifactId>aalto-xml</artifactId>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>com.squareup.okhttp3</groupId>
+ <artifactId>mockwebserver</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff5ec3b8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java
new file mode 100644
index 0000000..fa0b2cf
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.http;
+
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PositionedReadable;
+import org.apache.hadoop.fs.Seekable;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.Progressable;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URLConnection;
+
+abstract class AbstractHttpFileSystem extends FileSystem {
+ private static final long DEFAULT_BLOCK_SIZE = 4096;
+ private static final Path WORKING_DIR = new Path("/");
+
+ private URI uri;
+
+ @Override
+ public void initialize(URI name, Configuration conf) throws IOException {
+ super.initialize(name, conf);
+ this.uri = name;
+ }
+
+ public abstract String getScheme();
+
+ @Override
+ public URI getUri() {
+ return uri;
+ }
+
+ @Override
+ public FSDataInputStream open(Path path, int bufferSize) throws IOException {
+ URLConnection conn = path.toUri().toURL().openConnection();
+ InputStream in = conn.getInputStream();
+ return new FSDataInputStream(new HttpDataInputStream(in));
+ }
+
+ @Override
+ public FSDataOutputStream create(Path path, FsPermission fsPermission,
+ boolean b, int i, short i1, long l,
+ Progressable progressable)
+ throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public FSDataOutputStream append(Path path, int i, Progressable progressable)
+ throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean rename(Path path, Path path1) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean delete(Path path, boolean b) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path path) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void setWorkingDirectory(Path path) {
+ }
+
+ @Override
+ public Path getWorkingDirectory() {
+ return WORKING_DIR;
+ }
+
+ @Override
+ public boolean mkdirs(Path path, FsPermission fsPermission)
+ throws IOException {
+ return false;
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path path) throws IOException {
+ return new FileStatus(-1, false, 1, DEFAULT_BLOCK_SIZE, 0, path);
+ }
+
+ private static class HttpDataInputStream extends FilterInputStream
+ implements Seekable, PositionedReadable {
+
+ HttpDataInputStream(InputStream in) {
+ super(in);
+ }
+
+ @Override
+ public int read(long position, byte[] buffer, int offset, int length)
+ throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void readFully(long position, byte[] buffer, int offset, int length)
+ throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void readFully(long position, byte[] buffer) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void seek(long pos) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getPos() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean seekToNewSource(long targetPos) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff5ec3b8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/HttpFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/HttpFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/HttpFileSystem.java
new file mode 100644
index 0000000..a4d1505
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/HttpFileSystem.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.http;
+
+/**
+ * A Filesystem that reads from HTTP endpoint.
+ */
+public class HttpFileSystem extends AbstractHttpFileSystem {
+ @Override
+ public String getScheme() {
+ return "http";
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff5ec3b8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/HttpsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/HttpsFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/HttpsFileSystem.java
new file mode 100644
index 0000000..88e0968
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/HttpsFileSystem.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.http;
+
+/**
+ * A Filesystem that reads from HTTPS endpoint.
+ */
+public class HttpsFileSystem extends AbstractHttpFileSystem {
+ @Override
+ public String getScheme() {
+ return "https";
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff5ec3b8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/package-info.java
new file mode 100644
index 0000000..a5b5206
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Filesystem implementations that allow Hadoop to read directly from
+ * HTTP / HTTPS endpoints.
+ */
+package org.apache.hadoop.fs.http;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff5ec3b8/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
index 17ffa7f..cbf2d6d 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
+++ b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
@@ -17,3 +17,5 @@ org.apache.hadoop.fs.LocalFileSystem
org.apache.hadoop.fs.viewfs.ViewFileSystem
org.apache.hadoop.fs.ftp.FTPFileSystem
org.apache.hadoop.fs.HarFileSystem
+org.apache.hadoop.fs.http.HttpFileSystem
+org.apache.hadoop.fs.http.HttpsFileSystem
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff5ec3b8/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/http/TestHttpFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/http/TestHttpFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/http/TestHttpFileSystem.java
new file mode 100644
index 0000000..0902c04
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/http/TestHttpFileSystem.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.http;
+
+import okhttp3.mockwebserver.MockResponse;
+import okhttp3.mockwebserver.MockWebServer;
+import okhttp3.mockwebserver.RecordedRequest;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Testing HttpFileSystem.
+ */
+public class TestHttpFileSystem {
+ @Test
+ public void testHttpFileSystem() throws IOException, URISyntaxException,
+ InterruptedException {
+ Configuration conf = new Configuration(false);
+ conf.set("fs.http.impl", HttpFileSystem.class.getCanonicalName());
+ final String data = "foo";
+
+ try (MockWebServer server = new MockWebServer()) {
+ server.enqueue(new MockResponse().setBody(data));
+ server.start();
+ URI uri = URI.create(String.format("http://%s:%d", server.getHostName(),
+ server.getPort()));
+ FileSystem fs = FileSystem.get(uri, conf);
+ try (InputStream is = fs.open(
+ new Path(new URL(uri.toURL(), "/foo").toURI()),
+ 4096)) {
+ byte[] buf = new byte[data.length()];
+ IOUtils.readFully(is, buf, 0, buf.length);
+ assertEquals(data, new String(buf, StandardCharsets.UTF_8));
+ }
+ RecordedRequest req = server.takeRequest();
+ assertEquals("/foo", req.getPath());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff5ec3b8/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index fdb4fb1..c9b6522 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -146,6 +146,12 @@
<version>2.4.0</version>
</dependency>
<dependency>
+ <groupId>com.squareup.okhttp3</groupId>
+ <artifactId>mockwebserver</artifactId>
+ <version>3.7.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>jdiff</groupId>
<artifactId>jdiff</artifactId>
<version>${jdiff.version}</version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: YARN-6571. Fix JavaDoc issues in
SchedulingPolicy (Contributed by Weiwei Yang via Daniel Templeton)
Posted by in...@apache.org.
YARN-6571. Fix JavaDoc issues in SchedulingPolicy
(Contributed by Weiwei Yang via Daniel Templeton)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e7654c4a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e7654c4a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e7654c4a
Branch: refs/heads/HDFS-10467
Commit: e7654c4a1f3599a2032a2d02186af12124c23f7d
Parents: 3873904
Author: Daniel Templeton <te...@apache.org>
Authored: Wed May 10 14:30:34 2017 -0700
Committer: Daniel Templeton <te...@apache.org>
Committed: Wed May 10 14:34:13 2017 -0700
----------------------------------------------------------------------
.../scheduler/fair/SchedulingPolicy.java | 38 +++++++++++++++-----
1 file changed, 30 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e7654c4a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
index 9a9be8c..1fed9b0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
@@ -33,6 +33,21 @@ import java.util.Collection;
import java.util.Comparator;
import java.util.concurrent.ConcurrentHashMap;
+/**
+ * The SchedulingPolicy is used by the fair scheduler mainly to determine
+ * what a queue's fair share and steady fair share should be as well as
+ * calculating available headroom. This determines how resources can be
+ * shared between running applications within a queue.
+ * <p>
+ * Every queue has a policy, including parents and children. If a child
+ * queue doesn't specify one, it inherits the parent's policy.
+ * The policy for a child queue must be compatible with the policy of
+ * the parent queue; there are some combinations that aren't allowed.
+ * See {@link SchedulingPolicy#isChildPolicyAllowed(SchedulingPolicy)}.
+ * The policy for a queue is specified by setting property
+ * <i>schedulingPolicy</i> in the fair scheduler configuration file.
+ * The default policy is {@link FairSharePolicy} if not specified.
+ */
@Public
@Evolving
public abstract class SchedulingPolicy {
@@ -41,11 +56,16 @@ public abstract class SchedulingPolicy {
public static final SchedulingPolicy DEFAULT_POLICY =
getInstance(FairSharePolicy.class);
-
+
/**
- * Returns a {@link SchedulingPolicy} instance corresponding to the passed clazz
+ * Returns a {@link SchedulingPolicy} instance corresponding
+ * to the passed clazz.
+ *
+ * @param clazz a class that extends {@link SchedulingPolicy}
+ * @return a {@link SchedulingPolicy} instance
*/
- public static SchedulingPolicy getInstance(Class<? extends SchedulingPolicy> clazz) {
+ public static SchedulingPolicy getInstance(
+ Class<? extends SchedulingPolicy> clazz) {
SchedulingPolicy policy = ReflectionUtils.newInstance(clazz, null);
SchedulingPolicy policyRet = instances.putIfAbsent(clazz, policy);
if(policyRet != null) {
@@ -63,7 +83,9 @@ public abstract class SchedulingPolicy {
* canonical class name of the {@link SchedulingPolicy}.
*
* @param policy canonical class name or "drf" or "fair" or "fifo"
+ * @return a {@link SchedulingPolicy} instance parsed from given policy
* @throws AllocationConfigurationException
+ *
*/
@SuppressWarnings("unchecked")
public static SchedulingPolicy parse(String policy)
@@ -94,7 +116,7 @@ public abstract class SchedulingPolicy {
/**
* Initialize the scheduling policy with cluster resources.
- * @deprecated Since it doesn't track cluster resource changes, replaced by
+ * @deprecated Since it doesn't track cluster resource changes, replaced by
* {@link #initialize(FSContext)}.
*
* @param clusterCapacity cluster resources
@@ -158,7 +180,7 @@ public abstract class SchedulingPolicy {
Collection<? extends FSQueue> queues, Resource totalResources);
/**
- * Check if the resource usage is over the fair share under this policy
+ * Check if the resource usage is over the fair share under this policy.
*
* @param usage {@link Resource} the resource usage
* @param fairShare {@link Resource} the fair share
@@ -168,10 +190,10 @@ public abstract class SchedulingPolicy {
Resource usage, Resource fairShare);
/**
- * Get headroom by calculating the min of <code>clusterAvailable</code> and
- * (<code>queueFairShare</code> - <code>queueUsage</code>) resources that are
+ * Get headroom by calculating the min of {@code clusterAvailable} and
+ * ({@code queueFairShare} - {@code queueUsage}) resources that are
* applicable to this policy. For eg if only memory then leave other
- * resources such as CPU to same as clusterAvailable.
+ * resources such as CPU to same as {@code clusterAvailable}.
*
* @param queueFairShare fairshare in the queue
* @param queueUsage resources used in the queue
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: YARN-6457. Allow custom SSL
configuration to be supplied in WebApps. (Sanjay M Pujare via Haibo Chen)
Posted by in...@apache.org.
YARN-6457. Allow custom SSL configuration to be supplied in WebApps. (Sanjay M Pujare via Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1769b12a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1769b12a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1769b12a
Branch: refs/heads/HDFS-10467
Commit: 1769b12a773dc6c83f13663a77da33fa78878730
Parents: ff5ec3b
Author: Haibo Chen <ha...@cloudera.com>
Authored: Mon May 8 15:43:20 2017 -0700
Committer: Haibo Chen <ha...@cloudera.com>
Committed: Mon May 8 15:46:12 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/webapp/util/WebAppUtils.java | 17 +++++++++++------
1 file changed, 11 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1769b12a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
index 64a4b2b..eabd9b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
@@ -375,16 +375,21 @@ public class WebAppUtils {
/**
* Load the SSL keystore / truststore into the HttpServer builder.
* @param builder the HttpServer2.Builder to populate with ssl config
- * @param sslConf the Configuration instance to use during loading of SSL conf
+ * @param conf the Configuration instance to load custom SSL config from
+ *
+ * @return HttpServer2.Builder instance (passed in as the first parameter)
+ * after loading SSL stores
*/
public static HttpServer2.Builder loadSslConfiguration(
- HttpServer2.Builder builder, Configuration sslConf) {
- if (sslConf == null) {
- sslConf = new Configuration(false);
+ HttpServer2.Builder builder, Configuration conf) {
+
+ Configuration sslConf = new Configuration(false);
+
+ sslConf.addResource(YarnConfiguration.YARN_SSL_SERVER_RESOURCE_DEFAULT);
+ if (conf != null) {
+ sslConf.addResource(conf);
}
boolean needsClientAuth = YarnConfiguration.YARN_SSL_CLIENT_HTTPS_NEED_AUTH_DEFAULT;
- sslConf.addResource(YarnConfiguration.YARN_SSL_SERVER_RESOURCE_DEFAULT);
-
return builder
.needsClientAuth(needsClientAuth)
.keyPassword(getPassword(sslConf, WEB_APP_KEY_PASSWORD_KEY))
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: HDFS-11800. Document output of
'hdfs count -u' should contain PATHNAME. Contributed by Xiaobing Zhou
Posted by in...@apache.org.
HDFS-11800. Document output of 'hdfs count -u' should contain PATHNAME. Contributed by Xiaobing Zhou
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3873904a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3873904a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3873904a
Branch: refs/heads/HDFS-10467
Commit: 3873904a943fbbd5dcfd154e923b90c0610ed05b
Parents: ad1e3e4
Author: Mingliang Liu <li...@apache.org>
Authored: Wed May 10 14:33:33 2017 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Wed May 10 14:33:33 2017 -0700
----------------------------------------------------------------------
.../hadoop-common/src/site/markdown/FileSystemShell.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3873904a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index 6614b31..0412f1f 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -149,7 +149,7 @@ The -u and -q options control what columns the output contains. -q means show q
The output columns with -count -q are: QUOTA, REMAINING\_QUOTA, SPACE\_QUOTA, REMAINING\_SPACE\_QUOTA, DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME
-The output columns with -count -u are: QUOTA, REMAINING\_QUOTA, SPACE\_QUOTA, REMAINING\_SPACE\_QUOTA
+The output columns with -count -u are: QUOTA, REMAINING\_QUOTA, SPACE\_QUOTA, REMAINING\_SPACE\_QUOTA, PATHNAME
The -t option shows the quota and usage for each storage type. The -t option is ignored if -u or -q option is not given. The list of possible parameters that can be used in -t option(case insensitive except the parameter ""): "", "all", "ram_disk", "ssd", "disk" or "archive".
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: HDFS-11546. Federation Router RPC
server. Contributed by Jason Kace and Inigo Goiri.
Posted by in...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6632c5a4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
new file mode 100644
index 0000000..3a32be1
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
@@ -0,0 +1,856 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.net.InetSocketAddress;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadFactory;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.NameNodeProxiesClient.ProxyAndInfo;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.io.retry.RetryPolicies;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.io.retry.RetryPolicy.RetryAction.RetryDecision;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.ipc.StandbyException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+/**
+ * A client proxy for Router -> NN communication using the NN ClientProtocol.
+ * <p>
+ * Provides routers to invoke remote ClientProtocol methods and handle
+ * retries/failover.
+ * <ul>
+ * <li>invokeSingle Make a single request to a single namespace
+ * <li>invokeSequential Make a sequential series of requests to multiple
+ * ordered namespaces until a condition is met.
+ * <li>invokeConcurrent Make concurrent requests to multiple namespaces and
+ * return all of the results.
+ * </ul>
+ * Also maintains a cached pool of connections to NNs. Connections are managed
+ * by the ConnectionManager and are unique to each user + NN. The size of the
+ * connection pool can be configured. Larger pools allow for more simultaneous
+ * requests to a single NN from a single user.
+ */
+public class RouterRpcClient {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RouterRpcClient.class);
+
+
+ /** Router identifier. */
+ private final String routerId;
+
+ /** Interface to identify the active NN for a nameservice or blockpool ID. */
+ private final ActiveNamenodeResolver namenodeResolver;
+
+ /** Connection pool to the Namenodes per user for performance. */
+ private final ConnectionManager connectionManager;
+ /** Service to run asynchronous calls. */
+ private final ExecutorService executorService;
+ /** Retry policy for router -> NN communication. */
+ private final RetryPolicy retryPolicy;
+
+ /** Pattern to parse a stack trace line. */
+ private static final Pattern STACK_TRACE_PATTERN =
+ Pattern.compile("\\tat (.*)\\.(.*)\\((.*):(\\d*)\\)");
+
+
+ /**
+ * Create a router RPC client to manage remote procedure calls to NNs.
+ *
+ * @param conf Hdfs Configuation.
+ * @param resolver A NN resolver to determine the currently active NN in HA.
+ * @param monitor Optional performance monitor.
+ */
+ public RouterRpcClient(Configuration conf, String identifier,
+ ActiveNamenodeResolver resolver) {
+
+ this.routerId = identifier;
+
+ this.namenodeResolver = resolver;
+
+ this.connectionManager = new ConnectionManager(conf);
+ this.connectionManager.start();
+
+ ThreadFactory threadFactory = new ThreadFactoryBuilder()
+ .setNameFormat("RPC Router Client-%d")
+ .build();
+ this.executorService = Executors.newCachedThreadPool(threadFactory);
+
+ int maxFailoverAttempts = conf.getInt(
+ HdfsClientConfigKeys.Failover.MAX_ATTEMPTS_KEY,
+ HdfsClientConfigKeys.Failover.MAX_ATTEMPTS_DEFAULT);
+ int maxRetryAttempts = conf.getInt(
+ HdfsClientConfigKeys.Retry.MAX_ATTEMPTS_KEY,
+ HdfsClientConfigKeys.Retry.MAX_ATTEMPTS_DEFAULT);
+ int failoverSleepBaseMillis = conf.getInt(
+ HdfsClientConfigKeys.Failover.SLEEPTIME_BASE_KEY,
+ HdfsClientConfigKeys.Failover.SLEEPTIME_BASE_DEFAULT);
+ int failoverSleepMaxMillis = conf.getInt(
+ HdfsClientConfigKeys.Failover.SLEEPTIME_MAX_KEY,
+ HdfsClientConfigKeys.Failover.SLEEPTIME_MAX_DEFAULT);
+ this.retryPolicy = RetryPolicies.failoverOnNetworkException(
+ RetryPolicies.TRY_ONCE_THEN_FAIL, maxFailoverAttempts, maxRetryAttempts,
+ failoverSleepBaseMillis, failoverSleepMaxMillis);
+ }
+
+ /**
+ * Shutdown the client.
+ */
+ public void shutdown() {
+ if (this.connectionManager != null) {
+ this.connectionManager.close();
+ }
+ if (this.executorService != null) {
+ this.executorService.shutdownNow();
+ }
+ }
+
+ /**
+ * Total number of available sockets between the router and NNs.
+ *
+ * @return Number of namenode clients.
+ */
+ public int getNumConnections() {
+ return this.connectionManager.getNumConnections();
+ }
+
+ /**
+ * Total number of available sockets between the router and NNs.
+ *
+ * @return Number of namenode clients.
+ */
+ public int getNumActiveConnections() {
+ return this.connectionManager.getNumActiveConnections();
+ }
+
+ /**
+ * Total number of open connection pools to a NN. Each connection pool.
+ * represents one user + one NN.
+ *
+ * @return Number of connection pools.
+ */
+ public int getNumConnectionPools() {
+ return this.connectionManager.getNumConnectionPools();
+ }
+
+ /**
+ * Number of connections between the router and NNs being created sockets.
+ *
+ * @return Number of connections waiting to be created.
+ */
+ public int getNumCreatingConnections() {
+ return this.connectionManager.getNumCreatingConnections();
+ }
+
+ /**
+ * Get ClientProtocol proxy client for a NameNode. Each combination of user +
+ * NN must use a unique proxy client. Previously created clients are cached
+ * and stored in a connection pool by the ConnectionManager.
+ *
+ * @param ugi User group information.
+ * @param nsId Nameservice identifier.
+ * @param rpcAddress ClientProtocol RPC server address of the NN.
+ * @return ConnectionContext containing a ClientProtocol proxy client for the
+ * NN + current user.
+ * @throws IOException If we cannot get a connection to the NameNode.
+ */
+ private ConnectionContext getConnection(
+ UserGroupInformation ugi, String nsId, String rpcAddress)
+ throws IOException {
+ ConnectionContext connection = null;
+ try {
+ // Each proxy holds the UGI info for the current user when it is created.
+ // This cache does not scale very well, one entry per user per namenode,
+ // and may need to be adjusted and/or selectively pruned. The cache is
+ // important due to the excessive overhead of creating a new proxy wrapper
+ // for each individual request.
+
+ // TODO Add tokens from the federated UGI
+ connection = this.connectionManager.getConnection(ugi, rpcAddress);
+ LOG.debug("User {} NN {} is using connection {}",
+ ugi.getUserName(), rpcAddress, connection);
+ } catch (Exception ex) {
+ LOG.error("Cannot open NN client to address: {}", rpcAddress, ex);
+ }
+
+ if (connection == null) {
+ throw new IOException("Cannot get a connection to " + rpcAddress);
+ }
+ return connection;
+ }
+
+ /**
+ * Convert an exception to an IOException.
+ *
+ * For a non-IOException, wrap it with IOException. For a RemoteException,
+ * unwrap it. For an IOException which is not a RemoteException, return it.
+ *
+ * @param e Exception to convert into an exception.
+ * @return Created IO exception.
+ */
+ private static IOException toIOException(Exception e) {
+ if (e instanceof RemoteException) {
+ return ((RemoteException) e).unwrapRemoteException();
+ }
+ if (e instanceof IOException) {
+ return (IOException)e;
+ }
+ return new IOException(e);
+ }
+
+ /**
+ * If we should retry the RPC call.
+ *
+ * @param ex Exception reported.
+ * @param retryCount Number of retries.
+ * @return Retry decision.
+ * @throws IOException Original exception if the retry policy generates one.
+ */
+ private RetryDecision shouldRetry(final IOException ioe, final int retryCount)
+ throws IOException {
+ try {
+ final RetryPolicy.RetryAction a =
+ this.retryPolicy.shouldRetry(ioe, retryCount, 0, true);
+ return a.action;
+ } catch (Exception ex) {
+ LOG.error("Re-throwing API exception, no more retries", ex);
+ throw toIOException(ex);
+ }
+ }
+
+ /**
+ * Invokes a method against the ClientProtocol proxy server. If a standby
+ * exception is generated by the call to the client, retries using the
+ * alternate server.
+ *
+ * Re-throws exceptions generated by the remote RPC call as either
+ * RemoteException or IOException.
+ *
+ * @param ugi User group information.
+ * @param namenodes A prioritized list of namenodes within the same
+ * nameservice.
+ * @param method Remote ClientProtcol method to invoke.
+ * @param params Variable list of parameters matching the method.
+ * @return The result of invoking the method.
+ * @throws IOException
+ */
+ private Object invokeMethod(
+ final UserGroupInformation ugi,
+ final List<? extends FederationNamenodeContext> namenodes,
+ final Method method, final Object... params) throws IOException {
+
+ if (namenodes == null || namenodes.isEmpty()) {
+ throw new IOException("No namenodes to invoke " + method.getName() +
+ " with params " + Arrays.toString(params) + " from " + this.routerId);
+ }
+
+ Object ret = null;
+ boolean failover = false;
+ Map<FederationNamenodeContext, IOException> ioes = new LinkedHashMap<>();
+ for (FederationNamenodeContext namenode : namenodes) {
+ ConnectionContext connection = null;
+ try {
+ String nsId = namenode.getNameserviceId();
+ String rpcAddress = namenode.getRpcAddress();
+ connection = this.getConnection(ugi, nsId, rpcAddress);
+ ProxyAndInfo<ClientProtocol> client = connection.getClient();
+ ClientProtocol proxy = client.getProxy();
+ ret = invoke(0, method, proxy, params);
+ if (failover) {
+ // Success on alternate server, update
+ InetSocketAddress address = client.getAddress();
+ namenodeResolver.updateActiveNamenode(nsId, address);
+ }
+ return ret;
+ } catch (IOException ioe) {
+ ioes.put(namenode, ioe);
+ if (ioe instanceof StandbyException) {
+ // Fail over indicated by retry policy and/or NN
+ failover = true;
+ } else if (ioe instanceof RemoteException) {
+ // RemoteException returned by NN
+ throw (RemoteException) ioe;
+ } else {
+ // Other communication error, this is a failure
+ // Communication retries are handled by the retry policy
+ throw ioe;
+ }
+ } finally {
+ if (connection != null) {
+ connection.release();
+ }
+ }
+ }
+
+ // All namenodes were unavailable or in standby
+ String msg = "No namenode available to invoke " + method.getName() + " " +
+ Arrays.toString(params);
+ LOG.error(msg);
+ for (Entry<FederationNamenodeContext, IOException> entry :
+ ioes.entrySet()) {
+ FederationNamenodeContext namenode = entry.getKey();
+ String nsId = namenode.getNameserviceId();
+ String nnId = namenode.getNamenodeId();
+ String addr = namenode.getRpcAddress();
+ IOException ioe = entry.getValue();
+ if (ioe instanceof StandbyException) {
+ LOG.error("{} {} at {} is in Standby", nsId, nnId, addr);
+ } else {
+ LOG.error("{} {} at {} error: \"{}\"",
+ nsId, nnId, addr, ioe.getMessage());
+ }
+ }
+ throw new StandbyException(msg);
+ }
+
+ /**
+ * Invokes a method on the designated object. Catches exceptions specific to
+ * the invocation.
+ *
+ * Re-throws exceptions generated by the remote RPC call as either
+ * RemoteException or IOException.
+ *
+ * @param method Method to invoke
+ * @param obj Target object for the method
+ * @param params Variable parameters
+ * @return Response from the remote server
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ private Object invoke(int retryCount, final Method method, final Object obj,
+ final Object... params) throws IOException {
+ try {
+ return method.invoke(obj, params);
+ } catch (IllegalAccessException e) {
+ LOG.error("Unexpected exception while proxying API", e);
+ return null;
+ } catch (IllegalArgumentException e) {
+ LOG.error("Unexpected exception while proxying API", e);
+ return null;
+ } catch (InvocationTargetException e) {
+ Throwable cause = e.getCause();
+ if (cause instanceof IOException) {
+ IOException ioe = (IOException) cause;
+ // Check if we should retry.
+ RetryDecision decision = shouldRetry(ioe, retryCount);
+ if (decision == RetryDecision.RETRY) {
+ // retry
+ return invoke(++retryCount, method, obj, params);
+ } else if (decision == RetryDecision.FAILOVER_AND_RETRY) {
+ // failover, invoker looks for standby exceptions for failover.
+ if (ioe instanceof StandbyException) {
+ throw ioe;
+ } else {
+ throw new StandbyException(ioe.getMessage());
+ }
+ } else {
+ if (ioe instanceof RemoteException) {
+ RemoteException re = (RemoteException) ioe;
+ ioe = re.unwrapRemoteException();
+ ioe = getCleanException(ioe);
+ }
+ throw ioe;
+ }
+ } else {
+ throw new IOException(e);
+ }
+ }
+ }
+
+ /**
+ * Get a clean copy of the exception. Sometimes the exceptions returned by the
+ * server contain the full stack trace in the message.
+ *
+ * @param ioe Exception to clean up.
+ * @return Copy of the original exception with a clean message.
+ */
+ private static IOException getCleanException(IOException ioe) {
+ IOException ret = null;
+
+ String msg = ioe.getMessage();
+ Throwable cause = ioe.getCause();
+ StackTraceElement[] stackTrace = ioe.getStackTrace();
+
+ // Clean the message by removing the stack trace
+ int index = msg.indexOf("\n");
+ if (index > 0) {
+ String[] msgSplit = msg.split("\n");
+ msg = msgSplit[0];
+
+ // Parse stack trace from the message
+ List<StackTraceElement> elements = new LinkedList<>();
+ for (int i=1; i<msgSplit.length; i++) {
+ String line = msgSplit[i];
+ Matcher matcher = STACK_TRACE_PATTERN.matcher(line);
+ if (matcher.find()) {
+ String declaringClass = matcher.group(1);
+ String methodName = matcher.group(2);
+ String fileName = matcher.group(3);
+ int lineNumber = Integer.parseInt(matcher.group(4));
+ StackTraceElement element = new StackTraceElement(
+ declaringClass, methodName, fileName, lineNumber);
+ elements.add(element);
+ }
+ }
+ stackTrace = elements.toArray(new StackTraceElement[elements.size()]);
+ }
+
+ // Create the new output exception
+ if (ioe instanceof RemoteException) {
+ RemoteException re = (RemoteException)ioe;
+ ret = new RemoteException(re.getClassName(), msg);
+ } else {
+ // Try the simple constructor and initialize the fields
+ Class<? extends IOException> ioeClass = ioe.getClass();
+ try {
+ Constructor<? extends IOException> constructor =
+ ioeClass.getDeclaredConstructor(String.class);
+ ret = constructor.newInstance(msg);
+ } catch (ReflectiveOperationException e) {
+ // If there are errors, just use the input one
+ LOG.error("Could not create exception {}", ioeClass.getSimpleName(), e);
+ ret = ioe;
+ }
+ }
+ if (ret != null) {
+ ret.initCause(cause);
+ ret.setStackTrace(stackTrace);
+ }
+
+ return ret;
+ }
+
+ /**
+ * Invokes a ClientProtocol method. Determines the target nameservice via a
+ * provided block.
+ *
+ * Re-throws exceptions generated by the remote RPC call as either
+ * RemoteException or IOException.
+ *
+ * @param block Block used to determine appropriate nameservice.
+ * @param method The remote method and parameters to invoke.
+ * @return The result of invoking the method.
+ * @throws IOException
+ */
+ public Object invokeSingle(final ExtendedBlock block, RemoteMethod method)
+ throws IOException {
+ String bpId = block.getBlockPoolId();
+ return invokeSingleBlockPool(bpId, method);
+ }
+
+ /**
+ * Invokes a ClientProtocol method. Determines the target nameservice using
+ * the block pool id.
+ *
+ * Re-throws exceptions generated by the remote RPC call as either
+ * RemoteException or IOException.
+ *
+ * @param bpId Block pool identifier.
+ * @param method The remote method and parameters to invoke.
+ * @return The result of invoking the method.
+ * @throws IOException
+ */
+ public Object invokeSingleBlockPool(final String bpId, RemoteMethod method)
+ throws IOException {
+ String nsId = getNameserviceForBlockPoolId(bpId);
+ return invokeSingle(nsId, method);
+ }
+
+ /**
+ * Invokes a ClientProtocol method against the specified namespace.
+ *
+ * Re-throws exceptions generated by the remote RPC call as either
+ * RemoteException or IOException.
+ *
+ * @param nsId Target namespace for the method.
+ * @param method The remote method and parameters to invoke.
+ * @return The result of invoking the method.
+ * @throws IOException
+ */
+ public Object invokeSingle(final String nsId, RemoteMethod method)
+ throws IOException {
+ UserGroupInformation ugi = RouterRpcServer.getRemoteUser();
+ List<? extends FederationNamenodeContext> nns =
+ getNamenodesForNameservice(nsId);
+ RemoteLocationContext loc = new RemoteLocation(nsId, "/");
+ return invokeMethod(ugi, nns, method.getMethod(), method.getParams(loc));
+ }
+
+ /**
+ * Invokes a single proxy call for a single location.
+ *
+ * Re-throws exceptions generated by the remote RPC call as either
+ * RemoteException or IOException.
+ *
+ * @param location RemoteLocation to invoke.
+ * @param remoteMethod The remote method and parameters to invoke.
+ * @return The result of invoking the method if successful.
+ * @throws IOException
+ */
+ public Object invokeSingle(final RemoteLocationContext location,
+ RemoteMethod remoteMethod) throws IOException {
+ List<RemoteLocationContext> locations = Collections.singletonList(location);
+ return invokeSequential(locations, remoteMethod);
+ }
+
+ /**
+ * Invokes sequential proxy calls to different locations. Continues to invoke
+ * calls until a call returns without throwing a remote exception.
+ *
+ * @param locations List of locations/nameservices to call concurrently.
+ * @param remoteMethod The remote method and parameters to invoke.
+ * @return The result of the first successful call, or if no calls are
+ * successful, the result of the last RPC call executed.
+ * @throws IOException if the success condition is not met and one of the RPC
+ * calls generated a remote exception.
+ */
+ public Object invokeSequential(
+ final List<? extends RemoteLocationContext> locations,
+ final RemoteMethod remoteMethod) throws IOException {
+ return invokeSequential(locations, remoteMethod, null, null);
+ }
+
+ /**
+ * Invokes sequential proxy calls to different locations. Continues to invoke
+ * calls until the success condition is met, or until all locations have been
+ * attempted.
+ *
+ * The success condition may be specified by:
+ * <ul>
+ * <li>An expected result class
+ * <li>An expected result value
+ * </ul>
+ *
+ * If no expected result class/values are specified, the success condition is
+ * a call that does not throw a remote exception.
+ *
+ * @param locations List of locations/nameservices to call concurrently.
+ * @param remoteMethod The remote method and parameters to invoke.
+ * @param expectedResultClass In order to be considered a positive result, the
+ * return type must be of this class.
+ * @param expectedResultValue In order to be considered a positive result, the
+ * return value must equal the value of this object.
+ * @return The result of the first successful call, or if no calls are
+ * successful, the result of the first RPC call executed.
+ * @throws IOException if the success condition is not met, return the first
+ * remote exception generated.
+ */
+ public Object invokeSequential(
+ final List<? extends RemoteLocationContext> locations,
+ final RemoteMethod remoteMethod, Class<?> expectedResultClass,
+ Object expectedResultValue) throws IOException {
+
+ final UserGroupInformation ugi = RouterRpcServer.getRemoteUser();
+ final Method m = remoteMethod.getMethod();
+ IOException firstThrownException = null;
+ IOException lastThrownException = null;
+ Object firstResult = null;
+ // Invoke in priority order
+ for (final RemoteLocationContext loc : locations) {
+ String ns = loc.getNameserviceId();
+ List<? extends FederationNamenodeContext> namenodes =
+ getNamenodesForNameservice(ns);
+ try {
+ Object[] params = remoteMethod.getParams(loc);
+ Object result = invokeMethod(ugi, namenodes, m, params);
+ // Check if the result is what we expected
+ if (isExpectedClass(expectedResultClass, result) &&
+ isExpectedValue(expectedResultValue, result)) {
+ // Valid result, stop here
+ return result;
+ }
+ if (firstResult == null) {
+ firstResult = result;
+ }
+ } catch (IOException ioe) {
+ // Record it and move on
+ lastThrownException = (IOException) ioe;
+ if (firstThrownException == null) {
+ firstThrownException = lastThrownException;
+ }
+ } catch (Exception e) {
+ // Unusual error, ClientProtocol calls always use IOException (or
+ // RemoteException). Re-wrap in IOException for compatibility with
+ // ClientProtcol.
+ LOG.error("Unexpected exception {} proxying {} to {}",
+ e.getClass(), m.getName(), ns, e);
+ lastThrownException = new IOException(
+ "Unexpected exception proxying API " + e.getMessage(), e);
+ if (firstThrownException == null) {
+ firstThrownException = lastThrownException;
+ }
+ }
+ }
+
+ if (firstThrownException != null) {
+ // re-throw the last exception thrown for compatibility
+ throw firstThrownException;
+ }
+ // Return the last result, whether it is the value we are looking for or a
+ return firstResult;
+ }
+
+ /**
+ * Checks if a result matches the required result class.
+ *
+ * @param expectedResultClass Required result class, null to skip the check.
+ * @param result The result to check.
+ * @return True if the result is an instance of the required class or if the
+ * expected class is null.
+ */
+ private static boolean isExpectedClass(Class<?> expectedClass, Object clazz) {
+ if (expectedClass == null) {
+ return true;
+ } else if (clazz == null) {
+ return false;
+ } else {
+ return expectedClass.isInstance(clazz);
+ }
+ }
+
+ /**
+ * Checks if a result matches the expected value.
+ *
+ * @param expectedResultValue The expected value, null to skip the check.
+ * @param result The result to check.
+ * @return True if the result is equals to the expected value or if the
+ * expected value is null.
+ */
+ private static boolean isExpectedValue(Object expectedValue, Object value) {
+ if (expectedValue == null) {
+ return true;
+ } else if (value == null) {
+ return false;
+ } else {
+ return value.equals(expectedValue);
+ }
+ }
+
+ /**
+ * Invokes multiple concurrent proxy calls to different clients. Returns an
+ * array of results.
+ *
+ * Re-throws exceptions generated by the remote RPC call as either
+ * RemoteException or IOException.
+ *
+ * @param locations List of remote locations to call concurrently.
+ * @param remoteMethod The remote method and parameters to invoke.
+ * @param requireResponse If true an exception will be thrown if all calls do
+ * not complete. If false exceptions are ignored and all data results
+ * successfully received are returned.
+ * @param standby If the requests should go to the standby namenodes too.
+ * @return Result of invoking the method per subcluster: nsId -> result.
+ * @throws IOException If requiredResponse=true and any of the calls throw an
+ * exception.
+ */
+ @SuppressWarnings("unchecked")
+ public <T extends RemoteLocationContext> Map<T, Object> invokeConcurrent(
+ final Collection<T> locations, final RemoteMethod method,
+ boolean requireResponse, boolean standby) throws IOException {
+
+ final UserGroupInformation ugi = RouterRpcServer.getRemoteUser();
+ final Method m = method.getMethod();
+
+ if (locations.size() == 1) {
+ // Shortcut, just one call
+ T location = locations.iterator().next();
+ String ns = location.getNameserviceId();
+ final List<? extends FederationNamenodeContext> namenodes =
+ getNamenodesForNameservice(ns);
+ Object[] paramList = method.getParams(location);
+ Object result = invokeMethod(ugi, namenodes, m, paramList);
+ return Collections.singletonMap(location, result);
+ }
+
+ List<T> orderedLocations = new LinkedList<>();
+ Set<Callable<Object>> callables = new HashSet<>();
+ for (final T location : locations) {
+ String nsId = location.getNameserviceId();
+ final List<? extends FederationNamenodeContext> namenodes =
+ getNamenodesForNameservice(nsId);
+ final Object[] paramList = method.getParams(location);
+ if (standby) {
+ // Call the objectGetter to all NNs (including standby)
+ for (final FederationNamenodeContext nn : namenodes) {
+ String nnId = nn.getNamenodeId();
+ final List<FederationNamenodeContext> nnList =
+ Collections.singletonList(nn);
+ T nnLocation = location;
+ if (location instanceof RemoteLocation) {
+ nnLocation = (T)new RemoteLocation(nsId, nnId, location.getDest());
+ }
+ orderedLocations.add(nnLocation);
+ callables.add(new Callable<Object>() {
+ public Object call() throws Exception {
+ return invokeMethod(ugi, nnList, m, paramList);
+ }
+ });
+ }
+ } else {
+ // Call the objectGetter in order of nameservices in the NS list
+ orderedLocations.add(location);
+ callables.add(new Callable<Object>() {
+ public Object call() throws Exception {
+ return invokeMethod(ugi, namenodes, m, paramList);
+ }
+ });
+ }
+ }
+
+ try {
+ List<Future<Object>> futures = executorService.invokeAll(callables);
+ Map<T, Object> results = new TreeMap<>();
+ Map<T, IOException> exceptions = new TreeMap<>();
+ for (int i=0; i<futures.size(); i++) {
+ T location = orderedLocations.get(i);
+ try {
+ Future<Object> future = futures.get(i);
+ Object result = future.get();
+ results.put(location, result);
+ } catch (ExecutionException ex) {
+ Throwable cause = ex.getCause();
+ LOG.debug("Canot execute {} in {}: {}",
+ m.getName(), location, cause.getMessage());
+
+ // Convert into IOException if needed
+ IOException ioe = null;
+ if (cause instanceof IOException) {
+ ioe = (IOException) cause;
+ } else {
+ ioe = new IOException("Unhandled exception while proxying API " +
+ m.getName() + ": " + cause.getMessage(), cause);
+ }
+
+ // Response from all servers required, use this error.
+ if (requireResponse) {
+ throw ioe;
+ }
+
+ // Store the exceptions
+ exceptions.put(location, ioe);
+ }
+ }
+
+ // Throw the exception for the first location if there are no results
+ if (results.isEmpty()) {
+ T location = orderedLocations.get(0);
+ IOException ioe = exceptions.get(location);
+ if (ioe != null) {
+ throw ioe;
+ }
+ }
+
+ return results;
+ } catch (InterruptedException ex) {
+ LOG.error("Unexpected error while invoking API: {}", ex.getMessage());
+ throw new IOException(
+ "Unexpected error while invoking API " + ex.getMessage(), ex);
+ }
+ }
+
+ /**
+ * Get a prioritized list of NNs that share the same nameservice ID (in the
+ * same namespace). NNs that are reported as ACTIVE will be first in the list.
+ *
+ * @param nameserviceId The nameservice ID for the namespace.
+ * @return A prioritized list of NNs to use for communication.
+ * @throws IOException If a NN cannot be located for the nameservice ID.
+ */
+ private List<? extends FederationNamenodeContext> getNamenodesForNameservice(
+ final String nsId) throws IOException {
+
+ final List<? extends FederationNamenodeContext> namenodes =
+ namenodeResolver.getNamenodesForNameserviceId(nsId);
+
+ if (namenodes == null || namenodes.isEmpty()) {
+ throw new IOException("Cannot locate a registered namenode for " + nsId +
+ " from " + this.routerId);
+ }
+ return namenodes;
+ }
+
+ /**
+ * Get a prioritized list of NNs that share the same block pool ID (in the
+ * same namespace). NNs that are reported as ACTIVE will be first in the list.
+ *
+ * @param blockPoolId The blockpool ID for the namespace.
+ * @return A prioritized list of NNs to use for communication.
+ * @throws IOException If a NN cannot be located for the block pool ID.
+ */
+ private List<? extends FederationNamenodeContext> getNamenodesForBlockPoolId(
+ final String bpId) throws IOException {
+
+ List<? extends FederationNamenodeContext> namenodes =
+ namenodeResolver.getNamenodesForBlockPoolId(bpId);
+
+ if (namenodes == null || namenodes.isEmpty()) {
+ throw new IOException("Cannot locate a registered namenode for " + bpId +
+ " from " + this.routerId);
+ }
+ return namenodes;
+ }
+
+ /**
+ * Get the nameservice identifier for a block pool.
+ *
+ * @param bpId Identifier of the block pool.
+ * @return Nameservice identifier.
+ * @throws IOException If a NN cannot be located for the block pool ID.
+ */
+ private String getNameserviceForBlockPoolId(final String bpId)
+ throws IOException {
+ List<? extends FederationNamenodeContext> namenodes =
+ getNamenodesForBlockPoolId(bpId);
+ FederationNamenodeContext namenode = namenodes.get(0);
+ return namenode.getNameserviceId();
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: YARN-6475. Fix some long function
checkstyle issues (Contributed by Soumabrata Chakraborty via Daniel
Templeton)
Posted by in...@apache.org.
YARN-6475. Fix some long function checkstyle issues
(Contributed by Soumabrata Chakraborty via Daniel Templeton)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/74a61438
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/74a61438
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/74a61438
Branch: refs/heads/HDFS-10467
Commit: 74a61438ca01e2191b54000af73b654a2d0b8253
Parents: 1e71fe8
Author: Daniel Templeton <te...@apache.org>
Authored: Wed May 10 10:45:02 2017 -0700
Committer: Daniel Templeton <te...@apache.org>
Committed: Wed May 10 10:46:50 2017 -0700
----------------------------------------------------------------------
.../nodemanager/LinuxContainerExecutor.java | 92 ++---
.../nodemanager/NodeStatusUpdaterImpl.java | 390 ++++++++++---------
.../launcher/ContainerLaunch.java | 238 ++++++-----
.../runtime/DockerLinuxContainerRuntime.java | 85 ++--
4 files changed, 429 insertions(+), 376 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/74a61438/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index cb1d53d..9a3b2d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -62,7 +62,6 @@ import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import java.util.Map;
import java.util.regex.Pattern;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*;
@@ -442,24 +441,11 @@ public class LinuxContainerExecutor extends ContainerExecutor {
public int launchContainer(ContainerStartContext ctx)
throws IOException, ConfigurationException {
Container container = ctx.getContainer();
- Path nmPrivateContainerScriptPath = ctx.getNmPrivateContainerScriptPath();
- Path nmPrivateTokensPath = ctx.getNmPrivateTokensPath();
String user = ctx.getUser();
- String appId = ctx.getAppId();
- Path containerWorkDir = ctx.getContainerWorkDir();
- List<String> localDirs = ctx.getLocalDirs();
- List<String> logDirs = ctx.getLogDirs();
- List<String> filecacheDirs = ctx.getFilecacheDirs();
- List<String> userLocalDirs = ctx.getUserLocalDirs();
- List<String> containerLocalDirs = ctx.getContainerLocalDirs();
- List<String> containerLogDirs = ctx.getContainerLogDirs();
- Map<Path, List<String>> localizedResources = ctx.getLocalizedResources();
verifyUsernamePattern(user);
- String runAsUser = getRunAsUser(user);
ContainerId containerId = container.getContainerId();
- String containerIdStr = containerId.toString();
resourcesHandler.preExecute(containerId,
container.getResource());
@@ -514,39 +500,11 @@ public class LinuxContainerExecutor extends ContainerExecutor {
try {
Path pidFilePath = getPidFilePath(containerId);
if (pidFilePath != null) {
- List<String> prefixCommands = new ArrayList<>();
- ContainerRuntimeContext.Builder builder = new ContainerRuntimeContext
- .Builder(container);
- addSchedPriorityCommand(prefixCommands);
- if (prefixCommands.size() > 0) {
- builder.setExecutionAttribute(CONTAINER_LAUNCH_PREFIX_COMMANDS,
- prefixCommands);
- }
+ ContainerRuntimeContext runtimeContext = buildContainerRuntimeContext(
+ ctx, pidFilePath, resourcesOptions, tcCommandFile);
- builder.setExecutionAttribute(LOCALIZED_RESOURCES, localizedResources)
- .setExecutionAttribute(RUN_AS_USER, runAsUser)
- .setExecutionAttribute(USER, user)
- .setExecutionAttribute(APPID, appId)
- .setExecutionAttribute(CONTAINER_ID_STR, containerIdStr)
- .setExecutionAttribute(CONTAINER_WORK_DIR, containerWorkDir)
- .setExecutionAttribute(NM_PRIVATE_CONTAINER_SCRIPT_PATH,
- nmPrivateContainerScriptPath)
- .setExecutionAttribute(NM_PRIVATE_TOKENS_PATH, nmPrivateTokensPath)
- .setExecutionAttribute(PID_FILE_PATH, pidFilePath)
- .setExecutionAttribute(LOCAL_DIRS, localDirs)
- .setExecutionAttribute(LOG_DIRS, logDirs)
- .setExecutionAttribute(FILECACHE_DIRS, filecacheDirs)
- .setExecutionAttribute(USER_LOCAL_DIRS, userLocalDirs)
- .setExecutionAttribute(CONTAINER_LOCAL_DIRS, containerLocalDirs)
- .setExecutionAttribute(CONTAINER_LOG_DIRS, containerLogDirs)
- .setExecutionAttribute(RESOURCES_OPTIONS, resourcesOptions);
-
- if (tcCommandFile != null) {
- builder.setExecutionAttribute(TC_COMMAND_FILE, tcCommandFile);
- }
-
- linuxContainerRuntime.launchContainer(builder.build());
+ linuxContainerRuntime.launchContainer(runtimeContext);
} else {
LOG.info(
"Container was marked as inactive. Returning terminated error");
@@ -617,6 +575,50 @@ public class LinuxContainerExecutor extends ContainerExecutor {
return 0;
}
+ private ContainerRuntimeContext buildContainerRuntimeContext(
+ ContainerStartContext ctx, Path pidFilePath,
+ String resourcesOptions, String tcCommandFile) {
+
+ List<String> prefixCommands = new ArrayList<>();
+ addSchedPriorityCommand(prefixCommands);
+
+ Container container = ctx.getContainer();
+
+ ContainerRuntimeContext.Builder builder = new ContainerRuntimeContext
+ .Builder(container);
+ if (prefixCommands.size() > 0) {
+ builder.setExecutionAttribute(CONTAINER_LAUNCH_PREFIX_COMMANDS,
+ prefixCommands);
+ }
+
+ builder.setExecutionAttribute(LOCALIZED_RESOURCES,
+ ctx.getLocalizedResources())
+ .setExecutionAttribute(RUN_AS_USER, getRunAsUser(ctx.getUser()))
+ .setExecutionAttribute(USER, ctx.getUser())
+ .setExecutionAttribute(APPID, ctx.getAppId())
+ .setExecutionAttribute(CONTAINER_ID_STR,
+ container.getContainerId().toString())
+ .setExecutionAttribute(CONTAINER_WORK_DIR, ctx.getContainerWorkDir())
+ .setExecutionAttribute(NM_PRIVATE_CONTAINER_SCRIPT_PATH,
+ ctx.getNmPrivateContainerScriptPath())
+ .setExecutionAttribute(NM_PRIVATE_TOKENS_PATH,
+ ctx.getNmPrivateTokensPath())
+ .setExecutionAttribute(PID_FILE_PATH, pidFilePath)
+ .setExecutionAttribute(LOCAL_DIRS, ctx.getLocalDirs())
+ .setExecutionAttribute(LOG_DIRS, ctx.getLogDirs())
+ .setExecutionAttribute(FILECACHE_DIRS, ctx.getFilecacheDirs())
+ .setExecutionAttribute(USER_LOCAL_DIRS, ctx.getUserLocalDirs())
+ .setExecutionAttribute(CONTAINER_LOCAL_DIRS, ctx.getContainerLocalDirs())
+ .setExecutionAttribute(CONTAINER_LOG_DIRS, ctx.getContainerLogDirs())
+ .setExecutionAttribute(RESOURCES_OPTIONS, resourcesOptions);
+
+ if (tcCommandFile != null) {
+ builder.setExecutionAttribute(TC_COMMAND_FILE, tcCommandFile);
+ }
+
+ return builder.build();
+ }
+
@Override
public String[] getIpAndHost(Container container) {
return linuxContainerRuntime.getIpAndHost(container);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/74a61438/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index dd5b279..00073d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -761,200 +761,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
protected void startStatusUpdater() {
- statusUpdaterRunnable = new Runnable() {
- @Override
- @SuppressWarnings("unchecked")
- public void run() {
- int lastHeartbeatID = 0;
- while (!isStopped) {
- // Send heartbeat
- try {
- NodeHeartbeatResponse response = null;
- Set<NodeLabel> nodeLabelsForHeartbeat =
- nodeLabelsHandler.getNodeLabelsForHeartbeat();
- NodeStatus nodeStatus = getNodeStatus(lastHeartbeatID);
- NodeHeartbeatRequest request =
- NodeHeartbeatRequest.newInstance(nodeStatus,
- NodeStatusUpdaterImpl.this.context
- .getContainerTokenSecretManager().getCurrentKey(),
- NodeStatusUpdaterImpl.this.context
- .getNMTokenSecretManager().getCurrentKey(),
- nodeLabelsForHeartbeat,
- NodeStatusUpdaterImpl.this.context
- .getRegisteredCollectors());
-
- if (logAggregationEnabled) {
- // pull log aggregation status for application running in this NM
- List<LogAggregationReport> logAggregationReports =
- getLogAggregationReportsForApps(context
- .getLogAggregationStatusForApps());
- if (logAggregationReports != null
- && !logAggregationReports.isEmpty()) {
- request.setLogAggregationReportsForApps(logAggregationReports);
- }
- }
-
- response = resourceTracker.nodeHeartbeat(request);
- //get next heartbeat interval from response
- nextHeartBeatInterval = response.getNextHeartBeatInterval();
- updateMasterKeys(response);
-
- if (!handleShutdownOrResyncCommand(response)) {
- nodeLabelsHandler.verifyRMHeartbeatResponseForNodeLabels(
- response);
-
- // Explicitly put this method after checking the resync
- // response. We
- // don't want to remove the completed containers before resync
- // because these completed containers will be reported back to RM
- // when NM re-registers with RM.
- // Only remove the cleanedup containers that are acked
- removeOrTrackCompletedContainersFromContext(response
- .getContainersToBeRemovedFromNM());
-
- logAggregationReportForAppsTempList.clear();
- lastHeartbeatID = response.getResponseId();
- List<ContainerId> containersToCleanup = response
- .getContainersToCleanup();
- if (!containersToCleanup.isEmpty()) {
- dispatcher.getEventHandler().handle(
- new CMgrCompletedContainersEvent(containersToCleanup,
- CMgrCompletedContainersEvent.Reason
- .BY_RESOURCEMANAGER));
- }
- List<ApplicationId> appsToCleanup =
- response.getApplicationsToCleanup();
- //Only start tracking for keepAlive on FINISH_APP
- trackAppsForKeepAlive(appsToCleanup);
- if (!appsToCleanup.isEmpty()) {
- dispatcher.getEventHandler().handle(
- new CMgrCompletedAppsEvent(appsToCleanup,
- CMgrCompletedAppsEvent.Reason.BY_RESOURCEMANAGER));
- }
- Map<ApplicationId, ByteBuffer> systemCredentials =
- response.getSystemCredentialsForApps();
- if (systemCredentials != null && !systemCredentials.isEmpty()) {
- ((NMContext) context).setSystemCrendentialsForApps(
- parseCredentials(systemCredentials));
- }
- List<org.apache.hadoop.yarn.api.records.Container>
- containersToDecrease = response.getContainersToDecrease();
- if (!containersToDecrease.isEmpty()) {
- dispatcher.getEventHandler().handle(
- new CMgrDecreaseContainersResourceEvent(
- containersToDecrease)
- );
- }
-
- // SignalContainer request originally comes from end users via
- // ClientRMProtocol's SignalContainer. Forward the request to
- // ContainerManager which will dispatch the event to
- // ContainerLauncher.
- List<SignalContainerRequest> containersToSignal = response
- .getContainersToSignalList();
- if (containersToSignal.size() != 0) {
- dispatcher.getEventHandler().handle(
- new CMgrSignalContainersEvent(containersToSignal));
- }
-
- // Update QueuingLimits if ContainerManager supports queuing
- ContainerQueuingLimit queuingLimit =
- response.getContainerQueuingLimit();
- if (queuingLimit != null) {
- context.getContainerManager().updateQueuingLimit(queuingLimit);
- }
- }
- // Handling node resource update case.
- Resource newResource = response.getResource();
- if (newResource != null) {
- updateNMResource(newResource);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Node's resource is updated to " +
- newResource.toString());
- }
- }
- if (YarnConfiguration.timelineServiceV2Enabled(context.getConf())) {
- updateTimelineClientsAddress(response);
- }
-
- } catch (ConnectException e) {
- //catch and throw the exception if tried MAX wait time to connect RM
- dispatcher.getEventHandler().handle(
- new NodeManagerEvent(NodeManagerEventType.SHUTDOWN));
- // failed to connect to RM.
- failedToConnect = true;
- throw new YarnRuntimeException(e);
- } catch (Throwable e) {
-
- // TODO Better error handling. Thread can die with the rest of the
- // NM still running.
- LOG.error("Caught exception in status-updater", e);
- } finally {
- synchronized (heartbeatMonitor) {
- nextHeartBeatInterval = nextHeartBeatInterval <= 0 ?
- YarnConfiguration.DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS :
- nextHeartBeatInterval;
- try {
- heartbeatMonitor.wait(nextHeartBeatInterval);
- } catch (InterruptedException e) {
- // Do Nothing
- }
- }
- }
- }
- }
-
- private void updateTimelineClientsAddress(
- NodeHeartbeatResponse response) {
- Map<ApplicationId, String> knownCollectorsMap =
- response.getAppCollectorsMap();
- if (knownCollectorsMap == null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("No collectors to update RM");
- }
- } else {
- Set<Map.Entry<ApplicationId, String>> rmKnownCollectors =
- knownCollectorsMap.entrySet();
- for (Map.Entry<ApplicationId, String> entry : rmKnownCollectors) {
- ApplicationId appId = entry.getKey();
- String collectorAddr = entry.getValue();
-
- // Only handle applications running on local node.
- // Not include apps with timeline collectors running in local
- Application application = context.getApplications().get(appId);
- // TODO this logic could be problematic if the collector address
- // gets updated due to NM restart or collector service failure
- if (application != null &&
- !context.getRegisteredCollectors().containsKey(appId)) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Sync a new collector address: " + collectorAddr +
- " for application: " + appId + " from RM.");
- }
- NMTimelinePublisher nmTimelinePublisher =
- context.getNMTimelinePublisher();
- if (nmTimelinePublisher != null) {
- nmTimelinePublisher.setTimelineServiceAddress(
- application.getAppId(), collectorAddr);
- }
- }
- }
- }
- }
-
- private void updateMasterKeys(NodeHeartbeatResponse response) {
- // See if the master-key has rolled over
- MasterKey updatedMasterKey = response.getContainerTokenMasterKey();
- if (updatedMasterKey != null) {
- // Will be non-null only on roll-over on RM side
- context.getContainerTokenSecretManager().setMasterKey(updatedMasterKey);
- }
-
- updatedMasterKey = response.getNMTokenMasterKey();
- if (updatedMasterKey != null) {
- context.getNMTokenSecretManager().setMasterKey(updatedMasterKey);
- }
- }
- };
+ statusUpdaterRunnable = new StatusUpdaterRunnable();
statusUpdater =
new Thread(statusUpdaterRunnable, "Node Status Updater");
statusUpdater.start();
@@ -1215,4 +1022,199 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
}
}
}
+
+ private class StatusUpdaterRunnable implements Runnable {
+ @Override
+ @SuppressWarnings("unchecked")
+ public void run() {
+ int lastHeartbeatID = 0;
+ while (!isStopped) {
+ // Send heartbeat
+ try {
+ NodeHeartbeatResponse response = null;
+ Set<NodeLabel> nodeLabelsForHeartbeat =
+ nodeLabelsHandler.getNodeLabelsForHeartbeat();
+ NodeStatus nodeStatus = getNodeStatus(lastHeartbeatID);
+ NodeHeartbeatRequest request =
+ NodeHeartbeatRequest.newInstance(nodeStatus,
+ NodeStatusUpdaterImpl.this.context
+ .getContainerTokenSecretManager().getCurrentKey(),
+ NodeStatusUpdaterImpl.this.context
+ .getNMTokenSecretManager().getCurrentKey(),
+ nodeLabelsForHeartbeat,
+ NodeStatusUpdaterImpl.this.context
+ .getRegisteredCollectors());
+
+ if (logAggregationEnabled) {
+ // pull log aggregation status for application running in this NM
+ List<LogAggregationReport> logAggregationReports =
+ getLogAggregationReportsForApps(context
+ .getLogAggregationStatusForApps());
+ if (logAggregationReports != null
+ && !logAggregationReports.isEmpty()) {
+ request.setLogAggregationReportsForApps(logAggregationReports);
+ }
+ }
+
+ response = resourceTracker.nodeHeartbeat(request);
+ //get next heartbeat interval from response
+ nextHeartBeatInterval = response.getNextHeartBeatInterval();
+ updateMasterKeys(response);
+
+ if (!handleShutdownOrResyncCommand(response)) {
+ nodeLabelsHandler.verifyRMHeartbeatResponseForNodeLabels(
+ response);
+
+ // Explicitly put this method after checking the resync
+ // response. We
+ // don't want to remove the completed containers before resync
+ // because these completed containers will be reported back to RM
+ // when NM re-registers with RM.
+ // Only remove the cleanedup containers that are acked
+ removeOrTrackCompletedContainersFromContext(response
+ .getContainersToBeRemovedFromNM());
+
+ logAggregationReportForAppsTempList.clear();
+ lastHeartbeatID = response.getResponseId();
+ List<ContainerId> containersToCleanup = response
+ .getContainersToCleanup();
+ if (!containersToCleanup.isEmpty()) {
+ dispatcher.getEventHandler().handle(
+ new CMgrCompletedContainersEvent(containersToCleanup,
+ CMgrCompletedContainersEvent.Reason
+ .BY_RESOURCEMANAGER));
+ }
+ List<ApplicationId> appsToCleanup =
+ response.getApplicationsToCleanup();
+ //Only start tracking for keepAlive on FINISH_APP
+ trackAppsForKeepAlive(appsToCleanup);
+ if (!appsToCleanup.isEmpty()) {
+ dispatcher.getEventHandler().handle(
+ new CMgrCompletedAppsEvent(appsToCleanup,
+ CMgrCompletedAppsEvent.Reason.BY_RESOURCEMANAGER));
+ }
+ Map<ApplicationId, ByteBuffer> systemCredentials =
+ response.getSystemCredentialsForApps();
+ if (systemCredentials != null && !systemCredentials.isEmpty()) {
+ ((NMContext) context).setSystemCrendentialsForApps(
+ parseCredentials(systemCredentials));
+ }
+ List<org.apache.hadoop.yarn.api.records.Container>
+ containersToDecrease = response.getContainersToDecrease();
+ if (!containersToDecrease.isEmpty()) {
+ dispatcher.getEventHandler().handle(
+ new CMgrDecreaseContainersResourceEvent(
+ containersToDecrease)
+ );
+ }
+
+ // SignalContainer request originally comes from end users via
+ // ClientRMProtocol's SignalContainer. Forward the request to
+ // ContainerManager which will dispatch the event to
+ // ContainerLauncher.
+ List<SignalContainerRequest> containersToSignal = response
+ .getContainersToSignalList();
+ if (!containersToSignal.isEmpty()) {
+ dispatcher.getEventHandler().handle(
+ new CMgrSignalContainersEvent(containersToSignal));
+ }
+
+ // Update QueuingLimits if ContainerManager supports queuing
+ ContainerQueuingLimit queuingLimit =
+ response.getContainerQueuingLimit();
+ if (queuingLimit != null) {
+ context.getContainerManager().updateQueuingLimit(queuingLimit);
+ }
+ }
+ // Handling node resource update case.
+ Resource newResource = response.getResource();
+ if (newResource != null) {
+ updateNMResource(newResource);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Node's resource is updated to " +
+ newResource.toString());
+ }
+ }
+ if (YarnConfiguration.timelineServiceV2Enabled(context.getConf())) {
+ updateTimelineClientsAddress(response);
+ }
+
+ } catch (ConnectException e) {
+ //catch and throw the exception if tried MAX wait time to connect RM
+ dispatcher.getEventHandler().handle(
+ new NodeManagerEvent(NodeManagerEventType.SHUTDOWN));
+ // failed to connect to RM.
+ failedToConnect = true;
+ throw new YarnRuntimeException(e);
+ } catch (Exception e) {
+
+ // TODO Better error handling. Thread can die with the rest of the
+ // NM still running.
+ LOG.error("Caught exception in status-updater", e);
+ } finally {
+ synchronized (heartbeatMonitor) {
+ nextHeartBeatInterval = nextHeartBeatInterval <= 0 ?
+ YarnConfiguration.DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS :
+ nextHeartBeatInterval;
+ try {
+ heartbeatMonitor.wait(nextHeartBeatInterval);
+ } catch (InterruptedException e) {
+ // Do Nothing
+ }
+ }
+ }
+ }
+ }
+
+ private void updateTimelineClientsAddress(
+ NodeHeartbeatResponse response) {
+ Map<ApplicationId, String> knownCollectorsMap =
+ response.getAppCollectorsMap();
+ if (knownCollectorsMap == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("No collectors to update RM");
+ }
+ } else {
+ Set<Map.Entry<ApplicationId, String>> rmKnownCollectors =
+ knownCollectorsMap.entrySet();
+ for (Map.Entry<ApplicationId, String> entry : rmKnownCollectors) {
+ ApplicationId appId = entry.getKey();
+ String collectorAddr = entry.getValue();
+
+ // Only handle applications running on local node.
+ // Not include apps with timeline collectors running in local
+ Application application = context.getApplications().get(appId);
+ // TODO this logic could be problematic if the collector address
+ // gets updated due to NM restart or collector service failure
+ if (application != null &&
+ !context.getRegisteredCollectors().containsKey(appId)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Sync a new collector address: " + collectorAddr +
+ " for application: " + appId + " from RM.");
+ }
+ NMTimelinePublisher nmTimelinePublisher =
+ context.getNMTimelinePublisher();
+ if (nmTimelinePublisher != null) {
+ nmTimelinePublisher.setTimelineServiceAddress(
+ application.getAppId(), collectorAddr);
+ }
+ }
+ }
+ }
+ }
+
+ private void updateMasterKeys(NodeHeartbeatResponse response) {
+ // See if the master-key has rolled over
+ MasterKey updatedMasterKey = response.getContainerTokenMasterKey();
+ if (updatedMasterKey != null) {
+ // Will be non-null only on roll-over on RM side
+ context.getContainerTokenSecretManager().setMasterKey(updatedMasterKey);
+ }
+
+ updatedMasterKey = response.getNMTokenMasterKey();
+ if (updatedMasterKey != null) {
+ context.getNMTokenSecretManager().setMasterKey(updatedMasterKey);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/74a61438/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 1fcccde..0b599a8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -213,12 +213,7 @@ public class ContainerLaunch implements Callable<Integer> {
DataOutputStream tokensOutStream = null;
// Select the working directory for the container
- Path containerWorkDir =
- dirsHandler.getLocalPathForWrite(ContainerLocalizer.USERCACHE
- + Path.SEPARATOR + user + Path.SEPARATOR
- + ContainerLocalizer.APPCACHE + Path.SEPARATOR + appIdStr
- + Path.SEPARATOR + containerIdStr,
- LocalDirAllocator.SIZE_UNKNOWN, false);
+ Path containerWorkDir = deriveContainerWorkDir();
recordContainerWorkDir(containerID, containerWorkDir.toString());
String pidFileSubpath = getPidFileSubpath(appIdStr, containerIdStr);
@@ -259,12 +254,8 @@ public class ContainerLaunch implements Callable<Integer> {
sanitizeEnv(environment, containerWorkDir, appDirs, userLocalDirs,
containerLogDirs, localResources, nmPrivateClasspathJarDir);
- exec.prepareContainer(new ContainerPrepareContext.Builder()
- .setContainer(container)
- .setLocalizedResources(localResources)
- .setUser(user)
- .setContainerLocalDirs(containerLocalDirs)
- .setCommands(launchContext.getCommands()).build());
+ prepareContainer(localResources, containerLocalDirs);
+
// Write out the environment
exec.writeLaunchEnv(containerScriptOutStream, environment,
localResources, launchContext.getCommands(),
@@ -317,6 +308,39 @@ public class ContainerLaunch implements Callable<Integer> {
return ret;
}
+ private Path deriveContainerWorkDir() throws IOException {
+
+ final String containerWorkDirPath =
+ ContainerLocalizer.USERCACHE +
+ Path.SEPARATOR +
+ container.getUser() +
+ Path.SEPARATOR +
+ ContainerLocalizer.APPCACHE +
+ Path.SEPARATOR +
+ app.getAppId().toString() +
+ Path.SEPARATOR +
+ container.getContainerId().toString();
+
+ final Path containerWorkDir =
+ dirsHandler.getLocalPathForWrite(
+ containerWorkDirPath,
+ LocalDirAllocator.SIZE_UNKNOWN, false);
+
+ return containerWorkDir;
+ }
+
+ private void prepareContainer(Map<Path, List<String>> localResources,
+ List<String> containerLocalDirs) throws IOException {
+
+ exec.prepareContainer(new ContainerPrepareContext.Builder()
+ .setContainer(container)
+ .setLocalizedResources(localResources)
+ .setUser(container.getUser())
+ .setContainerLocalDirs(containerLocalDirs)
+ .setCommands(container.getLaunchContext().getCommands())
+ .build());
+ }
+
@SuppressWarnings("unchecked")
protected boolean validateContainerState() {
// CONTAINER_KILLED_ON_REQUEST should not be missed if the container
@@ -1116,98 +1140,9 @@ public class ContainerLaunch implements Callable<Integer> {
// TODO: Remove Windows check and use this approach on all platforms after
// additional testing. See YARN-358.
if (Shell.WINDOWS) {
-
- String inputClassPath = environment.get(Environment.CLASSPATH.name());
-
- if (inputClassPath != null && !inputClassPath.isEmpty()) {
-
- //On non-windows, localized resources
- //from distcache are available via the classpath as they were placed
- //there but on windows they are not available when the classpath
- //jar is created and so they "are lost" and have to be explicitly
- //added to the classpath instead. This also means that their position
- //is lost relative to other non-distcache classpath entries which will
- //break things like mapreduce.job.user.classpath.first. An environment
- //variable can be set to indicate that distcache entries should come
- //first
-
- boolean preferLocalizedJars = Boolean.parseBoolean(
- environment.get(Environment.CLASSPATH_PREPEND_DISTCACHE.name())
- );
-
- boolean needsSeparator = false;
- StringBuilder newClassPath = new StringBuilder();
- if (!preferLocalizedJars) {
- newClassPath.append(inputClassPath);
- needsSeparator = true;
- }
-
- // Localized resources do not exist at the desired paths yet, because the
- // container launch script has not run to create symlinks yet. This
- // means that FileUtil.createJarWithClassPath can't automatically expand
- // wildcards to separate classpath entries for each file in the manifest.
- // To resolve this, append classpath entries explicitly for each
- // resource.
- for (Map.Entry<Path,List<String>> entry : resources.entrySet()) {
- boolean targetIsDirectory = new File(entry.getKey().toUri().getPath())
- .isDirectory();
-
- for (String linkName : entry.getValue()) {
- // Append resource.
- if (needsSeparator) {
- newClassPath.append(File.pathSeparator);
- } else {
- needsSeparator = true;
- }
- newClassPath.append(pwd.toString())
- .append(Path.SEPARATOR).append(linkName);
-
- // FileUtil.createJarWithClassPath must use File.toURI to convert
- // each file to a URI to write into the manifest's classpath. For
- // directories, the classpath must have a trailing '/', but
- // File.toURI only appends the trailing '/' if it is a directory that
- // already exists. To resolve this, add the classpath entries with
- // explicit trailing '/' here for any localized resource that targets
- // a directory. Then, FileUtil.createJarWithClassPath will guarantee
- // that the resulting entry in the manifest's classpath will have a
- // trailing '/', and thus refer to a directory instead of a file.
- if (targetIsDirectory) {
- newClassPath.append(Path.SEPARATOR);
- }
- }
- }
- if (preferLocalizedJars) {
- if (needsSeparator) {
- newClassPath.append(File.pathSeparator);
- }
- newClassPath.append(inputClassPath);
- }
- // When the container launches, it takes the parent process's environment
- // and then adds/overwrites with the entries from the container launch
- // context. Do the same thing here for correct substitution of
- // environment variables in the classpath jar manifest.
- Map<String, String> mergedEnv = new HashMap<String, String>(
- System.getenv());
- mergedEnv.putAll(environment);
-
- // this is hacky and temporary - it's to preserve the windows secure
- // behavior but enable non-secure windows to properly build the class
- // path for access to job.jar/lib/xyz and friends (see YARN-2803)
- Path jarDir;
- if (exec instanceof WindowsSecureContainerExecutor) {
- jarDir = nmPrivateClasspathJarDir;
- } else {
- jarDir = pwd;
- }
- String[] jarCp = FileUtil.createJarWithClassPath(
- newClassPath.toString(), jarDir, pwd, mergedEnv);
- // In a secure cluster the classpath jar must be localized to grant access
- Path localizedClassPathJar = exec.localizeClasspathJar(
- new Path(jarCp[0]), pwd, container.getUser());
- String replacementClassPath = localizedClassPathJar.toString() + jarCp[1];
- environment.put(Environment.CLASSPATH.name(), replacementClassPath);
- }
+ sanitizeWindowsEnv(environment, pwd,
+ resources, nmPrivateClasspathJarDir);
}
// put AuxiliaryService data to environment
for (Map.Entry<String, ByteBuffer> meta : containerManager
@@ -1217,6 +1152,103 @@ public class ContainerLaunch implements Callable<Integer> {
}
}
+ private void sanitizeWindowsEnv(Map<String, String> environment, Path pwd,
+ Map<Path, List<String>> resources, Path nmPrivateClasspathJarDir)
+ throws IOException {
+
+ String inputClassPath = environment.get(Environment.CLASSPATH.name());
+
+ if (inputClassPath != null && !inputClassPath.isEmpty()) {
+
+ //On non-windows, localized resources
+ //from distcache are available via the classpath as they were placed
+ //there but on windows they are not available when the classpath
+ //jar is created and so they "are lost" and have to be explicitly
+ //added to the classpath instead. This also means that their position
+ //is lost relative to other non-distcache classpath entries which will
+ //break things like mapreduce.job.user.classpath.first. An environment
+ //variable can be set to indicate that distcache entries should come
+ //first
+
+ boolean preferLocalizedJars = Boolean.parseBoolean(
+ environment.get(Environment.CLASSPATH_PREPEND_DISTCACHE.name())
+ );
+
+ boolean needsSeparator = false;
+ StringBuilder newClassPath = new StringBuilder();
+ if (!preferLocalizedJars) {
+ newClassPath.append(inputClassPath);
+ needsSeparator = true;
+ }
+
+ // Localized resources do not exist at the desired paths yet, because the
+ // container launch script has not run to create symlinks yet. This
+ // means that FileUtil.createJarWithClassPath can't automatically expand
+ // wildcards to separate classpath entries for each file in the manifest.
+ // To resolve this, append classpath entries explicitly for each
+ // resource.
+ for (Map.Entry<Path, List<String>> entry : resources.entrySet()) {
+ boolean targetIsDirectory = new File(entry.getKey().toUri().getPath())
+ .isDirectory();
+
+ for (String linkName : entry.getValue()) {
+ // Append resource.
+ if (needsSeparator) {
+ newClassPath.append(File.pathSeparator);
+ } else {
+ needsSeparator = true;
+ }
+ newClassPath.append(pwd.toString())
+ .append(Path.SEPARATOR).append(linkName);
+
+ // FileUtil.createJarWithClassPath must use File.toURI to convert
+ // each file to a URI to write into the manifest's classpath. For
+ // directories, the classpath must have a trailing '/', but
+ // File.toURI only appends the trailing '/' if it is a directory that
+ // already exists. To resolve this, add the classpath entries with
+ // explicit trailing '/' here for any localized resource that targets
+ // a directory. Then, FileUtil.createJarWithClassPath will guarantee
+ // that the resulting entry in the manifest's classpath will have a
+ // trailing '/', and thus refer to a directory instead of a file.
+ if (targetIsDirectory) {
+ newClassPath.append(Path.SEPARATOR);
+ }
+ }
+ }
+ if (preferLocalizedJars) {
+ if (needsSeparator) {
+ newClassPath.append(File.pathSeparator);
+ }
+ newClassPath.append(inputClassPath);
+ }
+
+ // When the container launches, it takes the parent process's environment
+ // and then adds/overwrites with the entries from the container launch
+ // context. Do the same thing here for correct substitution of
+ // environment variables in the classpath jar manifest.
+ Map<String, String> mergedEnv = new HashMap<String, String>(
+ System.getenv());
+ mergedEnv.putAll(environment);
+
+ // this is hacky and temporary - it's to preserve the windows secure
+ // behavior but enable non-secure windows to properly build the class
+ // path for access to job.jar/lib/xyz and friends (see YARN-2803)
+ Path jarDir;
+ if (exec instanceof WindowsSecureContainerExecutor) {
+ jarDir = nmPrivateClasspathJarDir;
+ } else {
+ jarDir = pwd;
+ }
+ String[] jarCp = FileUtil.createJarWithClassPath(
+ newClassPath.toString(), jarDir, pwd, mergedEnv);
+ // In a secure cluster the classpath jar must be localized to grant access
+ Path localizedClassPathJar = exec.localizeClasspathJar(
+ new Path(jarCp[0]), pwd, container.getUser());
+ String replacementClassPath = localizedClassPathJar.toString() + jarCp[1];
+ environment.put(Environment.CLASSPATH.name(), replacementClassPath);
+ }
+ }
+
public static String getExitCodeFile(String pidFile) {
return pidFile + EXIT_CODE_FILE_SUFFIX;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/74a61438/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index b70a4e1..ed81331 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -424,10 +424,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
//List<String> -> stored as List -> fetched/converted to List<String>
//we can't do better here thanks to type-erasure
@SuppressWarnings("unchecked")
- List<String> localDirs = ctx.getExecutionAttribute(LOCAL_DIRS);
- @SuppressWarnings("unchecked")
- List<String> logDirs = ctx.getExecutionAttribute(LOG_DIRS);
- @SuppressWarnings("unchecked")
List<String> filecacheDirs = ctx.getExecutionAttribute(FILECACHE_DIRS);
@SuppressWarnings("unchecked")
List<String> containerLocalDirs = ctx.getExecutionAttribute(
@@ -489,9 +485,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
addCGroupParentIfRequired(resourcesOpts, containerIdStr, runCommand);
- Path nmPrivateContainerScriptPath = ctx.getExecutionAttribute(
- NM_PRIVATE_CONTAINER_SCRIPT_PATH);
-
String disableOverride = environment.get(
ENV_DOCKER_CONTAINER_RUN_OVERRIDE_DISABLE);
@@ -511,33 +504,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
String commandFile = dockerClient.writeCommandToTempFile(runCommand,
containerIdStr);
- PrivilegedOperation launchOp = new PrivilegedOperation(
- PrivilegedOperation.OperationType.LAUNCH_DOCKER_CONTAINER);
-
- launchOp.appendArgs(runAsUser, ctx.getExecutionAttribute(USER),
- Integer.toString(PrivilegedOperation
- .RunAsUserCommand.LAUNCH_DOCKER_CONTAINER.getValue()),
- ctx.getExecutionAttribute(APPID),
- containerIdStr, containerWorkDir.toString(),
- nmPrivateContainerScriptPath.toUri().getPath(),
- ctx.getExecutionAttribute(NM_PRIVATE_TOKENS_PATH).toUri().getPath(),
- ctx.getExecutionAttribute(PID_FILE_PATH).toString(),
- StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
- localDirs),
- StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
- logDirs),
- commandFile,
- resourcesOpts);
-
- String tcCommandFile = ctx.getExecutionAttribute(TC_COMMAND_FILE);
-
- if (tcCommandFile != null) {
- launchOp.appendArgs(tcCommandFile);
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Launching container with cmd: " + runCommand
- .getCommandWithArguments());
- }
+ PrivilegedOperation launchOp = buildLaunchOp(ctx,
+ commandFile, runCommand);
try {
privilegedOperationExecutor.executePrivilegedOperation(null,
@@ -635,4 +603,53 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
}
return null;
}
+
+
+
+ private PrivilegedOperation buildLaunchOp(ContainerRuntimeContext ctx,
+ String commandFile, DockerRunCommand runCommand) {
+
+ String runAsUser = ctx.getExecutionAttribute(RUN_AS_USER);
+ String containerIdStr = ctx.getContainer().getContainerId().toString();
+ Path nmPrivateContainerScriptPath = ctx.getExecutionAttribute(
+ NM_PRIVATE_CONTAINER_SCRIPT_PATH);
+ Path containerWorkDir = ctx.getExecutionAttribute(CONTAINER_WORK_DIR);
+ //we can't do better here thanks to type-erasure
+ @SuppressWarnings("unchecked")
+ List<String> localDirs = ctx.getExecutionAttribute(LOCAL_DIRS);
+ @SuppressWarnings("unchecked")
+ List<String> logDirs = ctx.getExecutionAttribute(LOG_DIRS);
+ String resourcesOpts = ctx.getExecutionAttribute(RESOURCES_OPTIONS);
+
+ PrivilegedOperation launchOp = new PrivilegedOperation(
+ PrivilegedOperation.OperationType.LAUNCH_DOCKER_CONTAINER);
+
+ launchOp.appendArgs(runAsUser, ctx.getExecutionAttribute(USER),
+ Integer.toString(PrivilegedOperation
+ .RunAsUserCommand.LAUNCH_DOCKER_CONTAINER.getValue()),
+ ctx.getExecutionAttribute(APPID),
+ containerIdStr,
+ containerWorkDir.toString(),
+ nmPrivateContainerScriptPath.toUri().getPath(),
+ ctx.getExecutionAttribute(NM_PRIVATE_TOKENS_PATH).toUri().getPath(),
+ ctx.getExecutionAttribute(PID_FILE_PATH).toString(),
+ StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
+ localDirs),
+ StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
+ logDirs),
+ commandFile,
+ resourcesOpts);
+
+ String tcCommandFile = ctx.getExecutionAttribute(TC_COMMAND_FILE);
+
+ if (tcCommandFile != null) {
+ launchOp.appendArgs(tcCommandFile);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Launching container with cmd: " + runCommand
+ .getCommandWithArguments());
+ }
+
+ return launchOp;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: HDFS-9807. Add an optional
StorageID to writes. Contributed by Ewan Higgs
Posted by in...@apache.org.
HDFS-9807. Add an optional StorageID to writes. Contributed by Ewan Higgs
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a3954cca
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a3954cca
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a3954cca
Branch: refs/heads/HDFS-10467
Commit: a3954ccab148bddc290cb96528e63ff19799bcc9
Parents: 4e6bbd0
Author: Chris Douglas <cd...@apache.org>
Authored: Fri May 5 12:01:26 2017 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Fri May 5 12:01:26 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DataStreamer.java | 35 +-
.../apache/hadoop/hdfs/StripedDataStreamer.java | 10 +-
.../datatransfer/DataTransferProtocol.java | 19 +-
.../hdfs/protocol/datatransfer/Sender.java | 29 +-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 13 +
.../token/block/BlockTokenIdentifier.java | 36 +-
.../src/main/proto/datatransfer.proto | 4 +
.../src/main/proto/hdfs.proto | 1 +
.../hdfs/protocol/datatransfer/Receiver.java | 20 +-
.../block/BlockPoolTokenSecretManager.java | 22 +-
.../token/block/BlockTokenSecretManager.java | 55 ++--
.../hadoop/hdfs/server/balancer/Dispatcher.java | 5 +-
.../hadoop/hdfs/server/balancer/KeyManager.java | 4 +-
.../server/blockmanagement/BlockManager.java | 6 +-
.../hdfs/server/datanode/BPOfferService.java | 3 +-
.../hdfs/server/datanode/BlockReceiver.java | 12 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 42 ++-
.../hdfs/server/datanode/DataXceiver.java | 68 ++--
.../erasurecode/ErasureCodingWorker.java | 3 +-
.../erasurecode/StripedBlockReader.java | 2 +-
.../erasurecode/StripedBlockWriter.java | 10 +-
.../erasurecode/StripedReconstructionInfo.java | 16 +-
.../datanode/erasurecode/StripedWriter.java | 5 +-
.../AvailableSpaceVolumeChoosingPolicy.java | 20 +-
.../server/datanode/fsdataset/FsDatasetSpi.java | 6 +-
.../RoundRobinVolumeChoosingPolicy.java | 2 +-
.../fsdataset/VolumeChoosingPolicy.java | 5 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 21 +-
.../datanode/fsdataset/impl/FsVolumeList.java | 19 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 3 +-
.../hadoop/hdfs/TestBlockStoragePolicy.java | 75 ++++-
.../hadoop/hdfs/TestDataTransferProtocol.java | 3 +-
.../hdfs/TestWriteBlockGetsBlockLengthHint.java | 6 +-
.../security/token/block/TestBlockToken.java | 98 +++---
.../server/datanode/BlockReportTestBase.java | 2 +-
.../server/datanode/SimulatedFSDataset.java | 19 +-
.../hdfs/server/datanode/TestBlockRecovery.java | 6 +-
.../server/datanode/TestBlockReplacement.java | 2 +-
.../TestDataXceiverLazyPersistHint.java | 4 +-
.../hdfs/server/datanode/TestDiskError.java | 5 +-
.../server/datanode/TestSimulatedFSDataset.java | 4 +-
.../extdataset/ExternalDatasetImpl.java | 10 +-
.../TestAvailableSpaceVolumeChoosingPolicy.java | 76 +++--
.../TestRoundRobinVolumeChoosingPolicy.java | 29 +-
.../fsdataset/impl/TestFsDatasetImpl.java | 4 +-
.../fsdataset/impl/TestFsVolumeList.java | 2 +-
.../fsdataset/impl/TestWriteToReplica.java | 29 +-
.../namenode/TestNamenodeStorageDirectives.java | 330 +++++++++++++++++++
48 files changed, 903 insertions(+), 297 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
index 0268537..49c17b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
@@ -174,10 +174,12 @@ class DataStreamer extends Daemon {
void sendTransferBlock(final DatanodeInfo[] targets,
final StorageType[] targetStorageTypes,
+ final String[] targetStorageIDs,
final Token<BlockTokenIdentifier> blockToken) throws IOException {
//send the TRANSFER_BLOCK request
new Sender(out).transferBlock(block.getCurrentBlock(), blockToken,
- dfsClient.clientName, targets, targetStorageTypes);
+ dfsClient.clientName, targets, targetStorageTypes,
+ targetStorageIDs);
out.flush();
//ack
BlockOpResponseProto transferResponse = BlockOpResponseProto
@@ -1367,9 +1369,11 @@ class DataStreamer extends Daemon {
final DatanodeInfo src = original[tried % original.length];
final DatanodeInfo[] targets = {nodes[d]};
final StorageType[] targetStorageTypes = {storageTypes[d]};
+ final String[] targetStorageIDs = {storageIDs[d]};
try {
- transfer(src, targets, targetStorageTypes, lb.getBlockToken());
+ transfer(src, targets, targetStorageTypes, targetStorageIDs,
+ lb.getBlockToken());
} catch (IOException ioe) {
DFSClient.LOG.warn("Error transferring data from " + src + " to " +
nodes[d] + ": " + ioe.getMessage());
@@ -1400,6 +1404,7 @@ class DataStreamer extends Daemon {
private void transfer(final DatanodeInfo src, final DatanodeInfo[] targets,
final StorageType[] targetStorageTypes,
+ final String[] targetStorageIDs,
final Token<BlockTokenIdentifier> blockToken)
throws IOException {
//transfer replica to the new datanode
@@ -1412,7 +1417,8 @@ class DataStreamer extends Daemon {
streams = new StreamerStreams(src, writeTimeout, readTimeout,
blockToken);
- streams.sendTransferBlock(targets, targetStorageTypes, blockToken);
+ streams.sendTransferBlock(targets, targetStorageTypes,
+ targetStorageIDs, blockToken);
return;
} catch (InvalidEncryptionKeyException e) {
policy.recordFailure(e);
@@ -1440,11 +1446,12 @@ class DataStreamer extends Daemon {
streamerClosed = true;
return;
}
- setupPipelineInternal(nodes, storageTypes);
+ setupPipelineInternal(nodes, storageTypes, storageIDs);
}
protected void setupPipelineInternal(DatanodeInfo[] datanodes,
- StorageType[] nodeStorageTypes) throws IOException {
+ StorageType[] nodeStorageTypes, String[] nodeStorageIDs)
+ throws IOException {
boolean success = false;
long newGS = 0L;
while (!success && !streamerClosed && dfsClient.clientRunning) {
@@ -1465,7 +1472,8 @@ class DataStreamer extends Daemon {
accessToken = lb.getBlockToken();
// set up the pipeline again with the remaining nodes
- success = createBlockOutputStream(nodes, storageTypes, newGS, isRecovery);
+ success = createBlockOutputStream(nodes, storageTypes, storageIDs, newGS,
+ isRecovery);
failPacket4Testing();
@@ -1601,7 +1609,8 @@ class DataStreamer extends Daemon {
protected LocatedBlock nextBlockOutputStream() throws IOException {
LocatedBlock lb;
DatanodeInfo[] nodes;
- StorageType[] storageTypes;
+ StorageType[] nextStorageTypes;
+ String[] nextStorageIDs;
int count = dfsClient.getConf().getNumBlockWriteRetry();
boolean success;
final ExtendedBlock oldBlock = block.getCurrentBlock();
@@ -1617,10 +1626,12 @@ class DataStreamer extends Daemon {
bytesSent = 0;
accessToken = lb.getBlockToken();
nodes = lb.getLocations();
- storageTypes = lb.getStorageTypes();
+ nextStorageTypes = lb.getStorageTypes();
+ nextStorageIDs = lb.getStorageIDs();
// Connect to first DataNode in the list.
- success = createBlockOutputStream(nodes, storageTypes, 0L, false);
+ success = createBlockOutputStream(nodes, nextStorageTypes, nextStorageIDs,
+ 0L, false);
if (!success) {
LOG.warn("Abandoning " + block);
@@ -1643,7 +1654,8 @@ class DataStreamer extends Daemon {
// Returns true if success, otherwise return failure.
//
boolean createBlockOutputStream(DatanodeInfo[] nodes,
- StorageType[] nodeStorageTypes, long newGS, boolean recoveryFlag) {
+ StorageType[] nodeStorageTypes, String[] nodeStorageIDs,
+ long newGS, boolean recoveryFlag) {
if (nodes.length == 0) {
LOG.info("nodes are empty for write pipeline of " + block);
return false;
@@ -1696,7 +1708,8 @@ class DataStreamer extends Daemon {
dfsClient.clientName, nodes, nodeStorageTypes, null, bcs,
nodes.length, block.getNumBytes(), bytesSent, newGS,
checksum4WriteBlock, cachingStrategy.get(), isLazyPersistFile,
- (targetPinnings != null && targetPinnings[0]), targetPinnings);
+ (targetPinnings != null && targetPinnings[0]), targetPinnings,
+ nodeStorageIDs[0], nodeStorageIDs);
// receive ack for connect
BlockOpResponseProto resp = BlockOpResponseProto.parseFrom(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripedDataStreamer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripedDataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripedDataStreamer.java
index b457edb..d920f18 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripedDataStreamer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripedDataStreamer.java
@@ -100,9 +100,11 @@ public class StripedDataStreamer extends DataStreamer {
DatanodeInfo[] nodes = lb.getLocations();
StorageType[] storageTypes = lb.getStorageTypes();
+ String[] storageIDs = lb.getStorageIDs();
// Connect to the DataNode. If fail the internal error state will be set.
- success = createBlockOutputStream(nodes, storageTypes, 0L, false);
+ success = createBlockOutputStream(nodes, storageTypes, storageIDs, 0L,
+ false);
if (!success) {
block.setCurrentBlock(null);
@@ -121,7 +123,8 @@ public class StripedDataStreamer extends DataStreamer {
@Override
protected void setupPipelineInternal(DatanodeInfo[] nodes,
- StorageType[] nodeStorageTypes) throws IOException {
+ StorageType[] nodeStorageTypes, String[] nodeStorageIDs)
+ throws IOException {
boolean success = false;
while (!success && !streamerClosed() && dfsClient.clientRunning) {
if (!handleRestartingDatanode()) {
@@ -141,7 +144,8 @@ public class StripedDataStreamer extends DataStreamer {
// set up the pipeline again with the remaining nodes. when a striped
// data streamer comes here, it must be in external error state.
assert getErrorState().hasExternalError();
- success = createBlockOutputStream(nodes, nodeStorageTypes, newGS, true);
+ success = createBlockOutputStream(nodes, nodeStorageTypes,
+ nodeStorageIDs, newGS, true);
failPacket4Testing();
getErrorState().checkRestartingNodeDeadline(nodes);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtocol.java
index 6c5883c..fe20c37 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtocol.java
@@ -101,6 +101,11 @@ public interface DataTransferProtocol {
* written to disk lazily
* @param pinning whether to pin the block, so Balancer won't move it.
* @param targetPinnings whether to pin the block on target datanode
+ * @param storageID optional StorageIDs designating where to write the
+ * block. An empty String or null indicates that this
+ * has not been provided.
+ * @param targetStorageIDs target StorageIDs corresponding to the target
+ * datanodes.
*/
void writeBlock(final ExtendedBlock blk,
final StorageType storageType,
@@ -118,7 +123,9 @@ public interface DataTransferProtocol {
final CachingStrategy cachingStrategy,
final boolean allowLazyPersist,
final boolean pinning,
- final boolean[] targetPinnings) throws IOException;
+ final boolean[] targetPinnings,
+ final String storageID,
+ final String[] targetStorageIDs) throws IOException;
/**
* Transfer a block to another datanode.
* The block stage must be
@@ -129,12 +136,15 @@ public interface DataTransferProtocol {
* @param blockToken security token for accessing the block.
* @param clientName client's name.
* @param targets target datanodes.
+ * @param targetStorageIDs StorageID designating where to write the
+ * block.
*/
void transferBlock(final ExtendedBlock blk,
final Token<BlockTokenIdentifier> blockToken,
final String clientName,
final DatanodeInfo[] targets,
- final StorageType[] targetStorageTypes) throws IOException;
+ final StorageType[] targetStorageTypes,
+ final String[] targetStorageIDs) throws IOException;
/**
* Request short circuit access file descriptors from a DataNode.
@@ -179,12 +189,15 @@ public interface DataTransferProtocol {
* @param blockToken security token for accessing the block.
* @param delHint the hint for deleting the block in the original datanode.
* @param source the source datanode for receiving the block.
+ * @param storageId an optional storage ID to designate where the block is
+ * replaced to.
*/
void replaceBlock(final ExtendedBlock blk,
final StorageType storageType,
final Token<BlockTokenIdentifier> blockToken,
final String delHint,
- final DatanodeInfo source) throws IOException;
+ final DatanodeInfo source,
+ final String storageId) throws IOException;
/**
* Copy a block.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java
index e133975..8a8d20d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java
@@ -22,6 +22,7 @@ import static org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
+import java.util.Arrays;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -132,7 +133,9 @@ public class Sender implements DataTransferProtocol {
final CachingStrategy cachingStrategy,
final boolean allowLazyPersist,
final boolean pinning,
- final boolean[] targetPinnings) throws IOException {
+ final boolean[] targetPinnings,
+ final String storageId,
+ final String[] targetStorageIds) throws IOException {
ClientOperationHeaderProto header = DataTransferProtoUtil.buildClientHeader(
blk, clientName, blockToken);
@@ -154,11 +157,14 @@ public class Sender implements DataTransferProtocol {
.setCachingStrategy(getCachingStrategy(cachingStrategy))
.setAllowLazyPersist(allowLazyPersist)
.setPinning(pinning)
- .addAllTargetPinnings(PBHelperClient.convert(targetPinnings, 1));
-
+ .addAllTargetPinnings(PBHelperClient.convert(targetPinnings, 1))
+ .addAllTargetStorageIds(PBHelperClient.convert(targetStorageIds, 1));
if (source != null) {
proto.setSource(PBHelperClient.convertDatanodeInfo(source));
}
+ if (storageId != null) {
+ proto.setStorageId(storageId);
+ }
send(out, Op.WRITE_BLOCK, proto.build());
}
@@ -168,7 +174,8 @@ public class Sender implements DataTransferProtocol {
final Token<BlockTokenIdentifier> blockToken,
final String clientName,
final DatanodeInfo[] targets,
- final StorageType[] targetStorageTypes) throws IOException {
+ final StorageType[] targetStorageTypes,
+ final String[] targetStorageIds) throws IOException {
OpTransferBlockProto proto = OpTransferBlockProto.newBuilder()
.setHeader(DataTransferProtoUtil.buildClientHeader(
@@ -176,6 +183,7 @@ public class Sender implements DataTransferProtocol {
.addAllTargets(PBHelperClient.convert(targets))
.addAllTargetStorageTypes(
PBHelperClient.convertStorageTypes(targetStorageTypes))
+ .addAllTargetStorageIds(Arrays.asList(targetStorageIds))
.build();
send(out, Op.TRANSFER_BLOCK, proto);
@@ -233,15 +241,18 @@ public class Sender implements DataTransferProtocol {
final StorageType storageType,
final Token<BlockTokenIdentifier> blockToken,
final String delHint,
- final DatanodeInfo source) throws IOException {
- OpReplaceBlockProto proto = OpReplaceBlockProto.newBuilder()
+ final DatanodeInfo source,
+ final String storageId) throws IOException {
+ OpReplaceBlockProto.Builder proto = OpReplaceBlockProto.newBuilder()
.setHeader(DataTransferProtoUtil.buildBaseHeader(blk, blockToken))
.setStorageType(PBHelperClient.convertStorageType(storageType))
.setDelHint(delHint)
- .setSource(PBHelperClient.convertDatanodeInfo(source))
- .build();
+ .setSource(PBHelperClient.convertDatanodeInfo(source));
+ if (storageId != null) {
+ proto.setStorageId(storageId);
+ }
- send(out, Op.REPLACE_BLOCK, proto);
+ send(out, Op.REPLACE_BLOCK, proto.build());
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index 2b8f102..614f653 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -345,6 +345,16 @@ public class PBHelperClient {
return pinnings;
}
+ public static List<String> convert(String[] targetIds, int idx) {
+ List<String> ids = new ArrayList<>();
+ if (targetIds != null) {
+ for (; idx < targetIds.length; ++idx) {
+ ids.add(targetIds[idx]);
+ }
+ }
+ return ids;
+ }
+
public static ExtendedBlock convert(ExtendedBlockProto eb) {
if (eb == null) return null;
return new ExtendedBlock( eb.getPoolId(), eb.getBlockId(), eb.getNumBytes(),
@@ -640,6 +650,9 @@ public class PBHelperClient {
for (StorageType storageType : blockTokenSecret.getStorageTypes()) {
builder.addStorageTypes(convertStorageType(storageType));
}
+ for (String storageId : blockTokenSecret.getStorageIds()) {
+ builder.addStorageIds(storageId);
+ }
return builder.build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
index 228a7b6..5950752 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
@@ -53,16 +53,19 @@ public class BlockTokenIdentifier extends TokenIdentifier {
private long blockId;
private final EnumSet<AccessMode> modes;
private StorageType[] storageTypes;
+ private String[] storageIds;
private boolean useProto;
private byte [] cache;
public BlockTokenIdentifier() {
- this(null, null, 0, EnumSet.noneOf(AccessMode.class), null, false);
+ this(null, null, 0, EnumSet.noneOf(AccessMode.class), null, null,
+ false);
}
public BlockTokenIdentifier(String userId, String bpid, long blockId,
- EnumSet<AccessMode> modes, StorageType[] storageTypes, boolean useProto) {
+ EnumSet<AccessMode> modes, StorageType[] storageTypes,
+ String[] storageIds, boolean useProto) {
this.cache = null;
this.userId = userId;
this.blockPoolId = bpid;
@@ -70,6 +73,8 @@ public class BlockTokenIdentifier extends TokenIdentifier {
this.modes = modes == null ? EnumSet.noneOf(AccessMode.class) : modes;
this.storageTypes = Optional.ofNullable(storageTypes)
.orElse(StorageType.EMPTY_ARRAY);
+ this.storageIds = Optional.ofNullable(storageIds)
+ .orElse(new String[0]);
this.useProto = useProto;
}
@@ -125,6 +130,10 @@ public class BlockTokenIdentifier extends TokenIdentifier {
return storageTypes;
}
+ public String[] getStorageIds(){
+ return storageIds;
+ }
+
@Override
public String toString() {
return "block_token_identifier (expiryDate=" + this.getExpiryDate()
@@ -132,7 +141,8 @@ public class BlockTokenIdentifier extends TokenIdentifier {
+ ", blockPoolId=" + this.getBlockPoolId()
+ ", blockId=" + this.getBlockId() + ", access modes="
+ this.getAccessModes() + ", storageTypes= "
- + Arrays.toString(this.getStorageTypes()) + ")";
+ + Arrays.toString(this.getStorageTypes()) + ", storageIds= "
+ + Arrays.toString(this.getStorageIds()) + ")";
}
static boolean isEqual(Object a, Object b) {
@@ -151,7 +161,8 @@ public class BlockTokenIdentifier extends TokenIdentifier {
&& isEqual(this.blockPoolId, that.blockPoolId)
&& this.blockId == that.blockId
&& isEqual(this.modes, that.modes)
- && Arrays.equals(this.storageTypes, that.storageTypes);
+ && Arrays.equals(this.storageTypes, that.storageTypes)
+ && Arrays.equals(this.storageIds, that.storageIds);
}
return false;
}
@@ -161,7 +172,8 @@ public class BlockTokenIdentifier extends TokenIdentifier {
return (int) expiryDate ^ keyId ^ (int) blockId ^ modes.hashCode()
^ (userId == null ? 0 : userId.hashCode())
^ (blockPoolId == null ? 0 : blockPoolId.hashCode())
- ^ (storageTypes == null ? 0 : Arrays.hashCode(storageTypes));
+ ^ (storageTypes == null ? 0 : Arrays.hashCode(storageTypes))
+ ^ (storageIds == null ? 0 : Arrays.hashCode(storageIds));
}
/**
@@ -220,6 +232,14 @@ public class BlockTokenIdentifier extends TokenIdentifier {
readStorageTypes[i] = WritableUtils.readEnum(in, StorageType.class);
}
storageTypes = readStorageTypes;
+
+ length = WritableUtils.readVInt(in);
+ String[] readStorageIds = new String[length];
+ for (int i = 0; i < length; i++) {
+ readStorageIds[i] = WritableUtils.readString(in);
+ }
+ storageIds = readStorageIds;
+
useProto = false;
}
@@ -248,6 +268,8 @@ public class BlockTokenIdentifier extends TokenIdentifier {
storageTypes = blockTokenSecretProto.getStorageTypesList().stream()
.map(PBHelperClient::convertStorageType)
.toArray(StorageType[]::new);
+ storageIds = blockTokenSecretProto.getStorageIdsList().stream()
+ .toArray(String[]::new);
useProto = true;
}
@@ -275,6 +297,10 @@ public class BlockTokenIdentifier extends TokenIdentifier {
for (StorageType type: storageTypes){
WritableUtils.writeEnum(out, type);
}
+ WritableUtils.writeVInt(out, storageIds.length);
+ for (String id: storageIds) {
+ WritableUtils.writeString(out, id);
+ }
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/datatransfer.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/datatransfer.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/datatransfer.proto
index 889361a..2356201 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/datatransfer.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/datatransfer.proto
@@ -125,12 +125,15 @@ message OpWriteBlockProto {
//whether to pin the block, so Balancer won't move it.
optional bool pinning = 14 [default = false];
repeated bool targetPinnings = 15;
+ optional string storageId = 16;
+ repeated string targetStorageIds = 17;
}
message OpTransferBlockProto {
required ClientOperationHeaderProto header = 1;
repeated DatanodeInfoProto targets = 2;
repeated StorageTypeProto targetStorageTypes = 3;
+ repeated string targetStorageIds = 4;
}
message OpReplaceBlockProto {
@@ -138,6 +141,7 @@ message OpReplaceBlockProto {
required string delHint = 2;
required DatanodeInfoProto source = 3;
optional StorageTypeProto storageType = 4 [default = DISK];
+ optional string storageId = 5;
}
message OpCopyBlockProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index 3e27427..08ed3c8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -570,4 +570,5 @@ message BlockTokenSecretProto {
optional uint64 blockId = 5;
repeated AccessModeProto modes = 6;
repeated StorageTypeProto storageTypes = 7;
+ repeated string storageIds = 8;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
index 08ab967..bab2e8d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
@@ -25,7 +25,9 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.StripedBlockInfo;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BaseHeaderProto;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.CachingStrategyProto;
@@ -185,7 +187,9 @@ public abstract class Receiver implements DataTransferProtocol {
CachingStrategy.newDefaultStrategy()),
(proto.hasAllowLazyPersist() ? proto.getAllowLazyPersist() : false),
(proto.hasPinning() ? proto.getPinning(): false),
- (PBHelperClient.convertBooleanList(proto.getTargetPinningsList())));
+ (PBHelperClient.convertBooleanList(proto.getTargetPinningsList())),
+ proto.getStorageId(),
+ proto.getTargetStorageIdsList().toArray(new String[0]));
} finally {
if (traceScope != null) traceScope.close();
}
@@ -199,11 +203,18 @@ public abstract class Receiver implements DataTransferProtocol {
TraceScope traceScope = continueTraceSpan(proto.getHeader(),
proto.getClass().getSimpleName());
try {
- transferBlock(PBHelperClient.convert(proto.getHeader().getBaseHeader().getBlock()),
+ final ExtendedBlock block =
+ PBHelperClient.convert(proto.getHeader().getBaseHeader().getBlock());
+ final StorageType[] targetStorageTypes =
+ PBHelperClient.convertStorageTypes(proto.getTargetStorageTypesList(),
+ targets.length);
+ transferBlock(block,
PBHelperClient.convert(proto.getHeader().getBaseHeader().getToken()),
proto.getHeader().getClientName(),
targets,
- PBHelperClient.convertStorageTypes(proto.getTargetStorageTypesList(), targets.length));
+ targetStorageTypes,
+ proto.getTargetStorageIdsList().toArray(new String[0])
+ );
} finally {
if (traceScope != null) traceScope.close();
}
@@ -264,7 +275,8 @@ public abstract class Receiver implements DataTransferProtocol {
PBHelperClient.convertStorageType(proto.getStorageType()),
PBHelperClient.convert(proto.getHeader().getToken()),
proto.getDelHint(),
- PBHelperClient.convert(proto.getSource()));
+ PBHelperClient.convert(proto.getSource()),
+ proto.getStorageId());
} finally {
if (traceScope != null) traceScope.close();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockPoolTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockPoolTokenSecretManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockPoolTokenSecretManager.java
index 29fb73f..8400b4f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockPoolTokenSecretManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockPoolTokenSecretManager.java
@@ -84,25 +84,27 @@ public class BlockPoolTokenSecretManager extends
/**
* See {@link BlockTokenSecretManager#checkAccess(BlockTokenIdentifier,
* String, ExtendedBlock, BlockTokenIdentifier.AccessMode,
- * StorageType[])}
+ * StorageType[], String[])}
*/
public void checkAccess(BlockTokenIdentifier id, String userId,
ExtendedBlock block, AccessMode mode,
- StorageType[] storageTypes) throws InvalidToken {
+ StorageType[] storageTypes, String[] storageIds)
+ throws InvalidToken {
get(block.getBlockPoolId()).checkAccess(id, userId, block, mode,
- storageTypes);
+ storageTypes, storageIds);
}
/**
* See {@link BlockTokenSecretManager#checkAccess(Token, String,
* ExtendedBlock, BlockTokenIdentifier.AccessMode,
- * StorageType[])}.
+ * StorageType[], String[])}
*/
public void checkAccess(Token<BlockTokenIdentifier> token,
String userId, ExtendedBlock block, AccessMode mode,
- StorageType[] storageTypes) throws InvalidToken {
+ StorageType[] storageTypes, String[] storageIds)
+ throws InvalidToken {
get(block.getBlockPoolId()).checkAccess(token, userId, block, mode,
- storageTypes);
+ storageTypes, storageIds);
}
/**
@@ -115,11 +117,13 @@ public class BlockPoolTokenSecretManager extends
/**
* See {@link BlockTokenSecretManager#generateToken(ExtendedBlock, EnumSet,
- * StorageType[])}
+ * StorageType[], String[])}.
*/
public Token<BlockTokenIdentifier> generateToken(ExtendedBlock b,
- EnumSet<AccessMode> of, StorageType[] storageTypes) throws IOException {
- return get(b.getBlockPoolId()).generateToken(b, of, storageTypes);
+ EnumSet<AccessMode> of, StorageType[] storageTypes, String[] storageIds)
+ throws IOException {
+ return get(b.getBlockPoolId()).generateToken(b, of, storageTypes,
+ storageIds);
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
index f3bec83..6b54490 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
@@ -247,18 +247,19 @@ public class BlockTokenSecretManager extends
/** Generate an block token for current user */
public Token<BlockTokenIdentifier> generateToken(ExtendedBlock block,
EnumSet<BlockTokenIdentifier.AccessMode> modes,
- StorageType[] storageTypes) throws IOException {
+ StorageType[] storageTypes, String[] storageIds) throws IOException {
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
String userID = (ugi == null ? null : ugi.getShortUserName());
- return generateToken(userID, block, modes, storageTypes);
+ return generateToken(userID, block, modes, storageTypes, storageIds);
}
/** Generate a block token for a specified user */
public Token<BlockTokenIdentifier> generateToken(String userId,
ExtendedBlock block, EnumSet<BlockTokenIdentifier.AccessMode> modes,
- StorageType[] storageTypes) throws IOException {
+ StorageType[] storageTypes, String[] storageIds) throws IOException {
BlockTokenIdentifier id = new BlockTokenIdentifier(userId, block
- .getBlockPoolId(), block.getBlockId(), modes, storageTypes, useProto);
+ .getBlockPoolId(), block.getBlockId(), modes, storageTypes,
+ storageIds, useProto);
return new Token<BlockTokenIdentifier>(id, this);
}
@@ -272,10 +273,13 @@ public class BlockTokenSecretManager extends
*/
public void checkAccess(BlockTokenIdentifier id, String userId,
ExtendedBlock block, BlockTokenIdentifier.AccessMode mode,
- StorageType[] storageTypes) throws InvalidToken {
+ StorageType[] storageTypes, String[] storageIds) throws InvalidToken {
checkAccess(id, userId, block, mode);
if (storageTypes != null && storageTypes.length > 0) {
- checkAccess(id.getStorageTypes(), storageTypes);
+ checkAccess(id.getStorageTypes(), storageTypes, "StorageTypes");
+ }
+ if (storageIds != null && storageIds.length > 0) {
+ checkAccess(id.getStorageIds(), storageIds, "StorageIDs");
}
}
@@ -309,30 +313,31 @@ public class BlockTokenSecretManager extends
}
/**
- * Check if the requested StorageTypes match the StorageTypes in the
- * BlockTokenIdentifier.
- * Empty candidateStorageTypes specifiers mean 'all is permitted'. They
- * would otherwise be nonsensical.
+ * Check if the requested values can be satisfied with the values in the
+ * BlockToken. This is intended for use with StorageTypes and StorageIDs.
+ *
+ * The current node can only verify that one of the storage [Type|ID] is
+ * available. The rest will be on different nodes.
*/
- public static void checkAccess(StorageType[] candidateStorageTypes,
- StorageType[] storageTypesRequested) throws InvalidToken {
- if (storageTypesRequested.length == 0) {
- throw new InvalidToken("The request has no StorageTypes. "
+ public static <T> void checkAccess(T[] candidates, T[] requested, String msg)
+ throws InvalidToken {
+ if (requested.length == 0) {
+ throw new InvalidToken("The request has no " + msg + ". "
+ "This is probably a configuration error.");
}
- if (candidateStorageTypes.length == 0) {
+ if (candidates.length == 0) {
return;
}
- List<StorageType> unseenCandidates = new ArrayList<StorageType>();
- unseenCandidates.addAll(Arrays.asList(candidateStorageTypes));
- for (StorageType storageType : storageTypesRequested) {
- final int index = unseenCandidates.indexOf(storageType);
+ List unseenCandidates = new ArrayList<T>();
+ unseenCandidates.addAll(Arrays.asList(candidates));
+ for (T req : requested) {
+ final int index = unseenCandidates.indexOf(req);
if (index == -1) {
- throw new InvalidToken("Block token with StorageTypes "
- + Arrays.toString(candidateStorageTypes)
- + " not valid for access with StorageTypes "
- + Arrays.toString(storageTypesRequested));
+ throw new InvalidToken("Block token with " + msg + " "
+ + Arrays.toString(candidates)
+ + " not valid for access with " + msg + " "
+ + Arrays.toString(requested));
}
Collections.swap(unseenCandidates, index, unseenCandidates.size()-1);
unseenCandidates.remove(unseenCandidates.size()-1);
@@ -342,7 +347,7 @@ public class BlockTokenSecretManager extends
/** Check if access should be allowed. userID is not checked if null */
public void checkAccess(Token<BlockTokenIdentifier> token, String userId,
ExtendedBlock block, BlockTokenIdentifier.AccessMode mode,
- StorageType[] storageTypes) throws InvalidToken {
+ StorageType[] storageTypes, String[] storageIds) throws InvalidToken {
BlockTokenIdentifier id = new BlockTokenIdentifier();
try {
id.readFields(new DataInputStream(new ByteArrayInputStream(token
@@ -352,7 +357,7 @@ public class BlockTokenSecretManager extends
"Unable to de-serialize block token identifier for user=" + userId
+ ", block=" + block + ", access mode=" + mode);
}
- checkAccess(id, userId, block, mode, storageTypes);
+ checkAccess(id, userId, block, mode, storageTypes, storageIds);
if (!Arrays.equals(retrievePassword(id), token.getPassword())) {
throw new InvalidToken("Block token with " + id.toString()
+ " doesn't have the correct token password");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
index 91dc907..f855e45 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
@@ -357,7 +357,7 @@ public class Dispatcher {
reportedBlock.getBlock());
final KeyManager km = nnc.getKeyManager();
Token<BlockTokenIdentifier> accessToken = km.getAccessToken(eb,
- new StorageType[]{target.storageType});
+ new StorageType[]{target.storageType}, new String[0]);
IOStreamPair saslStreams = saslClient.socketSend(sock, unbufOut,
unbufIn, km, accessToken, target.getDatanodeInfo());
unbufOut = saslStreams.out;
@@ -411,7 +411,8 @@ public class Dispatcher {
private void sendRequest(DataOutputStream out, ExtendedBlock eb,
Token<BlockTokenIdentifier> accessToken) throws IOException {
new Sender(out).replaceBlock(eb, target.storageType, accessToken,
- source.getDatanodeInfo().getDatanodeUuid(), proxySource.datanode);
+ source.getDatanodeInfo().getDatanodeUuid(), proxySource.datanode,
+ null);
}
/** Check whether to continue waiting for response */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/KeyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/KeyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/KeyManager.java
index 06bf07f..faf95b7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/KeyManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/KeyManager.java
@@ -95,7 +95,7 @@ public class KeyManager implements Closeable, DataEncryptionKeyFactory {
/** Get an access token for a block. */
public Token<BlockTokenIdentifier> getAccessToken(ExtendedBlock eb,
- StorageType[] storageTypes) throws IOException {
+ StorageType[] storageTypes, String[] storageIds) throws IOException {
if (!isBlockTokenEnabled) {
return BlockTokenSecretManager.DUMMY_TOKEN;
} else {
@@ -105,7 +105,7 @@ public class KeyManager implements Closeable, DataEncryptionKeyFactory {
}
return blockTokenSecretManager.generateToken(null, eb,
EnumSet.of(BlockTokenIdentifier.AccessMode.REPLACE,
- BlockTokenIdentifier.AccessMode.COPY), storageTypes);
+ BlockTokenIdentifier.AccessMode.COPY), storageTypes, storageIds);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index e63930a..8f58e25 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -1283,13 +1283,15 @@ public class BlockManager implements BlockStatsMXBean {
internalBlock.setBlockId(b.getBlock().getBlockId() + indices[i]);
blockTokens[i] = blockTokenSecretManager.generateToken(
NameNode.getRemoteUser().getShortUserName(),
- internalBlock, EnumSet.of(mode), b.getStorageTypes());
+ internalBlock, EnumSet.of(mode), b.getStorageTypes(),
+ b.getStorageIDs());
}
sb.setBlockTokens(blockTokens);
} else {
b.setBlockToken(blockTokenSecretManager.generateToken(
NameNode.getRemoteUser().getShortUserName(),
- b.getBlock(), EnumSet.of(mode), b.getStorageTypes()));
+ b.getBlock(), EnumSet.of(mode), b.getStorageTypes(),
+ b.getStorageIDs()));
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
index e0daca7..042169a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
@@ -679,7 +679,8 @@ class BPOfferService {
case DatanodeProtocol.DNA_TRANSFER:
// Send a copy of a block to another datanode
dn.transferBlocks(bcmd.getBlockPoolId(), bcmd.getBlocks(),
- bcmd.getTargets(), bcmd.getTargetStorageTypes());
+ bcmd.getTargets(), bcmd.getTargetStorageTypes(),
+ bcmd.getTargetStorageIDs());
break;
case DatanodeProtocol.DNA_INVALIDATE:
//
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
index 00109e0..2ab4067 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
@@ -151,7 +151,8 @@ class BlockReceiver implements Closeable {
final DataNode datanode, DataChecksum requestedChecksum,
CachingStrategy cachingStrategy,
final boolean allowLazyPersist,
- final boolean pinning) throws IOException {
+ final boolean pinning,
+ final String storageId) throws IOException {
try{
this.block = block;
this.in = in;
@@ -197,6 +198,7 @@ class BlockReceiver implements Closeable {
+ "\n allowLazyPersist=" + allowLazyPersist + ", pinning=" + pinning
+ ", isClient=" + isClient + ", isDatanode=" + isDatanode
+ ", responseInterval=" + responseInterval
+ + ", storageID=" + (storageId != null ? storageId : "null")
);
}
@@ -204,11 +206,13 @@ class BlockReceiver implements Closeable {
// Open local disk out
//
if (isDatanode) { //replication or move
- replicaHandler = datanode.data.createTemporary(storageType, block);
+ replicaHandler =
+ datanode.data.createTemporary(storageType, storageId, block);
} else {
switch (stage) {
case PIPELINE_SETUP_CREATE:
- replicaHandler = datanode.data.createRbw(storageType, block, allowLazyPersist);
+ replicaHandler = datanode.data.createRbw(storageType, storageId,
+ block, allowLazyPersist);
datanode.notifyNamenodeReceivingBlock(
block, replicaHandler.getReplica().getStorageUuid());
break;
@@ -233,7 +237,7 @@ class BlockReceiver implements Closeable {
case TRANSFER_FINALIZED:
// this is a transfer destination
replicaHandler =
- datanode.data.createTemporary(storageType, block);
+ datanode.data.createTemporary(storageType, storageId, block);
break;
default: throw new IOException("Unsupported stage " + stage +
" while receiving block " + block + " from " + inAddr);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 66ef89a..2305e0b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1943,7 +1943,7 @@ public class DataNode extends ReconfigurableBase
LOG.debug("Got: " + id.toString());
}
blockPoolTokenSecretManager.checkAccess(id, null, block, accessMode,
- null);
+ null, null);
}
}
@@ -2224,7 +2224,8 @@ public class DataNode extends ReconfigurableBase
@VisibleForTesting
void transferBlock(ExtendedBlock block, DatanodeInfo[] xferTargets,
- StorageType[] xferTargetStorageTypes) throws IOException {
+ StorageType[] xferTargetStorageTypes, String[] xferTargetStorageIDs)
+ throws IOException {
BPOfferService bpos = getBPOSForBlock(block);
DatanodeRegistration bpReg = getDNRegistrationForBP(block.getBlockPoolId());
@@ -2281,17 +2282,19 @@ public class DataNode extends ReconfigurableBase
LOG.info(bpReg + " Starting thread to transfer " +
block + " to " + xfersBuilder);
- new Daemon(new DataTransfer(xferTargets, xferTargetStorageTypes, block,
+ new Daemon(new DataTransfer(xferTargets, xferTargetStorageTypes,
+ xferTargetStorageIDs, block,
BlockConstructionStage.PIPELINE_SETUP_CREATE, "")).start();
}
}
void transferBlocks(String poolId, Block blocks[],
- DatanodeInfo xferTargets[][], StorageType[][] xferTargetStorageTypes) {
+ DatanodeInfo[][] xferTargets, StorageType[][] xferTargetStorageTypes,
+ String[][] xferTargetStorageIDs) {
for (int i = 0; i < blocks.length; i++) {
try {
transferBlock(new ExtendedBlock(poolId, blocks[i]), xferTargets[i],
- xferTargetStorageTypes[i]);
+ xferTargetStorageTypes[i], xferTargetStorageIDs[i]);
} catch (IOException ie) {
LOG.warn("Failed to transfer block " + blocks[i], ie);
}
@@ -2395,6 +2398,7 @@ public class DataNode extends ReconfigurableBase
private class DataTransfer implements Runnable {
final DatanodeInfo[] targets;
final StorageType[] targetStorageTypes;
+ final private String[] targetStorageIds;
final ExtendedBlock b;
final BlockConstructionStage stage;
final private DatanodeRegistration bpReg;
@@ -2406,8 +2410,8 @@ public class DataNode extends ReconfigurableBase
* entire target list, the block, and the data.
*/
DataTransfer(DatanodeInfo targets[], StorageType[] targetStorageTypes,
- ExtendedBlock b, BlockConstructionStage stage,
- final String clientname) {
+ String[] targetStorageIds, ExtendedBlock b,
+ BlockConstructionStage stage, final String clientname) {
if (DataTransferProtocol.LOG.isDebugEnabled()) {
DataTransferProtocol.LOG.debug(getClass().getSimpleName() + ": "
+ b + " (numBytes=" + b.getNumBytes() + ")"
@@ -2415,10 +2419,13 @@ public class DataNode extends ReconfigurableBase
+ ", clientname=" + clientname
+ ", targets=" + Arrays.asList(targets)
+ ", target storage types=" + (targetStorageTypes == null ? "[]" :
- Arrays.asList(targetStorageTypes)));
+ Arrays.asList(targetStorageTypes))
+ + ", target storage IDs=" + (targetStorageIds == null ? "[]" :
+ Arrays.asList(targetStorageIds)));
}
this.targets = targets;
this.targetStorageTypes = targetStorageTypes;
+ this.targetStorageIds = targetStorageIds;
this.b = b;
this.stage = stage;
BPOfferService bpos = blockPoolManager.get(b.getBlockPoolId());
@@ -2456,7 +2463,7 @@ public class DataNode extends ReconfigurableBase
//
Token<BlockTokenIdentifier> accessToken = getBlockAccessToken(b,
EnumSet.of(BlockTokenIdentifier.AccessMode.WRITE),
- targetStorageTypes);
+ targetStorageTypes, targetStorageIds);
long writeTimeout = dnConf.socketWriteTimeout +
HdfsConstants.WRITE_TIMEOUT_EXTENSION * (targets.length-1);
@@ -2477,10 +2484,13 @@ public class DataNode extends ReconfigurableBase
DatanodeInfo srcNode = new DatanodeInfoBuilder().setNodeID(bpReg)
.build();
+ String storageId = targetStorageIds.length > 0 ?
+ targetStorageIds[0] : null;
new Sender(out).writeBlock(b, targetStorageTypes[0], accessToken,
clientname, targets, targetStorageTypes, srcNode,
stage, 0, 0, 0, 0, blockSender.getChecksum(), cachingStrategy,
- false, false, null);
+ false, false, null, storageId,
+ targetStorageIds);
// send data & checksum
blockSender.sendBlock(out, unbufOut, null);
@@ -2540,12 +2550,12 @@ public class DataNode extends ReconfigurableBase
*/
public Token<BlockTokenIdentifier> getBlockAccessToken(ExtendedBlock b,
EnumSet<AccessMode> mode,
- StorageType[] storageTypes) throws IOException {
+ StorageType[] storageTypes, String[] storageIds) throws IOException {
Token<BlockTokenIdentifier> accessToken =
BlockTokenSecretManager.DUMMY_TOKEN;
if (isBlockTokenEnabled) {
accessToken = blockPoolTokenSecretManager.generateToken(b, mode,
- storageTypes);
+ storageTypes, storageIds);
}
return accessToken;
}
@@ -2918,7 +2928,7 @@ public class DataNode extends ReconfigurableBase
LOG.debug("Got: " + id.toString());
}
blockPoolTokenSecretManager.checkAccess(id, null, block,
- BlockTokenIdentifier.AccessMode.READ, null);
+ BlockTokenIdentifier.AccessMode.READ, null, null);
}
}
}
@@ -2934,7 +2944,8 @@ public class DataNode extends ReconfigurableBase
*/
void transferReplicaForPipelineRecovery(final ExtendedBlock b,
final DatanodeInfo[] targets, final StorageType[] targetStorageTypes,
- final String client) throws IOException {
+ final String[] targetStorageIds, final String client)
+ throws IOException {
final long storedGS;
final long visible;
final BlockConstructionStage stage;
@@ -2967,7 +2978,8 @@ public class DataNode extends ReconfigurableBase
b.setNumBytes(visible);
if (targets.length > 0) {
- new DataTransfer(targets, targetStorageTypes, b, stage, client).run();
+ new DataTransfer(targets, targetStorageTypes, targetStorageIds, b, stage,
+ client).run();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
index cc13799..d42e330 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
@@ -354,7 +354,8 @@ class DataXceiver extends Receiver implements Runnable {
updateCurrentThreadName("Passing file descriptors for block " + blk);
DataOutputStream out = getBufferedOutputStream();
checkAccess(out, true, blk, token,
- Op.REQUEST_SHORT_CIRCUIT_FDS, BlockTokenIdentifier.AccessMode.READ);
+ Op.REQUEST_SHORT_CIRCUIT_FDS, BlockTokenIdentifier.AccessMode.READ,
+ null, null);
BlockOpResponseProto.Builder bld = BlockOpResponseProto.newBuilder();
FileInputStream fis[] = null;
SlotId registeredSlotId = null;
@@ -662,7 +663,7 @@ class DataXceiver extends Receiver implements Runnable {
final Token<BlockTokenIdentifier> blockToken,
final String clientname,
final DatanodeInfo[] targets,
- final StorageType[] targetStorageTypes,
+ final StorageType[] targetStorageTypes,
final DatanodeInfo srcDataNode,
final BlockConstructionStage stage,
final int pipelineSize,
@@ -673,7 +674,9 @@ class DataXceiver extends Receiver implements Runnable {
CachingStrategy cachingStrategy,
boolean allowLazyPersist,
final boolean pinning,
- final boolean[] targetPinnings) throws IOException {
+ final boolean[] targetPinnings,
+ final String storageId,
+ final String[] targetStorageIds) throws IOException {
previousOpClientName = clientname;
updateCurrentThreadName("Receiving block " + block);
final boolean isDatanode = clientname.length() == 0;
@@ -692,8 +695,15 @@ class DataXceiver extends Receiver implements Runnable {
if (targetStorageTypes.length > 0) {
System.arraycopy(targetStorageTypes, 0, storageTypes, 1, nst);
}
+ int nsi = targetStorageIds.length;
+ String[] storageIds = new String[nsi + 1];
+ storageIds[0] = storageId;
+ if (targetStorageTypes.length > 0) {
+ System.arraycopy(targetStorageIds, 0, storageIds, 1, nsi);
+ }
checkAccess(replyOut, isClient, block, blockToken, Op.WRITE_BLOCK,
- BlockTokenIdentifier.AccessMode.WRITE, storageTypes);
+ BlockTokenIdentifier.AccessMode.WRITE,
+ storageTypes, storageIds);
// check single target for transfer-RBW/Finalized
if (isTransfer && targets.length > 0) {
@@ -743,7 +753,7 @@ class DataXceiver extends Receiver implements Runnable {
peer.getLocalAddressString(),
stage, latestGenerationStamp, minBytesRcvd, maxBytesRcvd,
clientname, srcDataNode, datanode, requestedChecksum,
- cachingStrategy, allowLazyPersist, pinning));
+ cachingStrategy, allowLazyPersist, pinning, storageId));
replica = blockReceiver.getReplica();
} else {
replica = datanode.data.recoverClose(
@@ -796,16 +806,18 @@ class DataXceiver extends Receiver implements Runnable {
if (targetPinnings != null && targetPinnings.length > 0) {
new Sender(mirrorOut).writeBlock(originalBlock, targetStorageTypes[0],
- blockToken, clientname, targets, targetStorageTypes, srcDataNode,
- stage, pipelineSize, minBytesRcvd, maxBytesRcvd,
- latestGenerationStamp, requestedChecksum, cachingStrategy,
- allowLazyPersist, targetPinnings[0], targetPinnings);
+ blockToken, clientname, targets, targetStorageTypes,
+ srcDataNode, stage, pipelineSize, minBytesRcvd, maxBytesRcvd,
+ latestGenerationStamp, requestedChecksum, cachingStrategy,
+ allowLazyPersist, targetPinnings[0], targetPinnings,
+ targetStorageIds[0], targetStorageIds);
} else {
new Sender(mirrorOut).writeBlock(originalBlock, targetStorageTypes[0],
- blockToken, clientname, targets, targetStorageTypes, srcDataNode,
- stage, pipelineSize, minBytesRcvd, maxBytesRcvd,
- latestGenerationStamp, requestedChecksum, cachingStrategy,
- allowLazyPersist, false, targetPinnings);
+ blockToken, clientname, targets, targetStorageTypes,
+ srcDataNode, stage, pipelineSize, minBytesRcvd, maxBytesRcvd,
+ latestGenerationStamp, requestedChecksum, cachingStrategy,
+ allowLazyPersist, false, targetPinnings,
+ targetStorageIds[0], targetStorageIds);
}
mirrorOut.flush();
@@ -929,17 +941,19 @@ class DataXceiver extends Receiver implements Runnable {
final Token<BlockTokenIdentifier> blockToken,
final String clientName,
final DatanodeInfo[] targets,
- final StorageType[] targetStorageTypes) throws IOException {
+ final StorageType[] targetStorageTypes,
+ final String[] targetStorageIds) throws IOException {
previousOpClientName = clientName;
updateCurrentThreadName(Op.TRANSFER_BLOCK + " " + blk);
final DataOutputStream out = new DataOutputStream(
getOutputStream());
checkAccess(out, true, blk, blockToken, Op.TRANSFER_BLOCK,
- BlockTokenIdentifier.AccessMode.COPY, targetStorageTypes);
+ BlockTokenIdentifier.AccessMode.COPY, targetStorageTypes,
+ targetStorageIds);
try {
datanode.transferReplicaForPipelineRecovery(blk, targets,
- targetStorageTypes, clientName);
+ targetStorageTypes, targetStorageIds, clientName);
writeResponse(Status.SUCCESS, null, out);
} catch (IOException ioe) {
LOG.info("transferBlock " + blk + " received exception " + ioe);
@@ -1105,12 +1119,14 @@ class DataXceiver extends Receiver implements Runnable {
final StorageType storageType,
final Token<BlockTokenIdentifier> blockToken,
final String delHint,
- final DatanodeInfo proxySource) throws IOException {
+ final DatanodeInfo proxySource,
+ final String storageId) throws IOException {
updateCurrentThreadName("Replacing block " + block + " from " + delHint);
DataOutputStream replyOut = new DataOutputStream(getOutputStream());
checkAccess(replyOut, true, block, blockToken,
Op.REPLACE_BLOCK, BlockTokenIdentifier.AccessMode.REPLACE,
- new StorageType[]{ storageType });
+ new StorageType[]{storageType},
+ new String[]{storageId});
if (!dataXceiverServer.balanceThrottler.acquire()) { // not able to start
String msg = "Not able to receive block " + block.getBlockId() +
@@ -1131,7 +1147,7 @@ class DataXceiver extends Receiver implements Runnable {
// Move the block to different storage in the same datanode
if (proxySource.equals(datanode.getDatanodeId())) {
ReplicaInfo oldReplica = datanode.data.moveBlockAcrossStorage(block,
- storageType);
+ storageType, storageId);
if (oldReplica != null) {
LOG.info("Moved " + block + " from StorageType "
+ oldReplica.getVolume().getStorageType() + " to " + storageType);
@@ -1188,7 +1204,7 @@ class DataXceiver extends Receiver implements Runnable {
proxyReply, proxySock.getRemoteSocketAddress().toString(),
proxySock.getLocalSocketAddress().toString(),
null, 0, 0, 0, "", null, datanode, remoteChecksum,
- CachingStrategy.newDropBehind(), false, false));
+ CachingStrategy.newDropBehind(), false, false, storageId));
// receive a block
blockReceiver.receiveBlock(null, null, replyOut, null,
@@ -1258,11 +1274,12 @@ class DataXceiver extends Receiver implements Runnable {
final DataNode dn, DataChecksum requestedChecksum,
CachingStrategy cachingStrategy,
final boolean allowLazyPersist,
- final boolean pinning) throws IOException {
+ final boolean pinning,
+ final String storageId) throws IOException {
return new BlockReceiver(block, storageType, in,
inAddr, myAddr, stage, newGs, minBytesRcvd, maxBytesRcvd,
clientname, srcDataNode, dn, requestedChecksum,
- cachingStrategy, allowLazyPersist, pinning);
+ cachingStrategy, allowLazyPersist, pinning, storageId);
}
/**
@@ -1365,7 +1382,7 @@ class DataXceiver extends Receiver implements Runnable {
private void checkAccess(OutputStream out, final boolean reply,
ExtendedBlock blk, Token<BlockTokenIdentifier> t, Op op,
BlockTokenIdentifier.AccessMode mode) throws IOException {
- checkAccess(out, reply, blk, t, op, mode, null);
+ checkAccess(out, reply, blk, t, op, mode, null, null);
}
private void checkAccess(OutputStream out, final boolean reply,
@@ -1373,7 +1390,8 @@ class DataXceiver extends Receiver implements Runnable {
final Token<BlockTokenIdentifier> t,
final Op op,
final BlockTokenIdentifier.AccessMode mode,
- final StorageType[] storageTypes) throws IOException {
+ final StorageType[] storageTypes,
+ final String[] storageIds) throws IOException {
checkAndWaitForBP(blk);
if (datanode.isBlockTokenEnabled) {
if (LOG.isDebugEnabled()) {
@@ -1382,7 +1400,7 @@ class DataXceiver extends Receiver implements Runnable {
}
try {
datanode.blockPoolTokenSecretManager.checkAccess(t, null, blk, mode,
- storageTypes);
+ storageTypes, storageIds);
} catch(InvalidToken e) {
try {
if (reply) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
index 1492e5d..e076dda 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
@@ -111,7 +111,8 @@ public final class ErasureCodingWorker {
new StripedReconstructionInfo(
reconInfo.getExtendedBlock(), reconInfo.getErasureCodingPolicy(),
reconInfo.getLiveBlockIndices(), reconInfo.getSourceDnInfos(),
- reconInfo.getTargetDnInfos(), reconInfo.getTargetStorageTypes());
+ reconInfo.getTargetDnInfos(), reconInfo.getTargetStorageTypes(),
+ reconInfo.getTargetStorageIDs());
final StripedBlockReconstructor task =
new StripedBlockReconstructor(this, stripedReconInfo);
if (task.hasValidTargets()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
index b3884c2..39ef67e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
@@ -110,7 +110,7 @@ class StripedBlockReader {
stripedReader.getSocketAddress4Transfer(source);
Token<BlockTokenIdentifier> blockToken = datanode.getBlockAccessToken(
block, EnumSet.of(BlockTokenIdentifier.AccessMode.READ),
- StorageType.EMPTY_ARRAY);
+ StorageType.EMPTY_ARRAY, new String[0]);
/*
* This can be further improved if the replica is local, then we can
* read directly from DN and need to check the replica is FINALIZED
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
index a6989d4..24c1d61 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
@@ -61,6 +61,7 @@ class StripedBlockWriter {
private final ExtendedBlock block;
private final DatanodeInfo target;
private final StorageType storageType;
+ private final String storageId;
private Socket targetSocket;
private DataOutputStream targetOutputStream;
@@ -72,8 +73,8 @@ class StripedBlockWriter {
StripedBlockWriter(StripedWriter stripedWriter, DataNode datanode,
Configuration conf, ExtendedBlock block,
- DatanodeInfo target, StorageType storageType)
- throws IOException {
+ DatanodeInfo target, StorageType storageType,
+ String storageId) throws IOException {
this.stripedWriter = stripedWriter;
this.datanode = datanode;
this.conf = conf;
@@ -81,6 +82,7 @@ class StripedBlockWriter {
this.block = block;
this.target = target;
this.storageType = storageType;
+ this.storageId = storageId;
this.targetBuffer = stripedWriter.allocateWriteBuffer();
@@ -117,7 +119,7 @@ class StripedBlockWriter {
Token<BlockTokenIdentifier> blockToken =
datanode.getBlockAccessToken(block,
EnumSet.of(BlockTokenIdentifier.AccessMode.WRITE),
- new StorageType[]{storageType});
+ new StorageType[]{storageType}, new String[]{storageId});
long writeTimeout = datanode.getDnConf().getSocketWriteTimeout();
OutputStream unbufOut = NetUtils.getOutputStream(socket, writeTimeout);
@@ -141,7 +143,7 @@ class StripedBlockWriter {
new StorageType[]{storageType}, source,
BlockConstructionStage.PIPELINE_SETUP_CREATE, 0, 0, 0, 0,
stripedWriter.getChecksum(), stripedWriter.getCachingStrategy(),
- false, false, null);
+ false, false, null, storageId, new String[]{storageId});
targetSocket = socket;
targetOutputStream = out;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructionInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructionInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructionInfo.java
index a5c328b..a619c34 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructionInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructionInfo.java
@@ -40,24 +40,27 @@ public class StripedReconstructionInfo {
private final byte[] targetIndices;
private final DatanodeInfo[] targets;
private final StorageType[] targetStorageTypes;
+ private final String[] targetStorageIds;
public StripedReconstructionInfo(ExtendedBlock blockGroup,
ErasureCodingPolicy ecPolicy, byte[] liveIndices, DatanodeInfo[] sources,
byte[] targetIndices) {
- this(blockGroup, ecPolicy, liveIndices, sources, targetIndices, null, null);
+ this(blockGroup, ecPolicy, liveIndices, sources, targetIndices, null,
+ null, null);
}
StripedReconstructionInfo(ExtendedBlock blockGroup,
ErasureCodingPolicy ecPolicy, byte[] liveIndices, DatanodeInfo[] sources,
- DatanodeInfo[] targets, StorageType[] targetStorageTypes) {
+ DatanodeInfo[] targets, StorageType[] targetStorageTypes,
+ String[] targetStorageIds) {
this(blockGroup, ecPolicy, liveIndices, sources, null, targets,
- targetStorageTypes);
+ targetStorageTypes, targetStorageIds);
}
private StripedReconstructionInfo(ExtendedBlock blockGroup,
ErasureCodingPolicy ecPolicy, byte[] liveIndices, DatanodeInfo[] sources,
byte[] targetIndices, DatanodeInfo[] targets,
- StorageType[] targetStorageTypes) {
+ StorageType[] targetStorageTypes, String[] targetStorageIds) {
this.blockGroup = blockGroup;
this.ecPolicy = ecPolicy;
@@ -66,6 +69,7 @@ public class StripedReconstructionInfo {
this.targetIndices = targetIndices;
this.targets = targets;
this.targetStorageTypes = targetStorageTypes;
+ this.targetStorageIds = targetStorageIds;
}
ExtendedBlock getBlockGroup() {
@@ -95,5 +99,9 @@ public class StripedReconstructionInfo {
StorageType[] getTargetStorageTypes() {
return targetStorageTypes;
}
+
+ String[] getTargetStorageIds() {
+ return targetStorageIds;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java
index 225a7ed..762506c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java
@@ -55,6 +55,7 @@ class StripedWriter {
private final short[] targetIndices;
private boolean hasValidTargets;
private final StorageType[] targetStorageTypes;
+ private final String[] targetStorageIds;
private StripedBlockWriter[] writers;
@@ -77,6 +78,8 @@ class StripedWriter {
assert targets != null;
this.targetStorageTypes = stripedReconInfo.getTargetStorageTypes();
assert targetStorageTypes != null;
+ this.targetStorageIds = stripedReconInfo.getTargetStorageIds();
+ assert targetStorageIds != null;
writers = new StripedBlockWriter[targets.length];
@@ -192,7 +195,7 @@ class StripedWriter {
private StripedBlockWriter createWriter(short index) throws IOException {
return new StripedBlockWriter(this, datanode, conf,
reconstructor.getBlock(targetIndices[index]), targets[index],
- targetStorageTypes[index]);
+ targetStorageTypes[index], targetStorageIds[index]);
}
ByteBuffer allocateWriteBuffer() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java
index 39d9547..efe222f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java
@@ -113,8 +113,8 @@ public class AvailableSpaceVolumeChoosingPolicy<V extends FsVolumeSpi>
new RoundRobinVolumeChoosingPolicy<V>();
@Override
- public V chooseVolume(List<V> volumes,
- long replicaSize) throws IOException {
+ public V chooseVolume(List<V> volumes, long replicaSize, String storageId)
+ throws IOException {
if (volumes.size() < 1) {
throw new DiskOutOfSpaceException("No more available volumes");
}
@@ -125,19 +125,20 @@ public class AvailableSpaceVolumeChoosingPolicy<V extends FsVolumeSpi>
storageType.ordinal() : StorageType.DEFAULT.ordinal();
synchronized (syncLocks[index]) {
- return doChooseVolume(volumes, replicaSize);
+ return doChooseVolume(volumes, replicaSize, storageId);
}
}
- private V doChooseVolume(final List<V> volumes,
- long replicaSize) throws IOException {
+ private V doChooseVolume(final List<V> volumes, long replicaSize,
+ String storageId) throws IOException {
AvailableSpaceVolumeList volumesWithSpaces =
new AvailableSpaceVolumeList(volumes);
if (volumesWithSpaces.areAllVolumesWithinFreeSpaceThreshold()) {
// If they're actually not too far out of whack, fall back on pure round
// robin.
- V volume = roundRobinPolicyBalanced.chooseVolume(volumes, replicaSize);
+ V volume = roundRobinPolicyBalanced.chooseVolume(volumes, replicaSize,
+ storageId);
if (LOG.isDebugEnabled()) {
LOG.debug("All volumes are within the configured free space balance " +
"threshold. Selecting " + volume + " for write of block size " +
@@ -165,7 +166,7 @@ public class AvailableSpaceVolumeChoosingPolicy<V extends FsVolumeSpi>
if (mostAvailableAmongLowVolumes < replicaSize ||
random.nextFloat() < scaledPreferencePercent) {
volume = roundRobinPolicyHighAvailable.chooseVolume(
- highAvailableVolumes, replicaSize);
+ highAvailableVolumes, replicaSize, storageId);
if (LOG.isDebugEnabled()) {
LOG.debug("Volumes are imbalanced. Selecting " + volume +
" from high available space volumes for write of block size "
@@ -173,7 +174,7 @@ public class AvailableSpaceVolumeChoosingPolicy<V extends FsVolumeSpi>
}
} else {
volume = roundRobinPolicyLowAvailable.chooseVolume(
- lowAvailableVolumes, replicaSize);
+ lowAvailableVolumes, replicaSize, storageId);
if (LOG.isDebugEnabled()) {
LOG.debug("Volumes are imbalanced. Selecting " + volume +
" from low available space volumes for write of block size "
@@ -266,7 +267,8 @@ public class AvailableSpaceVolumeChoosingPolicy<V extends FsVolumeSpi>
/**
* Used so that we only check the available space on a given volume once, at
- * the beginning of {@link AvailableSpaceVolumeChoosingPolicy#chooseVolume(List, long)}.
+ * the beginning of
+ * {@link AvailableSpaceVolumeChoosingPolicy#chooseVolume}.
*/
private class AvailableSpaceVolumePair {
private final V volume;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
index 9e979f7..d7e29cf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
@@ -318,7 +318,7 @@ public interface FsDatasetSpi<V extends FsVolumeSpi> extends FSDatasetMBean {
* @return the meta info of the replica which is being written to
* @throws IOException if an error occurs
*/
- ReplicaHandler createTemporary(StorageType storageType,
+ ReplicaHandler createTemporary(StorageType storageType, String storageId,
ExtendedBlock b) throws IOException;
/**
@@ -328,7 +328,7 @@ public interface FsDatasetSpi<V extends FsVolumeSpi> extends FSDatasetMBean {
* @return the meta info of the replica which is being written to
* @throws IOException if an error occurs
*/
- ReplicaHandler createRbw(StorageType storageType,
+ ReplicaHandler createRbw(StorageType storageType, String storageId,
ExtendedBlock b, boolean allowLazyPersist) throws IOException;
/**
@@ -623,7 +623,7 @@ public interface FsDatasetSpi<V extends FsVolumeSpi> extends FSDatasetMBean {
* Move block from one storage to another storage
*/
ReplicaInfo moveBlockAcrossStorage(final ExtendedBlock block,
- StorageType targetStorageType) throws IOException;
+ StorageType targetStorageType, String storageId) throws IOException;
/**
* Set a block to be pinned on this datanode so that it cannot be moved
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RoundRobinVolumeChoosingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RoundRobinVolumeChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RoundRobinVolumeChoosingPolicy.java
index 9474b92..b9bcf1f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RoundRobinVolumeChoosingPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RoundRobinVolumeChoosingPolicy.java
@@ -50,7 +50,7 @@ public class RoundRobinVolumeChoosingPolicy<V extends FsVolumeSpi>
}
@Override
- public V chooseVolume(final List<V> volumes, long blockSize)
+ public V chooseVolume(final List<V> volumes, long blockSize, String storageId)
throws IOException {
if (volumes.size() < 1) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/VolumeChoosingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/VolumeChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/VolumeChoosingPolicy.java
index 62b1e75..8cbc058 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/VolumeChoosingPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/VolumeChoosingPolicy.java
@@ -36,8 +36,11 @@ public interface VolumeChoosingPolicy<V extends FsVolumeSpi> {
*
* @param volumes - a list of available volumes.
* @param replicaSize - the size of the replica for which a volume is sought.
+ * @param storageId - the storage id of the Volume nominated by the namenode.
+ * This can usually be ignored by the VolumeChoosingPolicy.
* @return the chosen volume.
* @throws IOException when disks are unavailable or are full.
*/
- public V chooseVolume(List<V> volumes, long replicaSize) throws IOException;
+ V chooseVolume(List<V> volumes, long replicaSize, String storageId)
+ throws IOException;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 169e0e6..9a5002a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -927,7 +927,8 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
*/
@Override
public ReplicaInfo moveBlockAcrossStorage(ExtendedBlock block,
- StorageType targetStorageType) throws IOException {
+ StorageType targetStorageType, String targetStorageId)
+ throws IOException {
ReplicaInfo replicaInfo = getReplicaInfo(block);
if (replicaInfo.getState() != ReplicaState.FINALIZED) {
throw new ReplicaNotFoundException(
@@ -952,7 +953,8 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
FsVolumeReference volumeRef = null;
try (AutoCloseableLock lock = datasetLock.acquire()) {
- volumeRef = volumes.getNextVolume(targetStorageType, block.getNumBytes());
+ volumeRef = volumes.getNextVolume(targetStorageType, targetStorageId,
+ block.getNumBytes());
}
try {
moveBlock(block, replicaInfo, volumeRef);
@@ -1298,11 +1300,11 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
}
}
-
+
@Override // FsDatasetSpi
public ReplicaHandler createRbw(
- StorageType storageType, ExtendedBlock b, boolean allowLazyPersist)
- throws IOException {
+ StorageType storageType, String storageId, ExtendedBlock b,
+ boolean allowLazyPersist) throws IOException {
try (AutoCloseableLock lock = datasetLock.acquire()) {
ReplicaInfo replicaInfo = volumeMap.get(b.getBlockPoolId(),
b.getBlockId());
@@ -1335,7 +1337,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
if (ref == null) {
- ref = volumes.getNextVolume(storageType, b.getNumBytes());
+ ref = volumes.getNextVolume(storageType, storageId, b.getNumBytes());
}
FsVolumeImpl v = (FsVolumeImpl) ref.getVolume();
@@ -1503,7 +1505,8 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
@Override // FsDatasetSpi
public ReplicaHandler createTemporary(
- StorageType storageType, ExtendedBlock b) throws IOException {
+ StorageType storageType, String storageId, ExtendedBlock b)
+ throws IOException {
long startTimeMs = Time.monotonicNow();
long writerStopTimeoutMs = datanode.getDnConf().getXceiverStopTimeout();
ReplicaInfo lastFoundReplicaInfo = null;
@@ -1516,7 +1519,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
invalidate(b.getBlockPoolId(), new Block[] { lastFoundReplicaInfo });
}
FsVolumeReference ref =
- volumes.getNextVolume(storageType, b.getNumBytes());
+ volumes.getNextVolume(storageType, storageId, b.getNumBytes());
FsVolumeImpl v = (FsVolumeImpl) ref.getVolume();
ReplicaInPipeline newReplicaInfo;
try {
@@ -2899,7 +2902,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
replicaInfo.getVolume().isTransientStorage()) {
// Pick a target volume to persist the block.
targetReference = volumes.getNextVolume(
- StorageType.DEFAULT, replicaInfo.getNumBytes());
+ StorageType.DEFAULT, null, replicaInfo.getNumBytes());
targetVolume = (FsVolumeImpl) targetReference.getVolume();
ramDiskReplicaTracker.recordStartLazyPersist(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: HADOOP-14384. Reduce the
visibility of FileSystem.newFSDataOutputStreamBuilder before the API becomes
stable. (lei)
Posted by in...@apache.org.
HADOOP-14384. Reduce the visibility of FileSystem.newFSDataOutputStreamBuilder before the API becomes stable. (lei)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/149b903b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/149b903b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/149b903b
Branch: refs/heads/HDFS-10467
Commit: 149b903be30e2b973691c30ee02f0d5b53ca0abc
Parents: 543aac9
Author: Lei Xu <le...@apache.org>
Authored: Tue May 9 10:27:17 2017 -0700
Committer: Lei Xu <le...@apache.org>
Committed: Tue May 9 10:27:17 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java | 8 ++++++--
.../src/main/java/org/apache/hadoop/fs/FileSystem.java | 6 +++++-
.../src/main/java/org/apache/hadoop/fs/FilterFileSystem.java | 2 +-
3 files changed, 12 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/149b903b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
index 2e885f3..55836cc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.fs;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.Progressable;
@@ -29,7 +31,9 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;
/** Base of specific file system FSDataOutputStreamBuilder. */
-public class FSDataOutputStreamBuilder{
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class FSDataOutputStreamBuilder {
private Path path = null;
private FsPermission permission = null;
private Integer bufferSize;
@@ -41,7 +45,7 @@ public class FSDataOutputStreamBuilder{
private final FileSystem fs;
- public FSDataOutputStreamBuilder(FileSystem fileSystem, Path p) {
+ protected FSDataOutputStreamBuilder(FileSystem fileSystem, Path p) {
fs = fileSystem;
path = p;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/149b903b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 29da214..1907475 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -4144,8 +4144,12 @@ public abstract class FileSystem extends Configured implements Closeable {
* Create a new FSDataOutputStreamBuilder for the file with path.
* @param path file path
* @return a FSDataOutputStreamBuilder object to build the file
+ *
+ * HADOOP-14384. Temporarily reduce the visibility of method before the
+ * builder interface becomes stable.
*/
- public FSDataOutputStreamBuilder newFSDataOutputStreamBuilder(Path path) {
+ @InterfaceAudience.Private
+ protected FSDataOutputStreamBuilder newFSDataOutputStreamBuilder(Path path) {
return new FSDataOutputStreamBuilder(this, path);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/149b903b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
index ef09458..3466922 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
@@ -667,7 +667,7 @@ public class FilterFileSystem extends FileSystem {
}
@Override
- public FSDataOutputStreamBuilder newFSDataOutputStreamBuilder(Path path) {
+ protected FSDataOutputStreamBuilder newFSDataOutputStreamBuilder(Path path) {
return fs.newFSDataOutputStreamBuilder(path);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: HDFS-11681.
DatanodeStorageInfo#getBlockIterator() should return an iterator to an
unmodifiable set Contributed by Virajith Jalaparti
Posted by in...@apache.org.
HDFS-11681. DatanodeStorageInfo#getBlockIterator() should return an iterator to an unmodifiable set Contributed by Virajith Jalaparti
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/51b671ef
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/51b671ef
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/51b671ef
Branch: refs/heads/HDFS-10467
Commit: 51b671ef1844069888f976cd16f66c88f9bbc7de
Parents: eed7314
Author: Chris Douglas <cd...@apache.org>
Authored: Wed May 10 22:20:27 2017 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Wed May 10 22:25:28 2017 -0700
----------------------------------------------------------------------
.../server/blockmanagement/BlockManager.java | 23 +++++++++++---------
.../blockmanagement/DatanodeStorageInfo.java | 7 +++++-
.../org/apache/hadoop/hdfs/TestGetBlocks.java | 7 ++++++
3 files changed, 26 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51b671ef/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 41662a4..a9592bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -1409,12 +1409,15 @@ public class BlockManager implements BlockStatsMXBean {
void removeBlocksAssociatedTo(final DatanodeDescriptor node) {
for (DatanodeStorageInfo storage : node.getStorageInfos()) {
final Iterator<BlockInfo> it = storage.getBlockIterator();
+ //add the BlockInfos to a new collection as the
+ //returned iterator is not modifiable.
+ Collection<BlockInfo> toRemove = new ArrayList<>();
while (it.hasNext()) {
- BlockInfo block = it.next();
- // DatanodeStorageInfo must be removed using the iterator to avoid
- // ConcurrentModificationException in the underlying storage
- it.remove();
- removeStoredBlock(block, node);
+ toRemove.add(it.next());
+ }
+
+ for (BlockInfo b : toRemove) {
+ removeStoredBlock(b, node);
}
}
// Remove all pending DN messages referencing this DN.
@@ -1429,11 +1432,11 @@ public class BlockManager implements BlockStatsMXBean {
assert namesystem.hasWriteLock();
final Iterator<BlockInfo> it = storageInfo.getBlockIterator();
DatanodeDescriptor node = storageInfo.getDatanodeDescriptor();
- while(it.hasNext()) {
- BlockInfo block = it.next();
- // DatanodeStorageInfo must be removed using the iterator to avoid
- // ConcurrentModificationException in the underlying storage
- it.remove();
+ Collection<BlockInfo> toRemove = new ArrayList<>();
+ while (it.hasNext()) {
+ toRemove.add(it.next());
+ }
+ for (BlockInfo block : toRemove) {
removeStoredBlock(block, node);
final Block b = getBlockOnStorage(block, storageInfo);
if (b != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51b671ef/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index ab666b7..8af86d3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.blockmanagement;
import java.util.Arrays;
+import java.util.Collections;
import java.util.Iterator;
import java.util.List;
@@ -270,8 +271,12 @@ public class DatanodeStorageInfo {
return blocks.size();
}
+ /**
+ * @return iterator to an unmodifiable set of blocks
+ * related to this {@link DatanodeStorageInfo}
+ */
Iterator<BlockInfo> getBlockIterator() {
- return blocks.iterator();
+ return Collections.unmodifiableSet(blocks).iterator();
}
void updateState(StorageReport r) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51b671ef/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
index 32bdd78..7397507 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
@@ -307,6 +307,13 @@ public class TestGetBlocks {
BlockManagerTestUtil.getBlockIterator(s);
while(storageBlockIt.hasNext()) {
allBlocks[idx++] = storageBlockIt.next();
+ try {
+ storageBlockIt.remove();
+ assertTrue(
+ "BlockInfo iterator should have been unmodifiable", false);
+ } catch (UnsupportedOperationException e) {
+ //expected exception
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: HDFS-10881. Federation State Store
Driver API. Contributed by Jason Kace and Inigo Goiri.
Posted by in...@apache.org.
HDFS-10881. Federation State Store Driver API. Contributed by Jason Kace and Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/15976e01
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/15976e01
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/15976e01
Branch: refs/heads/HDFS-10467
Commit: 15976e01cdb0941c383e907b93ffa9ab79e5ba00
Parents: d710513
Author: Inigo <in...@apache.org>
Authored: Wed Mar 29 19:35:06 2017 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu May 11 09:54:07 2017 -0700
----------------------------------------------------------------------
.../store/StateStoreUnavailableException.java | 33 ++++
.../federation/store/StateStoreUtils.java | 72 +++++++
.../store/driver/StateStoreDriver.java | 172 +++++++++++++++++
.../driver/StateStoreRecordOperations.java | 164 ++++++++++++++++
.../store/driver/impl/StateStoreBaseImpl.java | 69 +++++++
.../store/driver/impl/package-info.java | 39 ++++
.../federation/store/driver/package-info.java | 37 ++++
.../federation/store/protocol/package-info.java | 31 +++
.../federation/store/records/BaseRecord.java | 189 +++++++++++++++++++
.../federation/store/records/QueryResult.java | 56 ++++++
.../federation/store/records/package-info.java | 36 ++++
11 files changed, 898 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15976e01/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUnavailableException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUnavailableException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUnavailableException.java
new file mode 100644
index 0000000..4e6f8c8
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUnavailableException.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import java.io.IOException;
+
+/**
+ * Thrown when the state store is not reachable or available. Cached APIs and
+ * queries may succeed. Client should retry again later.
+ */
+public class StateStoreUnavailableException extends IOException {
+
+ private static final long serialVersionUID = 1L;
+
+ public StateStoreUnavailableException(String msg) {
+ super(msg);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15976e01/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUtils.java
new file mode 100644
index 0000000..8c681df
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUtils.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+
+/**
+ * Set of utility functions used to query, create, update and delete data
+ * records in the state store.
+ */
+public final class StateStoreUtils {
+
+ private static final Log LOG = LogFactory.getLog(StateStoreUtils.class);
+
+ private StateStoreUtils() {
+ // Utility class
+ }
+
+ /**
+ * Get the base class for a record class. If we get an implementation of a
+ * record we will return the real parent record class.
+ *
+ * @param clazz Class of the data record to check.
+ * @return Base class for the record.
+ */
+ @SuppressWarnings("unchecked")
+ public static <T extends BaseRecord>
+ Class<? extends BaseRecord> getRecordClass(final Class<T> clazz) {
+
+ // We ignore the Impl classes and go to the super class
+ Class<? extends BaseRecord> actualClazz = clazz;
+ while (actualClazz.getSimpleName().endsWith("Impl")) {
+ actualClazz = (Class<? extends BaseRecord>) actualClazz.getSuperclass();
+ }
+
+ // Check if we went too far
+ if (actualClazz.equals(BaseRecord.class)) {
+ LOG.error("We went too far (" + actualClazz + ") with " + clazz);
+ actualClazz = clazz;
+ }
+ return actualClazz;
+ }
+
+ /**
+ * Get the base class for a record. If we get an implementation of a record we
+ * will return the real parent record class.
+ *
+ * @param record Record to check its main class.
+ * @return Base class for the record.
+ */
+ public static <T extends BaseRecord>
+ Class<? extends BaseRecord> getRecordClass(final T record) {
+ return getRecordClass(record.getClass());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15976e01/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java
new file mode 100644
index 0000000..a1527df
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java
@@ -0,0 +1,172 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver;
+
+import java.net.InetAddress;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreUnavailableException;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.util.Time;
+
+/**
+ * Driver class for an implementation of a {@link StateStoreService}
+ * provider. Driver implementations will extend this class and implement some of
+ * the default methods.
+ */
+public abstract class StateStoreDriver implements StateStoreRecordOperations {
+
+ private static final Log LOG = LogFactory.getLog(StateStoreDriver.class);
+
+
+ /** State Store configuration. */
+ private Configuration conf;
+
+ /** Identifier for the driver. */
+ private String identifier;
+
+
+ /**
+ * Initialize the state store connection.
+ * @param config Configuration for the driver.
+ * @param id Identifier for the driver.
+ * @param records Records that are supported.
+ * @return If initialized and ready, false if failed to initialize driver.
+ */
+ public boolean init(final Configuration config, final String id,
+ final List<Class<? extends BaseRecord>> records) {
+
+ this.conf = config;
+ this.identifier = id;
+
+ if (this.identifier == null) {
+ LOG.warn("The identifier for the State Store connection is not set");
+ }
+
+ // TODO stub
+ return false;
+ }
+
+ /**
+ * Get the State Store configuration.
+ *
+ * @return Configuration for the State Store.
+ */
+ protected Configuration getConf() {
+ return this.conf;
+ }
+
+ /**
+ * Gets a unique identifier for the running task/process. Typically the
+ * router address.
+ *
+ * @return Unique identifier for the running task.
+ */
+ public String getIdentifier() {
+ return this.identifier;
+ }
+
+ /**
+ * Prepare the driver to access data storage.
+ *
+ * @return True if the driver was successfully initialized. If false is
+ * returned, the state store will periodically attempt to
+ * re-initialize the driver and the router will remain in safe mode
+ * until the driver is initialized.
+ */
+ public abstract boolean initDriver();
+
+ /**
+ * Initialize storage for a single record class.
+ *
+ * @param name String reference of the record class to initialize, used to
+ * construct paths and file names for the record. Determined by
+ * configuration settings for the specific driver.
+ * @param clazz Record type corresponding to the provided name.
+ * @return True if successful, false otherwise.
+ */
+ public abstract <T extends BaseRecord> boolean initRecordStorage(
+ String className, Class<T> clazz);
+
+ /**
+ * Check if the driver is currently running and the data store connection is
+ * valid.
+ *
+ * @return True if the driver is initialized and the data store is ready.
+ */
+ public abstract boolean isDriverReady();
+
+ /**
+ * Check if the driver is ready to be used and throw an exception otherwise.
+ *
+ * @throws StateStoreUnavailableException If the driver is not ready.
+ */
+ public void verifyDriverReady() throws StateStoreUnavailableException {
+ if (!isDriverReady()) {
+ String driverName = getDriverName();
+ String hostname = getHostname();
+ throw new StateStoreUnavailableException("State Store driver " +
+ driverName + " in " + hostname + " is not ready.");
+ }
+ }
+
+ /**
+ * Close the State Store driver connection.
+ */
+ public abstract void close() throws Exception;
+
+ /**
+ * Returns the current time synchronization from the underlying store.
+ * Override for stores that supply a current date. The data store driver is
+ * responsible for maintaining the official synchronization time/date for all
+ * distributed components.
+ *
+ * @return Current time stamp, used for all synchronization dates.
+ */
+ public long getTime() {
+ return Time.now();
+ }
+
+ /**
+ * Get the name of the driver implementation for debugging.
+ *
+ * @return Name of the driver implementation.
+ */
+ private String getDriverName() {
+ return this.getClass().getSimpleName();
+ }
+
+ /**
+ * Get the host name of the machine running the driver for debugging.
+ *
+ * @return Host name of the machine running the driver.
+ */
+ private String getHostname() {
+ String hostname = "Unknown";
+ try {
+ hostname = InetAddress.getLocalHost().getHostName();
+ } catch (Exception e) {
+ LOG.error("Cannot get local address", e);
+ }
+ return hostname;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15976e01/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
new file mode 100644
index 0000000..739eeba
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
+import org.apache.hadoop.io.retry.AtMostOnce;
+import org.apache.hadoop.io.retry.Idempotent;
+
+/**
+ * Operations for a driver to manage records in the State Store.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface StateStoreRecordOperations {
+
+ /**
+ * Get all records of the requested record class from the data store. To use
+ * the default implementations in this class, getAll must return new instances
+ * of the records on each call. It is recommended to override the default
+ * implementations for better performance.
+ *
+ * @param clazz Class of record to fetch.
+ * @return List of all records that match the clazz.
+ * @throws IOException Throws exception if unable to query the data store.
+ */
+ @Idempotent
+ <T extends BaseRecord> QueryResult<T> get(Class<T> clazz) throws IOException;
+
+ /**
+ * Get all records of the requested record class from the data store. To use
+ * the default implementations in this class, getAll must return new instances
+ * of the records on each call. It is recommended to override the default
+ * implementations for better performance.
+ *
+ * @param clazz Class of record to fetch.
+ * @param sub Sub path.
+ * @return List of all records that match the clazz and the sub path.
+ * @throws IOException
+ */
+ @Idempotent
+ <T extends BaseRecord> QueryResult<T> get(Class<T> clazz, String sub)
+ throws IOException;
+
+ /**
+ * Get a single record from the store that matches the query.
+ *
+ * @param clazz Class of record to fetch.
+ * @param query Map of field names and objects to filter results.
+ * @return A single record matching the query. Null if there are no matching
+ * records or more than one matching record in the store.
+ * @throws IOException If multiple records match or if the data store cannot
+ * be queried.
+ */
+ @Idempotent
+ <T extends BaseRecord> T get(Class<T> clazz, Map<String, String> query)
+ throws IOException;
+
+ /**
+ * Get multiple records from the store that match a query. This method
+ * assumes the underlying driver does not support filtering. If the driver
+ * supports filtering it should overwrite this method.
+ *
+ * @param clazz Class of record to fetch.
+ * @param query Map of field names and objects to filter results.
+ * @return Records of type clazz that match the query or empty list if none
+ * are found.
+ * @throws IOException Throws exception if unable to query the data store.
+ */
+ @Idempotent
+ <T extends BaseRecord> List<T> getMultiple(
+ Class<T> clazz, Map<String, String> query) throws IOException;
+
+ /**
+ * Creates a single record. Optionally updates an existing record with same
+ * primary key.
+ *
+ * @param record The record to insert or update.
+ * @param allowUpdate True if update of exiting record is allowed.
+ * @param errorIfExists True if an error should be returned when inserting
+ * an existing record. Only used if allowUpdate = false.
+ * @return True if the operation was successful.
+ *
+ * @throws IOException Throws exception if unable to query the data store.
+ */
+ @AtMostOnce
+ <T extends BaseRecord> boolean put(
+ T record, boolean allowUpdate, boolean errorIfExists) throws IOException;
+
+ /**
+ * Creates multiple records. Optionally updates existing records that have
+ * the same primary key.
+ *
+ * @param records List of data records to update or create. All records must
+ * be of class clazz.
+ * @param clazz Record class of records.
+ * @param allowUpdate True if update of exiting record is allowed.
+ * @param errorIfExists True if an error should be returned when inserting
+ * an existing record. Only used if allowUpdate = false.
+ * @return true if all operations were successful.
+ *
+ * @throws IOException Throws exception if unable to query the data store.
+ */
+ @AtMostOnce
+ <T extends BaseRecord> boolean putAll(
+ List<T> records, boolean allowUpdate, boolean errorIfExists)
+ throws IOException;
+
+ /**
+ * Remove a single record.
+ *
+ * @param record Record to be removed.
+ * @return true If the record was successfully removed. False if the record
+ * could not be removed or not stored.
+ * @throws IOException Throws exception if unable to query the data store.
+ */
+ @AtMostOnce
+ <T extends BaseRecord> boolean remove(T record) throws IOException;
+
+ /**
+ * Remove all records of this class from the store.
+ *
+ * @param clazz Class of records to remove.
+ * @return True if successful.
+ * @throws IOException Throws exception if unable to query the data store.
+ */
+ @AtMostOnce
+ <T extends BaseRecord> boolean removeAll(Class<T> clazz) throws IOException;
+
+ /**
+ * Remove multiple records of a specific class that match a query. Requires
+ * the getAll implementation to fetch fresh records on each call.
+ *
+ * @param clazz Class of record to remove.
+ * @param filter matching filter to remove.
+ * @return The number of records removed.
+ * @throws IOException Throws exception if unable to query the data store.
+ */
+ @AtMostOnce
+ <T extends BaseRecord> int remove(Class<T> clazz, Map<String, String> filter)
+ throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15976e01/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreBaseImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreBaseImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreBaseImpl.java
new file mode 100644
index 0000000..b711fa9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreBaseImpl.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+
+/**
+ * Base implementation of a State Store driver. It contains default
+ * implementations for the optional functions. These implementations use an
+ * uncached read/write all algorithm for all changes. In most cases it is
+ * recommended to override the optional functions.
+ * <p>
+ * Drivers may optionally override additional routines for performance
+ * optimization, such as custom get/put/remove queries, depending on the
+ * capabilities of the data store.
+ * <p>
+ */
+public abstract class StateStoreBaseImpl extends StateStoreDriver {
+
+ @Override
+ public <T extends BaseRecord> T get(
+ Class<T> clazz, Map<String, String> query) throws IOException {
+ List<T> records = getMultiple(clazz, query);
+ if (records.size() > 1) {
+ throw new IOException("Found more than one object in collection");
+ } else if (records.size() == 1) {
+ return records.get(0);
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public <T extends BaseRecord> boolean put(
+ T record, boolean allowUpdate, boolean errorIfExists) throws IOException {
+ List<T> singletonList = new ArrayList<T>();
+ singletonList.add(record);
+ return putAll(singletonList, allowUpdate, errorIfExists);
+ }
+
+ @Override
+ public <T extends BaseRecord> boolean remove(T record) throws IOException {
+ Map<String, String> primaryKeys = record.getPrimaryKeys();
+ Class<? extends BaseRecord> clazz = StateStoreUtils.getRecordClass(record);
+ return remove(clazz, primaryKeys) == 1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15976e01/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/package-info.java
new file mode 100644
index 0000000..a18433e
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/package-info.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Implementations of state store data providers/drivers. Each driver is
+ * responsible for maintaining, querying, updating and deleting persistent data
+ * records. Data records are defined as subclasses of
+ * {@link org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord}.
+ * Each driver implements the
+ * {@link org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver
+ * StateStoreDriver} interface.
+ * <p>
+ * Currently supported drivers:
+ * <ul>
+ * <li>{@link StateStoreFileImpl} A file-based data storage backend.
+ * <li>{@link StateStoreZooKeeperImpl} A zookeeper based data storage backend.
+ * </ul>
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15976e01/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/package-info.java
new file mode 100644
index 0000000..da998b5
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/package-info.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The state store uses modular data storage classes derived from
+ * StateStoreDriver to handle querying, updating and deleting data records. The
+ * data storage driver is initialized and maintained by the StateStoreService.
+ * The state store supports fetching all records of a type, filtering by column
+ * values or fetching a single record by its primary key.
+ * <p>
+ * Each data storage backend is required to implement the methods contained in
+ * the StateStoreDriver interface. These methods allow the querying, updating,
+ * inserting and deleting of data records into the state store.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+
+package org.apache.hadoop.hdfs.server.federation.store.driver;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15976e01/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/package-info.java
new file mode 100644
index 0000000..0249d2c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/package-info.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Contains the abstract definitions of the API request and response objects for
+ * the various state store APIs. The state store supports multiple interface
+ * APIs and multiple data records. Each protocol object requires a serialization
+ * implementation, the default is protobuf.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15976e01/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java
new file mode 100644
index 0000000..4192a3d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java
@@ -0,0 +1,189 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+import java.util.Map;
+
+import org.apache.hadoop.util.Time;
+
+/**
+ * Abstract base of a data record in the StateStore. All StateStore records are
+ * derived from this class. Data records are persisted in the data store and
+ * are identified by their primary key. Each data record contains:
+ * <ul>
+ * <li>A primary key consisting of a combination of record data fields.
+ * <li>A modification date.
+ * <li>A creation date.
+ * </ul>
+ */
+public abstract class BaseRecord implements Comparable<BaseRecord> {
+
+ /**
+ * Set the modification time for the record.
+ *
+ * @param time Modification time of the record.
+ */
+ public abstract void setDateModified(long time);
+
+ /**
+ * Get the modification time for the record.
+ *
+ * @return Modification time of the record.
+ */
+ public abstract long getDateModified();
+
+ /**
+ * Set the creation time for the record.
+ *
+ * @param time Creation time of the record.
+ */
+ public abstract void setDateCreated(long time);
+
+ /**
+ * Get the creation time for the record.
+ *
+ * @return Creation time of the record
+ */
+ public abstract long getDateCreated();
+
+ /**
+ * Get the expiration time for the record.
+ *
+ * @return Expiration time for the record.
+ */
+ public abstract long getExpirationMs();
+
+ /**
+ * Map of primary key names->values for the record. The primary key can be a
+ * combination of 1-n different State Store serialized values.
+ *
+ * @return Map of key/value pairs that constitute this object's primary key.
+ */
+ public abstract Map<String, String> getPrimaryKeys();
+
+ /**
+ * Initialize the object.
+ */
+ public void init() {
+ // Call this after the object has been constructed
+ initDefaultTimes();
+ }
+
+ /**
+ * Initialize default times. The driver may update these timestamps on insert
+ * and/or update. This should only be called when initializing an object that
+ * is not backed by a data store.
+ */
+ private void initDefaultTimes() {
+ long now = Time.now();
+ this.setDateCreated(now);
+ this.setDateModified(now);
+ }
+
+ /**
+ * Join the primary keys into one single primary key.
+ *
+ * @return A string that is guaranteed to be unique amongst all records of
+ * this type.
+ */
+ public String getPrimaryKey() {
+ return generateMashupKey(getPrimaryKeys());
+ }
+
+ /**
+ * Generates a cache key from a map of values.
+ *
+ * @param keys Map of values.
+ * @return String mashup of key values.
+ */
+ protected static String generateMashupKey(final Map<String, String> keys) {
+ StringBuilder builder = new StringBuilder();
+ for (Object value : keys.values()) {
+ if (builder.length() > 0) {
+ builder.append("-");
+ }
+ builder.append(value);
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Override equals check to use primary key(s) for comparison.
+ */
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof BaseRecord)) {
+ return false;
+ }
+
+ BaseRecord baseObject = (BaseRecord) obj;
+ Map<String, String> keyset1 = this.getPrimaryKeys();
+ Map<String, String> keyset2 = baseObject.getPrimaryKeys();
+ return keyset1.equals(keyset2);
+ }
+
+ /**
+ * Override hash code to use primary key(s) for comparison.
+ */
+ @Override
+ public int hashCode() {
+ Map<String, String> keyset = this.getPrimaryKeys();
+ return keyset.hashCode();
+ }
+
+ @Override
+ public int compareTo(BaseRecord record) {
+ if (record == null) {
+ return -1;
+ }
+ // Descending date order
+ return (int) (record.getDateModified() - this.getDateModified());
+ }
+
+ /**
+ * Called when the modification time and current time is available, checks for
+ * expirations.
+ *
+ * @param currentTime The current timestamp in ms from the data store, to be
+ * compared against the modification and creation dates of the
+ * object.
+ * @return boolean True if the record has been updated and should be
+ * committed to the data store. Override for customized behavior.
+ */
+ public boolean checkExpired(long currentTime) {
+ long expiration = getExpirationMs();
+ if (getDateModified() > 0 && expiration > 0) {
+ return (getDateModified() + expiration) < currentTime;
+ }
+ return false;
+ }
+
+ /**
+ * Validates the record. Called when the record is created, populated from the
+ * state store, and before committing to the state store.
+ * @return If the record is valid.
+ */
+ public boolean validate() {
+ return getDateCreated() > 0 && getDateModified() > 0;
+ }
+
+ @Override
+ public String toString() {
+ return getPrimaryKey();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15976e01/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/QueryResult.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/QueryResult.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/QueryResult.java
new file mode 100644
index 0000000..64c2c71
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/QueryResult.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Encapsulates a state store query result that includes a set of records and a
+ * time stamp for the result.
+ */
+public class QueryResult<T extends BaseRecord> {
+
+ /** Data result. */
+ private final List<T> records;
+ /** Time stamp of the data results. */
+ private final long timestamp;
+
+ public QueryResult(final List<T> recs, final long time) {
+ this.records = recs;
+ this.timestamp = time;
+ }
+
+ /**
+ * Get the result of the query.
+ *
+ * @return List of records.
+ */
+ public List<T> getRecords() {
+ return Collections.unmodifiableList(this.records);
+ }
+
+ /**
+ * The timetamp in driver time of this query.
+ *
+ * @return Timestamp in driver time.
+ */
+ public long getTimestamp() {
+ return this.timestamp;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15976e01/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/package-info.java
new file mode 100644
index 0000000..63b13af
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/package-info.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Contains the abstract definitions of the state store data records. The state
+ * store supports multiple multiple data records.
+ * <p>
+ * Data records inherit from a common class
+ * {@link org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord
+ * BaseRecord}. Data records are serialized when written to the data store using
+ * a modular serialization implementation. The default is profobuf
+ * serialization. Data is stored as rows of records of the same type with each
+ * data member in a record representing a column.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: YARN-5301. NM mount cpu cgroups
failed on some systems (Contributed by Miklos Szegedi via Daniel Templeton)
Posted by in...@apache.org.
YARN-5301. NM mount cpu cgroups failed on some systems
(Contributed by Miklos Szegedi via Daniel Templeton)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a2f68049
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a2f68049
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a2f68049
Branch: refs/heads/HDFS-10467
Commit: a2f680493f040704e2b85108e286731ee3860a52
Parents: 7dd258d
Author: Daniel Templeton <te...@apache.org>
Authored: Tue May 9 11:18:21 2017 -0700
Committer: Daniel Templeton <te...@apache.org>
Committed: Tue May 9 12:05:46 2017 -0700
----------------------------------------------------------------------
.../linux/resources/CGroupsHandler.java | 6 +-
.../linux/resources/CGroupsHandlerImpl.java | 238 ++++++-----
.../util/CgroupsLCEResourcesHandler.java | 19 +-
.../impl/container-executor.c | 22 +-
.../linux/resources/TestCGroupsHandlerImpl.java | 392 ++++++++++++-------
.../resources/TestResourceHandlerModule.java | 3 -
.../util/TestCgroupsLCEResourcesHandler.java | 49 ++-
7 files changed, 452 insertions(+), 277 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2f68049/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java
index d09a25d..8fc35a8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java
@@ -39,7 +39,11 @@ public interface CGroupsHandler {
CPU("cpu"),
NET_CLS("net_cls"),
BLKIO("blkio"),
- MEMORY("memory");
+ MEMORY("memory"),
+ CPUACCT("cpuacct"),
+ CPUSET("cpuset"),
+ FREEZER("freezer"),
+ DEVICES("devices");
private final String name;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2f68049/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
index 0f4c17e..85b01cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
@@ -21,13 +21,16 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException;
@@ -40,8 +43,9 @@ import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.HashMap;
-import java.util.List;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.regex.Matcher;
@@ -66,6 +70,7 @@ class CGroupsHandlerImpl implements CGroupsHandler {
private final long deleteCGroupTimeout;
private final long deleteCGroupDelay;
private Map<CGroupController, String> controllerPaths;
+ private Map<String, Set<String>> parsedMtab;
private final ReadWriteLock rwLock;
private final PrivilegedOperationExecutor privilegedOperationExecutor;
private final Clock clock;
@@ -95,6 +100,7 @@ class CGroupsHandlerImpl implements CGroupsHandler {
conf.getLong(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_DELETE_DELAY,
YarnConfiguration.DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_DELAY);
this.controllerPaths = new HashMap<>();
+ this.parsedMtab = new HashMap<>();
this.rwLock = new ReentrantReadWriteLock();
this.privilegedOperationExecutor = privilegedOperationExecutor;
this.clock = SystemClock.getInstance();
@@ -128,51 +134,53 @@ class CGroupsHandlerImpl implements CGroupsHandler {
}
private void initializeControllerPaths() throws ResourceHandlerException {
- if (enableCGroupMount) {
- // nothing to do here - we support 'deferred' mounting of specific
- // controllers - we'll populate the path for a given controller when an
- // explicit mountCGroupController request is issued.
- LOG.info("CGroup controller mounting enabled.");
- } else {
- // cluster admins are expected to have mounted controllers in specific
- // locations - we'll attempt to figure out mount points
+ // Cluster admins may have some subsystems mounted in specific locations
+ // We'll attempt to figure out mount points. We do this even if we plan
+ // to mount cgroups into our own tree to control the path permissions or
+ // to mount subsystems that are not mounted previously.
+ // The subsystems for new and existing mount points have to match, and
+ // the same hierarchy will be mounted at each mount point with the same
+ // subsystem set.
+
+ Map<String, Set<String>> newMtab;
+ Map<CGroupController, String> cPaths;
+ try {
+ // parse mtab
+ newMtab = parseMtab(mtabFile);
- Map<CGroupController, String> cPaths =
- initializeControllerPathsFromMtab(mtabFile, this.cGroupPrefix);
- // we want to do a bulk update without the paths changing concurrently
- try {
- rwLock.writeLock().lock();
- controllerPaths = cPaths;
- } finally {
- rwLock.writeLock().unlock();
- }
+ // find cgroup controller paths
+ cPaths = initializeControllerPathsFromMtab(newMtab);
+ } catch (IOException e) {
+ LOG.warn("Failed to initialize controller paths! Exception: " + e);
+ throw new ResourceHandlerException(
+ "Failed to initialize controller paths!");
+ }
+
+ // we want to do a bulk update without the paths changing concurrently
+ try {
+ rwLock.writeLock().lock();
+ controllerPaths = cPaths;
+ parsedMtab = newMtab;
+ } finally {
+ rwLock.writeLock().unlock();
}
}
@VisibleForTesting
static Map<CGroupController, String> initializeControllerPathsFromMtab(
- String mtab, String cGroupPrefix) throws ResourceHandlerException {
- try {
- Map<String, List<String>> parsedMtab = parseMtab(mtab);
- Map<CGroupController, String> ret = new HashMap<>();
+ Map<String, Set<String>> parsedMtab)
+ throws ResourceHandlerException {
+ Map<CGroupController, String> ret = new HashMap<>();
- for (CGroupController controller : CGroupController.values()) {
- String subsystemName = controller.getName();
- String controllerPath = findControllerInMtab(subsystemName, parsedMtab);
+ for (CGroupController controller : CGroupController.values()) {
+ String subsystemName = controller.getName();
+ String controllerPath = findControllerInMtab(subsystemName, parsedMtab);
- if (controllerPath != null) {
- ret.put(controller, controllerPath);
- } else {
- LOG.warn("Controller not mounted but automount disabled: " +
- subsystemName);
- }
+ if (controllerPath != null) {
+ ret.put(controller, controllerPath);
}
- return ret;
- } catch (IOException e) {
- LOG.warn("Failed to initialize controller paths! Exception: " + e);
- throw new ResourceHandlerException(
- "Failed to initialize controller paths!");
}
+ return ret;
}
/* We are looking for entries of the form:
@@ -190,10 +198,15 @@ class CGroupsHandlerImpl implements CGroupsHandler {
* for mounts with type "cgroup". Cgroup controllers will
* appear in the list of options for a path.
*/
- private static Map<String, List<String>> parseMtab(String mtab)
+ @VisibleForTesting
+ static Map<String, Set<String>> parseMtab(String mtab)
throws IOException {
- Map<String, List<String>> ret = new HashMap<String, List<String>>();
+ Map<String, Set<String>> ret = new HashMap<>();
BufferedReader in = null;
+ HashSet<String> validCgroups = new HashSet<>();
+ for (CGroupController controller : CGroupController.values()) {
+ validCgroups.add(controller.getName());
+ }
try {
FileInputStream fis = new FileInputStream(new File(mtab));
@@ -209,13 +222,21 @@ class CGroupsHandlerImpl implements CGroupsHandler {
String options = m.group(3);
if (type.equals(CGROUPS_FSTYPE)) {
- List<String> value = Arrays.asList(options.split(","));
- ret.put(path, value);
+ Set<String> cgroupList =
+ new HashSet<>(Arrays.asList(options.split(",")));
+ // Collect the valid subsystem names
+ cgroupList.retainAll(validCgroups);
+ ret.put(path, cgroupList);
}
}
}
} catch (IOException e) {
- throw new IOException("Error while reading " + mtab, e);
+ if (Shell.LINUX) {
+ throw new IOException("Error while reading " + mtab, e);
+ } else {
+ // Ignore the error, if we are running on an os other than Linux
+ LOG.warn("Error while reading " + mtab, e);
+ }
} finally {
IOUtils.cleanup(LOG, in);
}
@@ -234,8 +255,8 @@ class CGroupsHandlerImpl implements CGroupsHandler {
*/
@VisibleForTesting
static String findControllerInMtab(String controller,
- Map<String, List<String>> entries) {
- for (Map.Entry<String, List<String>> e : entries.entrySet()) {
+ Map<String, Set<String>> entries) {
+ for (Map.Entry<String, Set<String>> e : entries.entrySet()) {
if (e.getValue().contains(controller)) {
if (new File(e.getKey()).canRead()) {
return e.getKey();
@@ -251,31 +272,45 @@ class CGroupsHandlerImpl implements CGroupsHandler {
private void mountCGroupController(CGroupController controller)
throws ResourceHandlerException {
- String path = getControllerPath(controller);
+ if (cGroupMountPath == null) {
+ throw new ResourceHandlerException(
+ String.format("Cgroups mount path not specified in %s.",
+ YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH));
+ }
+ String existingMountPath = getControllerPath(controller);
+ String requestedMountPath =
+ new File(cGroupMountPath, controller.getName()).getAbsolutePath();
- if (path == null) {
+ if (existingMountPath == null ||
+ !requestedMountPath.equals(existingMountPath)) {
try {
//lock out other readers/writers till we are done
rwLock.writeLock().lock();
- String hierarchy = cGroupPrefix;
- StringBuffer controllerPath = new StringBuffer()
- .append(cGroupMountPath).append('/').append(controller.getName());
- StringBuffer cGroupKV = new StringBuffer()
- .append(controller.getName()).append('=').append(controllerPath);
+ // If the controller was already mounted we have to mount it
+ // with the same options to clone the mount point otherwise
+ // the operation will fail
+ String mountOptions;
+ if (existingMountPath != null) {
+ mountOptions = Joiner.on(',')
+ .join(parsedMtab.get(existingMountPath));
+ } else {
+ mountOptions = controller.getName();
+ }
+
+ String cGroupKV =
+ mountOptions + "=" + requestedMountPath;
PrivilegedOperation.OperationType opType = PrivilegedOperation
.OperationType.MOUNT_CGROUPS;
PrivilegedOperation op = new PrivilegedOperation(opType);
- op.appendArgs(hierarchy, cGroupKV.toString());
+ op.appendArgs(cGroupPrefix, cGroupKV);
LOG.info("Mounting controller " + controller.getName() + " at " +
- controllerPath);
+ requestedMountPath);
privilegedOperationExecutor.executePrivilegedOperation(op, false);
//if privileged operation succeeds, update controller paths
- controllerPaths.put(controller, controllerPath.toString());
-
- return;
+ controllerPaths.put(controller, requestedMountPath);
} catch (PrivilegedOperationException e) {
LOG.error("Failed to mount controller: " + controller.getName());
throw new ResourceHandlerException("Failed to mount controller: "
@@ -284,37 +319,34 @@ class CGroupsHandlerImpl implements CGroupsHandler {
rwLock.writeLock().unlock();
}
} else {
- LOG.info("CGroup controller already mounted at: " + path);
- return;
+ LOG.info("CGroup controller already mounted at: " + existingMountPath);
}
}
@Override
public String getRelativePathForCGroup(String cGroupId) {
- return new StringBuffer(cGroupPrefix).append("/")
- .append(cGroupId).toString();
+ return cGroupPrefix + Path.SEPARATOR + cGroupId;
}
@Override
public String getPathForCGroup(CGroupController controller, String cGroupId) {
- return new StringBuffer(getControllerPath(controller))
- .append('/').append(cGroupPrefix).append("/")
- .append(cGroupId).toString();
+ return getControllerPath(controller) + Path.SEPARATOR + cGroupPrefix
+ + Path.SEPARATOR + cGroupId;
}
@Override
public String getPathForCGroupTasks(CGroupController controller,
String cGroupId) {
- return new StringBuffer(getPathForCGroup(controller, cGroupId))
- .append('/').append(CGROUP_FILE_TASKS).toString();
+ return getPathForCGroup(controller, cGroupId)
+ + Path.SEPARATOR + CGROUP_FILE_TASKS;
}
@Override
public String getPathForCGroupParam(CGroupController controller,
String cGroupId, String param) {
- return new StringBuffer(getPathForCGroup(controller, cGroupId))
- .append('/').append(controller.getName()).append('.')
- .append(param).toString();
+ return getPathForCGroup(controller, cGroupId)
+ + Path.SEPARATOR + controller.getName()
+ + "." + param;
}
/**
@@ -330,10 +362,21 @@ class CGroupsHandlerImpl implements CGroupsHandler {
// We have a controller that needs to be mounted
mountCGroupController(controller);
} else {
- // We are working with a pre-mounted contoller
- // Make sure that Yarn cgroup hierarchy path exists
- initializePreMountedCGroupController(controller);
+ String controllerPath = getControllerPath(controller);
+
+ if (controllerPath == null) {
+ throw new ResourceHandlerException(
+ String.format("Controller %s not mounted."
+ + " You either need to mount it with %s"
+ + " or mount cgroups before launching Yarn",
+ controller.getName(), YarnConfiguration.
+ NM_LINUX_CONTAINER_CGROUPS_MOUNT));
+ }
}
+
+ // We are working with a pre-mounted contoller
+ // Make sure that Yarn cgroup hierarchy path exists
+ initializePreMountedCGroupController(controller);
}
/**
@@ -347,11 +390,22 @@ class CGroupsHandlerImpl implements CGroupsHandler {
* @throws ResourceHandlerException yarn hierarchy cannot be created or
* accessed for any reason
*/
- public void initializePreMountedCGroupController(CGroupController controller)
+ private void initializePreMountedCGroupController(CGroupController controller)
throws ResourceHandlerException {
// Check permissions to cgroup hierarchy and
// create YARN cgroup if it does not exist, yet
- File rootHierarchy = new File(getControllerPath(controller));
+ String controllerPath = getControllerPath(controller);
+
+ if (controllerPath == null) {
+ throw new ResourceHandlerException(
+ String.format("Controller %s not mounted."
+ + " You either need to mount it with %s"
+ + " or mount cgroups before launching Yarn",
+ controller.getName(), YarnConfiguration.
+ NM_LINUX_CONTAINER_CGROUPS_MOUNT));
+ }
+
+ File rootHierarchy = new File(controllerPath);
File yarnHierarchy = new File(rootHierarchy, cGroupPrefix);
String subsystemName = controller.getName();
@@ -403,17 +457,9 @@ class CGroupsHandlerImpl implements CGroupsHandler {
String errorMessage,
String subsystemName,
String yarnCgroupPath) {
- return new StringBuilder()
- .append(errorMessage)
- .append(" Subsystem:")
- .append(subsystemName)
- .append(" Mount points:")
- .append(mtabFile)
- .append(" User:")
- .append(System.getProperty("user.name"))
- .append(" Path: ")
- .append(yarnCgroupPath)
- .toString();
+ return String.format("%s Subsystem:%s Mount points:%s User:%s Path:%s ",
+ errorMessage, subsystemName, mtabFile, System.getProperty("user.name"),
+ yarnCgroupPath);
}
@Override
@@ -456,7 +502,7 @@ class CGroupsHandlerImpl implements CGroupsHandler {
* @param cgf object referring to the cgroup to be deleted
* @return Boolean indicating whether cgroup was deleted
*/
- boolean checkAndDeleteCgroup(File cgf) throws InterruptedException {
+ private boolean checkAndDeleteCgroup(File cgf) throws InterruptedException {
boolean deleted = false;
// FileInputStream in = null;
try (FileInputStream in = new FileInputStream(cgf + "/tasks")) {
@@ -504,8 +550,8 @@ class CGroupsHandlerImpl implements CGroupsHandler {
} while (!deleted && (clock.getTime() - start) < deleteCGroupTimeout);
if (!deleted) {
- LOG.warn("Unable to delete " + cGroupPath +
- ", tried to delete for " + deleteCGroupTimeout + "ms");
+ LOG.warn(String.format("Unable to delete %s, tried to delete for %d ms",
+ cGroupPath, deleteCGroupTimeout));
}
}
@@ -517,8 +563,8 @@ class CGroupsHandlerImpl implements CGroupsHandler {
if (LOG.isDebugEnabled()) {
LOG.debug(
- "updateCGroupParam for path: " + cGroupParamPath + " with value " +
- value);
+ String.format("updateCGroupParam for path: %s with value %s",
+ cGroupParamPath, value));
}
try {
@@ -527,22 +573,22 @@ class CGroupsHandlerImpl implements CGroupsHandler {
pw = new PrintWriter(w);
pw.write(value);
} catch (IOException e) {
- throw new ResourceHandlerException(new StringBuffer("Unable to write to ")
- .append(cGroupParamPath).append(" with value: ").append(value)
- .toString(), e);
+ throw new ResourceHandlerException(
+ String.format("Unable to write to %s with value: %s",
+ cGroupParamPath, value), e);
} finally {
if (pw != null) {
boolean hasError = pw.checkError();
pw.close();
if (hasError) {
throw new ResourceHandlerException(
- new StringBuffer("Unable to write to ")
- .append(cGroupParamPath).append(" with value: ").append(value)
- .toString());
+ String.format("PrintWriter unable to write to %s with value: %s",
+ cGroupParamPath, value));
}
if (pw.checkError()) {
- throw new ResourceHandlerException("Error while closing cgroup file" +
- " " + cGroupParamPath);
+ throw new ResourceHandlerException(
+ String.format("Error while closing cgroup file %s",
+ cGroupParamPath));
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2f68049/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
index cb4dcf6..bca4fdc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
@@ -28,17 +28,18 @@ import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.Writer;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
-import java.util.List;
+import java.util.HashSet;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Sets;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -262,7 +263,7 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
/**
* If tasks file is empty, delete the cgroup.
*
- * @param file object referring to the cgroup to be deleted
+ * @param cgf object referring to the cgroup to be deleted
* @return Boolean indicating whether cgroup was deleted
*/
@VisibleForTesting
@@ -396,8 +397,8 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
* for mounts with type "cgroup". Cgroup controllers will
* appear in the list of options for a path.
*/
- private Map<String, List<String>> parseMtab() throws IOException {
- Map<String, List<String>> ret = new HashMap<String, List<String>>();
+ private Map<String, Set<String>> parseMtab() throws IOException {
+ Map<String, Set<String>> ret = new HashMap<String, Set<String>>();
BufferedReader in = null;
try {
@@ -414,7 +415,7 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
String options = m.group(3);
if (type.equals(CGROUPS_FSTYPE)) {
- List<String> value = Arrays.asList(options.split(","));
+ HashSet<String> value = Sets.newHashSet(options.split(","));
ret.put(path, value);
}
}
@@ -430,8 +431,8 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
@VisibleForTesting
String findControllerInMtab(String controller,
- Map<String, List<String>> entries) {
- for (Entry<String, List<String>> e : entries.entrySet()) {
+ Map<String, Set<String>> entries) {
+ for (Entry<String, Set<String>> e : entries.entrySet()) {
if (e.getValue().contains(controller)) {
if (new File(e.getKey()).canRead()) {
return e.getKey();
@@ -447,7 +448,7 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
private void initializeControllerPaths() throws IOException {
String controllerPath;
- Map<String, List<String>> parsedMtab = parseMtab();
+ Map<String, Set<String>> parsedMtab = parseMtab();
// CPU
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2f68049/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index bf08035..f11272e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -2050,17 +2050,31 @@ int mount_cgroup(const char *pair, const char *hierarchy) {
fprintf(LOGFILE, "Failed to mount cgroup controller, not supported\n");
return -1;
#else
- char *controller = malloc(strlen(pair));
- char *mount_path = malloc(strlen(pair));
+ size_t len = strlen(pair);
+ char *controller = malloc(len);
+ char *mount_path = malloc(len);
char hier_path[EXECUTOR_PATH_MAX];
int result = 0;
+ struct stat sb;
- if (get_kv_key(pair, controller, strlen(pair)) < 0 ||
- get_kv_value(pair, mount_path, strlen(pair)) < 0) {
+ if (controller == NULL || mount_path == NULL) {
+ fprintf(LOGFILE, "Failed to mount cgroup controller; not enough memory\n");
+ result = OUT_OF_MEMORY;
+ }
+ if (get_kv_key(pair, controller, len) < 0 ||
+ get_kv_value(pair, mount_path, len) < 0) {
fprintf(LOGFILE, "Failed to mount cgroup controller; invalid option: %s\n",
pair);
result = -1;
} else {
+ if (stat(mount_path, &sb) != 0) {
+ // Create mount point, if it does not exist
+ const mode_t mount_perms = S_IRWXU | S_IRGRP | S_IXGRP;
+ if (mkdirs(mount_path, mount_perms) == 0) {
+ fprintf(LOGFILE, "Failed to create cgroup mount point %s at %s\n",
+ controller, mount_path);
+ }
+ }
if (mount("none", mount_path, "cgroup", 0, controller) == 0) {
char *buf = stpncpy(hier_path, mount_path, strlen(mount_path));
*buf++ = '/';
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2f68049/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
index 804f8e7..dd8e338 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
@@ -25,6 +25,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException;
@@ -42,10 +43,11 @@ import java.nio.file.Files;
import java.security.Permission;
import java.util.Collections;
import java.util.LinkedHashMap;
-import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.UUID;
+import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
@@ -60,7 +62,6 @@ public class TestCGroupsHandlerImpl {
LogFactory.getLog(TestCGroupsHandlerImpl.class);
private PrivilegedOperationExecutor privilegedOperationExecutorMock;
- private Configuration conf;
private String tmpPath;
private String hierarchy;
private CGroupsHandler.CGroupController controller;
@@ -69,36 +70,143 @@ public class TestCGroupsHandlerImpl {
@Before
public void setup() {
privilegedOperationExecutorMock = mock(PrivilegedOperationExecutor.class);
- conf = new YarnConfiguration();
+
+ // Prepare test directory
tmpPath = System.getProperty("test.build.data") + "/cgroups";
+ File tmpDir = new File(tmpPath);
+ FileUtils.deleteQuietly(tmpDir);
+ assertTrue(tmpDir.mkdirs());
+
//no leading or trailing slashes here
hierarchy = "test-hadoop-yarn";
+ // Sample subsystem. Not used by all the tests
+ controller = CGroupsHandler.CGroupController.NET_CLS;
+ controllerPath =
+ new File(new File(tmpPath, controller.getName()), hierarchy)
+ .getAbsolutePath();
+ }
+
+ @After
+ public void teardown() {
+ FileUtil.fullyDelete(new File(tmpPath));
+ }
+
+ /**
+ * Security manager simulating access denied.
+ */
+ private class MockSecurityManagerDenyWrite extends SecurityManager {
+ @Override
+ public void checkPermission(Permission perm) {
+ if(perm.getActions().equals("write")) {
+ throw new SecurityException("Mock not allowed");
+ }
+ }
+ }
+
+ /**
+ * Create configuration to mount cgroups that do not exist.
+ * @return configuration object
+ */
+ private YarnConfiguration createMountConfiguration() {
+ YarnConfiguration conf = new YarnConfiguration();
conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY, hierarchy);
conf.setBoolean(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT, true);
conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH, tmpPath);
- controller = CGroupsHandler.CGroupController.NET_CLS;
- controllerPath = new StringBuffer(tmpPath).append('/')
- .append(controller.getName()).append('/').append(hierarchy).toString();
+ return conf;
+ }
+
+ /**
+ * Create configuration where the cgroups are premounted.
+ * @param myHierarchy Yarn cgroup
+ * @return configuration object
+ */
+ private Configuration createNoMountConfiguration(String myHierarchy) {
+ Configuration confNoMount = new Configuration();
+ confNoMount.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY,
+ myHierarchy);
+ confNoMount.setBoolean(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT,
+ false);
+ return confNoMount;
+ }
+
+ /**
+ * Create an empty mtab file. No cgroups are premounted
+ * @return mtab file
+ * @throws IOException could not create file
+ */
+ private File createEmptyCgroups() throws IOException {
+ File emptyMtab = new File(tmpPath, "mtab");
+ assertTrue("New file should have been created", emptyMtab.createNewFile());
+ return emptyMtab;
+ }
+
+ /**
+ * Create simulated cgroups mount point.
+ * @param parentDir cgroups mount point
+ * @param cpuAcct simulate newer Linux behavior by mounting cpu with cpuacct
+ * @return simulated mtab file location
+ * @throws IOException mtab file was not created
+ */
+ public static File createPremountedCgroups(File parentDir, boolean cpuAcct)
+ throws IOException {
+ // Mark an empty directory called 'cp' cgroup. It is processed before 'cpu'
+ String cpuMtabContentMissing =
+ "none " + parentDir.getAbsolutePath()
+ + "/cp cgroup rw,relatime,cpu 0 0\n";
+
+ File cpuCgroup = new File(parentDir, "cpu");
+ String cpuMtabContent =
+ "none " + cpuCgroup.getAbsolutePath()
+ + " cgroup rw,relatime,cpu"
+ + (cpuAcct ? ",cpuacct" :"")
+ + " 0 0\n";
+ assertTrue("Directory should be created", cpuCgroup.mkdirs());
+
+ File blkioCgroup = new File(parentDir, "blkio");
+ String blkioMtabContent =
+ "none " + blkioCgroup.getAbsolutePath()
+ + " cgroup rw,relatime,blkio 0 0\n";
+ assertTrue("Directory should be created", blkioCgroup.mkdirs());
+
+ File mockMtab = new File(parentDir, UUID.randomUUID().toString());
+ if (!mockMtab.exists()) {
+ if (!mockMtab.createNewFile()) {
+ String message = "Could not create file " + mockMtab.getAbsolutePath();
+ throw new IOException(message);
+ }
+ }
+ FileWriter mtabWriter = new FileWriter(mockMtab.getAbsoluteFile());
+ mtabWriter.write(cpuMtabContentMissing);
+ mtabWriter.write(cpuMtabContent);
+ mtabWriter.write(blkioMtabContent);
+ mtabWriter.close();
+ mockMtab.deleteOnExit();
+ return mockMtab;
}
@Test
- public void testMountController() {
- CGroupsHandler cGroupsHandler = null;
+ public void testMountController() throws IOException {
+ File parentDir = new File(tmpPath);
+ File cgroup = new File(parentDir, controller.getName());
+ assertTrue("cgroup dir should be cerated", cgroup.mkdirs());
//Since we enabled (deferred) cgroup controller mounting, no interactions
//should have occurred, with this mock
verifyZeroInteractions(privilegedOperationExecutorMock);
+ File emptyMtab = createEmptyCgroups();
try {
- cGroupsHandler = new CGroupsHandlerImpl(conf,
- privilegedOperationExecutorMock);
+ CGroupsHandler cGroupsHandler = new CGroupsHandlerImpl(
+ createMountConfiguration(),
+ privilegedOperationExecutorMock,
+ emptyMtab.getAbsolutePath());
PrivilegedOperation expectedOp = new PrivilegedOperation(
PrivilegedOperation.OperationType.MOUNT_CGROUPS);
//This is expected to be of the form :
//net_cls=<mount_path>/net_cls
- StringBuffer controllerKV = new StringBuffer(controller.getName())
- .append('=').append(tmpPath).append('/').append(controller.getName());
- expectedOp.appendArgs(hierarchy, controllerKV.toString());
+ String controllerKV = controller.getName() + "=" + tmpPath
+ + Path.SEPARATOR + controller.getName();
+ expectedOp.appendArgs(hierarchy, controllerKV);
cGroupsHandler.initializeCGroupController(controller);
try {
@@ -117,79 +225,90 @@ public class TestCGroupsHandlerImpl {
verifyNoMoreInteractions(privilegedOperationExecutorMock);
} catch (PrivilegedOperationException e) {
LOG.error("Caught exception: " + e);
- Assert.assertTrue("Unexpected PrivilegedOperationException from mock!",
+ assertTrue("Unexpected PrivilegedOperationException from mock!",
false);
}
} catch (ResourceHandlerException e) {
LOG.error("Caught exception: " + e);
- Assert.assertTrue("Unexpected ResourceHandler Exception!", false);
+ assertTrue("Unexpected ResourceHandler Exception!", false);
}
}
@Test
- public void testCGroupPaths() {
+ public void testCGroupPaths() throws IOException {
//As per junit behavior, we expect a new mock object to be available
//in this test.
verifyZeroInteractions(privilegedOperationExecutorMock);
CGroupsHandler cGroupsHandler = null;
+ File mtab = createEmptyCgroups();
+
+ // Lets manually create a path to (partially) simulate a controller mounted
+ // later in the test. This is required because the handler uses a mocked
+ // privileged operation executor
+ assertTrue("Sample subsystem should be created",
+ new File(controllerPath).mkdirs());
+
try {
- cGroupsHandler = new CGroupsHandlerImpl(conf,
- privilegedOperationExecutorMock);
+ cGroupsHandler = new CGroupsHandlerImpl(createMountConfiguration(),
+ privilegedOperationExecutorMock, mtab.getAbsolutePath());
cGroupsHandler.initializeCGroupController(controller);
} catch (ResourceHandlerException e) {
LOG.error("Caught exception: " + e);
- Assert.assertTrue(
+ assertTrue(
"Unexpected ResourceHandlerException when mounting controller!",
false);
}
String testCGroup = "container_01";
- String expectedPath = new StringBuffer(controllerPath).append('/')
- .append(testCGroup).toString();
+ String expectedPath =
+ controllerPath + Path.SEPARATOR + testCGroup;
String path = cGroupsHandler.getPathForCGroup(controller, testCGroup);
Assert.assertEquals(expectedPath, path);
- String expectedPathTasks = new StringBuffer(expectedPath).append('/')
- .append(CGroupsHandler.CGROUP_FILE_TASKS).toString();
+ String expectedPathTasks = expectedPath + Path.SEPARATOR
+ + CGroupsHandler.CGROUP_FILE_TASKS;
path = cGroupsHandler.getPathForCGroupTasks(controller, testCGroup);
Assert.assertEquals(expectedPathTasks, path);
String param = CGroupsHandler.CGROUP_PARAM_CLASSID;
- String expectedPathParam = new StringBuffer(expectedPath).append('/')
- .append(controller.getName()).append('.').append(param).toString();
+ String expectedPathParam = expectedPath + Path.SEPARATOR
+ + controller.getName() + "." + param;
path = cGroupsHandler.getPathForCGroupParam(controller, testCGroup, param);
Assert.assertEquals(expectedPathParam, path);
}
@Test
- public void testCGroupOperations() {
+ public void testCGroupOperations() throws IOException {
//As per junit behavior, we expect a new mock object to be available
//in this test.
verifyZeroInteractions(privilegedOperationExecutorMock);
CGroupsHandler cGroupsHandler = null;
+ File mtab = createEmptyCgroups();
+
+ // Lets manually create a path to (partially) simulate a controller mounted
+ // later in the test. This is required because the handler uses a mocked
+ // privileged operation executor
+ assertTrue("Sample subsystem should be created",
+ new File(controllerPath).mkdirs());
try {
- cGroupsHandler = new CGroupsHandlerImpl(conf,
- privilegedOperationExecutorMock);
+ cGroupsHandler = new CGroupsHandlerImpl(createMountConfiguration(),
+ privilegedOperationExecutorMock, mtab.getAbsolutePath());
cGroupsHandler.initializeCGroupController(controller);
} catch (ResourceHandlerException e) {
LOG.error("Caught exception: " + e);
- Assert.assertTrue(
+ assertTrue(
"Unexpected ResourceHandlerException when mounting controller!",
false);
}
- //Lets manually create a path to (partially) simulate a mounted controller
- //this is required because the handler uses a mocked privileged operation
- //executor
- new File(controllerPath).mkdirs();
String testCGroup = "container_01";
- String expectedPath = new StringBuffer(controllerPath).append('/')
- .append(testCGroup).toString();
+ String expectedPath = controllerPath
+ + Path.SEPARATOR + testCGroup;
try {
String path = cGroupsHandler.createCGroup(controller, testCGroup);
- Assert.assertTrue(new File(expectedPath).exists());
+ assertTrue(new File(expectedPath).exists());
Assert.assertEquals(expectedPath, path);
//update param and read param tests.
@@ -202,11 +321,12 @@ public class TestCGroupsHandlerImpl {
cGroupsHandler
.updateCGroupParam(controller, testCGroup, param, paramValue);
- String paramPath = new StringBuffer(expectedPath).append('/')
- .append(controller.getName()).append('.').append(param).toString();
+ String paramPath = expectedPath
+ + Path.SEPARATOR + controller.getName()
+ + "." + param;
File paramFile = new File(paramPath);
- Assert.assertTrue(paramFile.exists());
+ assertTrue(paramFile.exists());
try {
Assert.assertEquals(paramValue, new String(Files.readAllBytes(
paramFile.toPath())));
@@ -233,52 +353,6 @@ public class TestCGroupsHandlerImpl {
}
}
- public static File createMockCgroupMount(File parentDir, String type)
- throws IOException {
- return createMockCgroupMount(parentDir, type, "hadoop-yarn");
- }
-
- private static File createMockCgroupMount(File parentDir, String type,
- String hierarchy) throws IOException {
- File cgroupMountDir =
- new File(parentDir.getAbsolutePath(), type + "/" + hierarchy);
- FileUtils.deleteQuietly(cgroupMountDir);
- if (!cgroupMountDir.mkdirs()) {
- String message =
- "Could not create dir " + cgroupMountDir.getAbsolutePath();
- throw new IOException(message);
- }
- return cgroupMountDir;
- }
-
- public static File createMockMTab(File parentDir) throws IOException {
- String cpuMtabContent =
- "none " + parentDir.getAbsolutePath()
- + "/cpu cgroup rw,relatime,cpu 0 0\n";
- // Mark an empty directory called 'cp' cgroup. It is processed before 'cpu'
- String cpuMtabContentMissing =
- "none " + parentDir.getAbsolutePath()
- + "/cp cgroup rw,relatime,cpu 0 0\n";
- String blkioMtabContent =
- "none " + parentDir.getAbsolutePath()
- + "/blkio cgroup rw,relatime,blkio 0 0\n";
-
- File mockMtab = new File(parentDir, UUID.randomUUID().toString());
- if (!mockMtab.exists()) {
- if (!mockMtab.createNewFile()) {
- String message = "Could not create file " + mockMtab.getAbsolutePath();
- throw new IOException(message);
- }
- }
- FileWriter mtabWriter = new FileWriter(mockMtab.getAbsoluteFile());
- mtabWriter.write(cpuMtabContentMissing);
- mtabWriter.write(cpuMtabContent);
- mtabWriter.write(blkioMtabContent);
- mtabWriter.close();
- mockMtab.deleteOnExit();
- return mockMtab;
- }
-
/**
* Tests whether mtab parsing works as expected with a valid hierarchy set.
* @throws Exception the test will fail
@@ -288,24 +362,20 @@ public class TestCGroupsHandlerImpl {
// Initialize mtab and cgroup dir
File parentDir = new File(tmpPath);
// create mock cgroup
- File cpuCgroupMountDir = createMockCgroupMount(parentDir, "cpu",
- hierarchy);
- Assert.assertTrue(cpuCgroupMountDir.exists());
- File blkioCgroupMountDir = createMockCgroupMount(parentDir,
- "blkio", hierarchy);
- Assert.assertTrue(blkioCgroupMountDir.exists());
- File mockMtabFile = createMockMTab(parentDir);
+ File mockMtabFile = createPremountedCgroups(parentDir, false);
// Run mtabs parsing
+ Map<String, Set<String>> newMtab =
+ CGroupsHandlerImpl.parseMtab(mockMtabFile.getAbsolutePath());
Map<CGroupsHandler.CGroupController, String> controllerPaths =
CGroupsHandlerImpl.initializeControllerPathsFromMtab(
- mockMtabFile.getAbsolutePath(), hierarchy);
+ newMtab);
// Verify
Assert.assertEquals(2, controllerPaths.size());
- Assert.assertTrue(controllerPaths
+ assertTrue(controllerPaths
.containsKey(CGroupsHandler.CGroupController.CPU));
- Assert.assertTrue(controllerPaths
+ assertTrue(controllerPaths
.containsKey(CGroupsHandler.CGroupController.BLKIO));
String cpuDir = controllerPaths.get(CGroupsHandler.CGroupController.CPU);
String blkioDir =
@@ -315,17 +385,6 @@ public class TestCGroupsHandlerImpl {
}
/**
- * Tests whether mtab parsing works as expected with an empty hierarchy set.
- * @throws Exception the test will fail
- */
- @Test
- public void testPreMountedController() throws Exception {
- testPreMountedControllerInitialization("hadoop-yarn");
- testPreMountedControllerInitialization("");
- testPreMountedControllerInitialization("/");
- }
-
- /**
* Tests whether mtab parsing works as expected with the specified hierarchy.
* @param myHierarchy path to local cgroup hierarchy
* @throws Exception the test will fail
@@ -334,64 +393,57 @@ public class TestCGroupsHandlerImpl {
throws Exception {
// Initialize mount point
File parentDir = new File(tmpPath);
- FileUtils.deleteQuietly(parentDir);
- Assert.assertTrue("Could not create dirs", parentDir.mkdirs());
- File mtab = createMockMTab(parentDir);
+ File mtab = createPremountedCgroups(parentDir, false);
File mountPoint = new File(parentDir, "cpu");
- File cpuCgroupMountDir = createMockCgroupMount(
- parentDir, "cpu", myHierarchy);
// Initialize Yarn classes
- Configuration confNoMount = new Configuration();
- confNoMount.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY,
- myHierarchy);
- confNoMount.setBoolean(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT,
- false);
+ Configuration confNoMount = createNoMountConfiguration(myHierarchy);
CGroupsHandlerImpl cGroupsHandler = new CGroupsHandlerImpl(confNoMount,
privilegedOperationExecutorMock, mtab.getAbsolutePath());
-
+ File cpuCgroupMountDir = new File(
+ cGroupsHandler.getPathForCGroup(CGroupsHandler.CGroupController.CPU,
+ ""));
// Test that a missing yarn hierarchy will be created automatically
if (!cpuCgroupMountDir.equals(mountPoint)) {
- Assert.assertTrue("Could not delete cgroups", cpuCgroupMountDir.delete());
- Assert.assertTrue("Directory should be deleted",
+ assertTrue("Directory should be deleted",
!cpuCgroupMountDir.exists());
}
cGroupsHandler.initializeCGroupController(
CGroupsHandler.CGroupController.CPU);
- Assert.assertTrue("Cgroups not writable", cpuCgroupMountDir.exists() &&
+ assertTrue("Cgroups not writable", cpuCgroupMountDir.exists() &&
cpuCgroupMountDir.canWrite());
// Test that an inaccessible yarn hierarchy results in an exception
- Assert.assertTrue(cpuCgroupMountDir.setWritable(false));
+ assertTrue(cpuCgroupMountDir.setWritable(false));
try {
cGroupsHandler.initializeCGroupController(
CGroupsHandler.CGroupController.CPU);
Assert.fail("An inaccessible path should result in an exception");
} catch (Exception e) {
- Assert.assertTrue("Unexpected exception " + e.getClass().toString(),
+ assertTrue("Unexpected exception " + e.getClass().toString(),
e instanceof ResourceHandlerException);
} finally {
- Assert.assertTrue("Could not revert writable permission",
+ assertTrue("Could not revert writable permission",
cpuCgroupMountDir.setWritable(true));
}
// Test that a non-accessible mount directory results in an exception
if (!cpuCgroupMountDir.equals(mountPoint)) {
- Assert.assertTrue("Could not delete cgroups", cpuCgroupMountDir.delete());
- Assert.assertTrue("Directory should be deleted",
+ assertTrue("Could not delete cgroups", cpuCgroupMountDir.delete());
+ assertTrue("Directory should be deleted",
!cpuCgroupMountDir.exists());
}
- Assert.assertTrue(mountPoint.setWritable(false));
+ assertTrue(mountPoint.setWritable(false));
try {
cGroupsHandler.initializeCGroupController(
CGroupsHandler.CGroupController.CPU);
Assert.fail("An inaccessible path should result in an exception");
} catch (Exception e) {
- Assert.assertTrue("Unexpected exception " + e.getClass().toString(),
+ assertTrue("Unexpected exception " + e.getClass().toString(),
e instanceof ResourceHandlerException);
} finally {
- Assert.assertTrue("Could not revert writable permission",
+ assertTrue("Could not revert writable permission",
mountPoint.setWritable(true));
}
@@ -399,7 +451,7 @@ public class TestCGroupsHandlerImpl {
if (!cpuCgroupMountDir.equals(mountPoint)) {
Assert.assertFalse("Could not delete cgroups",
cpuCgroupMountDir.delete());
- Assert.assertTrue("Directory should be deleted",
+ assertTrue("Directory should be deleted",
!cpuCgroupMountDir.exists());
SecurityManager manager = System.getSecurityManager();
System.setSecurityManager(new MockSecurityManagerDenyWrite());
@@ -408,7 +460,7 @@ public class TestCGroupsHandlerImpl {
CGroupsHandler.CGroupController.CPU);
Assert.fail("An inaccessible path should result in an exception");
} catch (Exception e) {
- Assert.assertTrue("Unexpected exception " + e.getClass().toString(),
+ assertTrue("Unexpected exception " + e.getClass().toString(),
e instanceof ResourceHandlerException);
} finally {
System.setSecurityManager(manager);
@@ -419,18 +471,18 @@ public class TestCGroupsHandlerImpl {
if (!cpuCgroupMountDir.equals(mountPoint)) {
Assert.assertFalse("Could not delete cgroups",
cpuCgroupMountDir.delete());
- Assert.assertTrue("Directory should be deleted",
+ assertTrue("Directory should be deleted",
!cpuCgroupMountDir.exists());
}
FileUtils.deleteQuietly(mountPoint);
- Assert.assertTrue("cgroups mount point should be deleted",
+ assertTrue("cgroups mount point should be deleted",
!mountPoint.exists());
try {
cGroupsHandler.initializeCGroupController(
CGroupsHandler.CGroupController.CPU);
Assert.fail("An inaccessible path should result in an exception");
} catch (Exception e) {
- Assert.assertTrue("Unexpected exception " + e.getClass().toString(),
+ assertTrue("Unexpected exception " + e.getClass().toString(),
e instanceof ResourceHandlerException);
}
}
@@ -442,19 +494,19 @@ public class TestCGroupsHandlerImpl {
File memory = new File(tmpPath, "memory");
try {
CGroupsHandlerImpl handler = new CGroupsHandlerImpl(
- conf,
+ createNoMountConfiguration(tmpPath),
privilegedOperationExecutorMock);
- Map<String, List<String>> cgroups = new LinkedHashMap<>();
+ Map<String, Set<String>> cgroups = new LinkedHashMap<>();
Assert.assertTrue("temp dir should be created", cpu.mkdirs());
Assert.assertTrue("temp dir should be created", memory.mkdirs());
Assert.assertFalse("temp dir should not be created", cpuNoExist.exists());
cgroups.put(
- memory.getAbsolutePath(), Collections.singletonList("memory"));
+ memory.getAbsolutePath(), Collections.singleton("memory"));
cgroups.put(
- cpuNoExist.getAbsolutePath(), Collections.singletonList("cpu"));
- cgroups.put(cpu.getAbsolutePath(), Collections.singletonList("cpu"));
+ cpuNoExist.getAbsolutePath(), Collections.singleton("cpu"));
+ cgroups.put(cpu.getAbsolutePath(), Collections.singleton("cpu"));
String selectedCPU = handler.findControllerInMtab("cpu", cgroups);
Assert.assertEquals("Wrong CPU mount point selected",
cpu.getAbsolutePath(), selectedCPU);
@@ -464,17 +516,61 @@ public class TestCGroupsHandlerImpl {
}
}
- @After
- public void teardown() {
- FileUtil.fullyDelete(new File(tmpPath));
+ /**
+ * Tests whether mtab parsing works as expected with an empty hierarchy set.
+ * @throws Exception the test will fail
+ */
+ @Test
+ public void testPreMountedControllerEmpty() throws Exception {
+ testPreMountedControllerInitialization("");
}
- private class MockSecurityManagerDenyWrite extends SecurityManager {
- @Override
- public void checkPermission(Permission perm) {
- if(perm.getActions().equals("write")) {
- throw new SecurityException("Mock not allowed");
- }
- }
+ /**
+ * Tests whether mtab parsing works as expected with a / hierarchy set.
+ * @throws Exception the test will fail
+ */
+ @Test
+ public void testPreMountedControllerRoot() throws Exception {
+ testPreMountedControllerInitialization("/");
+ }
+
+ /**
+ * Tests whether mtab parsing works as expected with the specified hierarchy.
+ * @throws Exception the test will fail
+ */
+ @Test
+ public void testRemount()
+ throws Exception {
+ // Initialize mount point
+ File parentDir = new File(tmpPath);
+
+ final String oldMountPointDir = "oldmount";
+ final String newMountPointDir = "newmount";
+
+ File oldMountPoint = new File(parentDir, oldMountPointDir);
+ File mtab = createPremountedCgroups(
+ oldMountPoint, true);
+
+ File newMountPoint = new File(parentDir, newMountPointDir);
+ assertTrue("Could not create dirs",
+ new File(newMountPoint, "cpu").mkdirs());
+
+ // Initialize Yarn classes
+ Configuration confMount = createMountConfiguration();
+ confMount.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH,
+ parentDir.getAbsolutePath() + Path.SEPARATOR + newMountPointDir);
+ CGroupsHandlerImpl cGroupsHandler = new CGroupsHandlerImpl(confMount,
+ privilegedOperationExecutorMock, mtab.getAbsolutePath());
+
+ cGroupsHandler.initializeCGroupController(
+ CGroupsHandler.CGroupController.CPU);
+
+ ArgumentCaptor<PrivilegedOperation> opCaptor = ArgumentCaptor.forClass(
+ PrivilegedOperation.class);
+ verify(privilegedOperationExecutorMock)
+ .executePrivilegedOperation(opCaptor.capture(), eq(false));
+ File hierarchyFile =
+ new File(new File(newMountPoint, "cpu"), this.hierarchy);
+ assertTrue("Yarn cgroup should exist", hierarchyFile.exists());
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2f68049/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java
index 69479d1..9da37dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java
@@ -43,9 +43,6 @@ public class TestResourceHandlerModule {
networkEnabledConf.setBoolean(YarnConfiguration.NM_NETWORK_RESOURCE_ENABLED,
true);
- //We need to bypass mtab parsing for figuring out cgroups mount locations
- networkEnabledConf.setBoolean(YarnConfiguration
- .NM_LINUX_CONTAINER_CGROUPS_MOUNT, true);
ResourceHandlerModule.nullifyResourceHandlerChain();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2f68049/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
index 83c8d5d..b562133 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
@@ -38,11 +38,12 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
+import java.util.Set;
import java.util.concurrent.CountDownLatch;
@Deprecated
public class TestCgroupsLCEResourcesHandler {
- static File cgroupDir = null;
+ private static File cgroupDir = null;
@Before
public void setUp() throws Exception {
@@ -132,9 +133,9 @@ public class TestCgroupsLCEResourcesHandler {
static class CustomCgroupsLCEResourceHandler extends
CgroupsLCEResourcesHandler {
- String mtabFile;
- int[] limits = new int[2];
- boolean generateLimitsMode = false;
+ private String mtabFile;
+ private int[] limits = new int[2];
+ private boolean generateLimitsMode = false;
@Override
int[] getOverallLimits(float x) {
@@ -154,6 +155,20 @@ public class TestCgroupsLCEResourcesHandler {
}
}
+ private static File createMockCgroupMount(File parentDir,
+ String type)
+ throws IOException {
+ File cgroupMountDir =
+ new File(parentDir.getAbsolutePath(), type + "/hadoop-yarn");
+ FileUtils.deleteQuietly(cgroupMountDir);
+ if (!cgroupMountDir.mkdirs()) {
+ String message =
+ "Could not create dir " + cgroupMountDir.getAbsolutePath();
+ throw new IOException(message);
+ }
+ return cgroupMountDir;
+ }
+
@Test
public void testInit() throws IOException {
LinuxContainerExecutor mockLCE = new MockLinuxContainerExecutor();
@@ -168,13 +183,14 @@ public class TestCgroupsLCEResourcesHandler {
handler.setConf(conf);
handler.initConfig();
+ // create mock mtab
+ File mockMtab =
+ TestCGroupsHandlerImpl.createPremountedCgroups(cgroupDir, false);
+
// create mock cgroup
- File cpuCgroupMountDir = TestCGroupsHandlerImpl.createMockCgroupMount(
+ File cpuCgroupMountDir = createMockCgroupMount(
cgroupDir, "cpu");
- // create mock mtab
- File mockMtab = TestCGroupsHandlerImpl.createMockMTab(cgroupDir);
-
// setup our handler and call init()
handler.setMtabFile(mockMtab.getAbsolutePath());
@@ -265,13 +281,14 @@ public class TestCgroupsLCEResourcesHandler {
handler.setConf(conf);
handler.initConfig();
+ // create mock mtab
+ File mockMtab =
+ TestCGroupsHandlerImpl.createPremountedCgroups(cgroupDir, false);
+
// create mock cgroup
- File cpuCgroupMountDir = TestCGroupsHandlerImpl.createMockCgroupMount(
+ File cpuCgroupMountDir = createMockCgroupMount(
cgroupDir, "cpu");
- // create mock mtab
- File mockMtab = TestCGroupsHandlerImpl.createMockMTab(cgroupDir);
-
// setup our handler and call init()
handler.setMtabFile(mockMtab.getAbsolutePath());
handler.init(mockLCE, plugin);
@@ -352,17 +369,17 @@ public class TestCgroupsLCEResourcesHandler {
File memory = new File(cgroupDir, "memory");
try {
CgroupsLCEResourcesHandler handler = new CgroupsLCEResourcesHandler();
- Map<String, List<String>> cgroups = new LinkedHashMap<>();
+ Map<String, Set<String>> cgroups = new LinkedHashMap<>();
Assert.assertTrue("temp dir should be created", cpu.mkdirs());
Assert.assertTrue("temp dir should be created", memory.mkdirs());
Assert.assertFalse("temp dir should not be created", cpuNoExist.exists());
cgroups.put(
- memory.getAbsolutePath(), Collections.singletonList("memory"));
+ memory.getAbsolutePath(), Collections.singleton("memory"));
cgroups.put(
- cpuNoExist.getAbsolutePath(), Collections.singletonList("cpu"));
- cgroups.put(cpu.getAbsolutePath(), Collections.singletonList("cpu"));
+ cpuNoExist.getAbsolutePath(), Collections.singleton("cpu"));
+ cgroups.put(cpu.getAbsolutePath(), Collections.singleton("cpu"));
String selectedCPU = handler.findControllerInMtab("cpu", cgroups);
Assert.assertEquals("Wrong CPU mount point selected",
cpu.getAbsolutePath(), selectedCPU);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: HDFS-10629. Federation Roter.
Contributed by Jason Kace and Inigo Goiri.
Posted by in...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d710513a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
new file mode 100644
index 0000000..ee6f57d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
@@ -0,0 +1,290 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.NamenodePriorityComparator;
+import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport;
+import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.util.Time;
+
+/**
+ * In-memory cache/mock of a namenode and file resolver. Stores the most
+ * recently updated NN information for each nameservice and block pool. Also
+ * stores a virtual mount table for resolving global namespace paths to local NN
+ * paths.
+ */
+public class MockResolver
+ implements ActiveNamenodeResolver, FileSubclusterResolver {
+
+ private Map<String, List<? extends FederationNamenodeContext>> resolver =
+ new HashMap<String, List<? extends FederationNamenodeContext>>();
+ private Map<String, List<RemoteLocation>> locations =
+ new HashMap<String, List<RemoteLocation>>();
+ private Set<FederationNamespaceInfo> namespaces =
+ new HashSet<FederationNamespaceInfo>();
+ private String defaultNamespace = null;
+
+ public MockResolver(Configuration conf, StateStoreService store) {
+ this.cleanRegistrations();
+ }
+
+ public void addLocation(String mount, String nameservice, String location) {
+ RemoteLocation remoteLocation = new RemoteLocation(nameservice, location);
+ List<RemoteLocation> locationsList = locations.get(mount);
+ if (locationsList == null) {
+ locationsList = new LinkedList<RemoteLocation>();
+ locations.put(mount, locationsList);
+ }
+ if (!locationsList.contains(remoteLocation)) {
+ locationsList.add(remoteLocation);
+ }
+
+ if (this.defaultNamespace == null) {
+ this.defaultNamespace = nameservice;
+ }
+ }
+
+ public synchronized void cleanRegistrations() {
+ this.resolver =
+ new HashMap<String, List<? extends FederationNamenodeContext>>();
+ this.namespaces = new HashSet<FederationNamespaceInfo>();
+ }
+
+ @Override
+ public void updateActiveNamenode(
+ String ns, InetSocketAddress successfulAddress) {
+
+ String address = successfulAddress.getHostName() + ":" +
+ successfulAddress.getPort();
+ String key = ns;
+ if (key != null) {
+ // Update the active entry
+ @SuppressWarnings("unchecked")
+ List<FederationNamenodeContext> iterator =
+ (List<FederationNamenodeContext>) resolver.get(key);
+ for (FederationNamenodeContext namenode : iterator) {
+ if (namenode.getRpcAddress().equals(address)) {
+ MockNamenodeContext nn = (MockNamenodeContext) namenode;
+ nn.setState(FederationNamenodeServiceState.ACTIVE);
+ break;
+ }
+ }
+ Collections.sort(iterator, new NamenodePriorityComparator());
+ }
+ }
+
+ @Override
+ public List<? extends FederationNamenodeContext>
+ getNamenodesForNameserviceId(String nameserviceId) {
+ return resolver.get(nameserviceId);
+ }
+
+ @Override
+ public List<? extends FederationNamenodeContext> getNamenodesForBlockPoolId(
+ String blockPoolId) {
+ return resolver.get(blockPoolId);
+ }
+
+ private static class MockNamenodeContext
+ implements FederationNamenodeContext {
+ private String webAddress;
+ private String rpcAddress;
+ private String serviceAddress;
+ private String lifelineAddress;
+ private String namenodeId;
+ private String nameserviceId;
+ private FederationNamenodeServiceState state;
+ private long dateModified;
+
+ MockNamenodeContext(
+ String rpc, String service, String lifeline, String web,
+ String ns, String nn, FederationNamenodeServiceState state) {
+ this.rpcAddress = rpc;
+ this.serviceAddress = service;
+ this.lifelineAddress = lifeline;
+ this.webAddress = web;
+ this.namenodeId = nn;
+ this.nameserviceId = ns;
+ this.state = state;
+ this.dateModified = Time.now();
+ }
+
+ public void setState(FederationNamenodeServiceState newState) {
+ this.state = newState;
+ this.dateModified = Time.now();
+ }
+
+ @Override
+ public String getRpcAddress() {
+ return rpcAddress;
+ }
+
+ @Override
+ public String getServiceAddress() {
+ return serviceAddress;
+ }
+
+ @Override
+ public String getLifelineAddress() {
+ return lifelineAddress;
+ }
+
+ @Override
+ public String getWebAddress() {
+ return webAddress;
+ }
+
+ @Override
+ public String getNamenodeKey() {
+ return nameserviceId + " " + namenodeId + " " + rpcAddress;
+ }
+
+ @Override
+ public String getNameserviceId() {
+ return nameserviceId;
+ }
+
+ @Override
+ public String getNamenodeId() {
+ return namenodeId;
+ }
+
+ @Override
+ public FederationNamenodeServiceState getState() {
+ return state;
+ }
+
+ @Override
+ public long getDateModified() {
+ return dateModified;
+ }
+ }
+
+ @Override
+ public synchronized boolean registerNamenode(NamenodeStatusReport report)
+ throws IOException {
+ MockNamenodeContext context = new MockNamenodeContext(
+ report.getRpcAddress(), report.getServiceAddress(),
+ report.getLifelineAddress(), report.getWebAddress(),
+ report.getNameserviceId(), report.getNamenodeId(), report.getState());
+
+ String nsId = report.getNameserviceId();
+ String bpId = report.getBlockPoolId();
+ String cId = report.getClusterId();
+ @SuppressWarnings("unchecked")
+ List<MockNamenodeContext> existingItems =
+ (List<MockNamenodeContext>) resolver.get(nsId);
+ if (existingItems == null) {
+ existingItems = new ArrayList<MockNamenodeContext>();
+ resolver.put(bpId, existingItems);
+ resolver.put(nsId, existingItems);
+ }
+ boolean added = false;
+ for (int i=0; i<existingItems.size() && !added; i++) {
+ MockNamenodeContext existing = existingItems.get(i);
+ if (existing.getNamenodeKey().equals(context.getNamenodeKey())) {
+ existingItems.set(i, context);
+ added = true;
+ }
+ }
+ if (!added) {
+ existingItems.add(context);
+ }
+ Collections.sort(existingItems, new NamenodePriorityComparator());
+
+ FederationNamespaceInfo info = new FederationNamespaceInfo(bpId, cId, nsId);
+ namespaces.add(info);
+ return true;
+ }
+
+ @Override
+ public Set<FederationNamespaceInfo> getNamespaces() throws IOException {
+ return this.namespaces;
+ }
+
+ @Override
+ public PathLocation getDestinationForPath(String path) throws IOException {
+ String finalPath = null;
+ String nameservice = null;
+ Set<String> namespaceSet = new HashSet<String>();
+ LinkedList<RemoteLocation> remoteLocations =
+ new LinkedList<RemoteLocation>();
+ for(String key : this.locations.keySet()) {
+ if(path.startsWith(key)) {
+ for (RemoteLocation location : this.locations.get(key)) {
+ finalPath = location.getDest() + path.substring(key.length());
+ nameservice = location.getNameserviceId();
+ RemoteLocation remoteLocation =
+ new RemoteLocation(nameservice, finalPath);
+ remoteLocations.add(remoteLocation);
+ namespaceSet.add(nameservice);
+ }
+ break;
+ }
+ }
+ if (remoteLocations.isEmpty()) {
+ // Path isn't supported, mimic resolver behavior.
+ return null;
+ }
+ return new PathLocation(path, remoteLocations, namespaceSet);
+ }
+
+ @Override
+ public List<String> getMountPoints(String path) throws IOException {
+ List<String> mounts = new ArrayList<String>();
+ if (path.equals("/")) {
+ // Mounts only supported under root level
+ for (String mount : this.locations.keySet()) {
+ if (mount.length() > 1) {
+ // Remove leading slash, this is the behavior of the mount tree,
+ // return only names.
+ mounts.add(mount.replace("/", ""));
+ }
+ }
+ }
+ return mounts;
+ }
+
+ @Override
+ public void setRouterId(String router) {
+ }
+
+ @Override
+ public String getDefaultNamespace() {
+ return defaultNamespace;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d710513a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
new file mode 100644
index 0000000..16d624c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Constructs a router configuration with individual features enabled/disabled.
+ */
+public class RouterConfigBuilder {
+
+ private Configuration conf;
+
+ public RouterConfigBuilder(Configuration configuration) {
+ this.conf = configuration;
+ }
+
+ public RouterConfigBuilder() {
+ this.conf = new Configuration();
+ }
+
+ public Configuration build() {
+ return conf;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d710513a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
new file mode 100644
index 0000000..55d04ad
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
@@ -0,0 +1,767 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology.NNConf;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology.NSConf;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport;
+import org.apache.hadoop.hdfs.server.federation.router.Router;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.Service.STATE;
+
+/**
+ * Test utility to mimic a federated HDFS cluster with a router.
+ */
+public class RouterDFSCluster {
+ /**
+ * Router context.
+ */
+ public class RouterContext {
+ private Router router;
+ private FileContext fileContext;
+ private String nameserviceId;
+ private String namenodeId;
+ private int rpcPort;
+ private DFSClient client;
+ private Configuration conf;
+ private URI fileSystemUri;
+
+ public RouterContext(Configuration conf, String ns, String nn)
+ throws URISyntaxException {
+ this.namenodeId = nn;
+ this.nameserviceId = ns;
+ this.conf = conf;
+ router = new Router();
+ router.init(conf);
+ }
+
+ public Router getRouter() {
+ return this.router;
+ }
+
+ public String getNameserviceId() {
+ return this.nameserviceId;
+ }
+
+ public String getNamenodeId() {
+ return this.namenodeId;
+ }
+
+ public int getRpcPort() {
+ return this.rpcPort;
+ }
+
+ public FileContext getFileContext() {
+ return this.fileContext;
+ }
+
+ public void initRouter() throws URISyntaxException {
+ }
+
+ public DistributedFileSystem getFileSystem() throws IOException {
+ DistributedFileSystem fs =
+ (DistributedFileSystem) DistributedFileSystem.get(conf);
+ return fs;
+ }
+
+ public DFSClient getClient(UserGroupInformation user)
+ throws IOException, URISyntaxException, InterruptedException {
+
+ LOG.info("Connecting to router at " + fileSystemUri);
+ return user.doAs(new PrivilegedExceptionAction<DFSClient>() {
+ @Override
+ public DFSClient run() throws IOException {
+ return new DFSClient(fileSystemUri, conf);
+ }
+ });
+ }
+
+ public DFSClient getClient() throws IOException, URISyntaxException {
+
+ if (client == null) {
+ LOG.info("Connecting to router at " + fileSystemUri);
+ client = new DFSClient(fileSystemUri, conf);
+ }
+ return client;
+ }
+ }
+
+ /**
+ * Namenode context.
+ */
+ public class NamenodeContext {
+ private NameNode namenode;
+ private String nameserviceId;
+ private String namenodeId;
+ private FileContext fileContext;
+ private int rpcPort;
+ private int servicePort;
+ private int lifelinePort;
+ private int httpPort;
+ private URI fileSystemUri;
+ private int index;
+ private Configuration conf;
+ private DFSClient client;
+
+ public NamenodeContext(Configuration conf, String ns, String nn,
+ int index) {
+ this.conf = conf;
+ this.namenodeId = nn;
+ this.nameserviceId = ns;
+ this.index = index;
+ }
+
+ public NameNode getNamenode() {
+ return this.namenode;
+ }
+
+ public String getNameserviceId() {
+ return this.nameserviceId;
+ }
+
+ public String getNamenodeId() {
+ return this.namenodeId;
+ }
+
+ public FileContext getFileContext() {
+ return this.fileContext;
+ }
+
+ public void setNamenode(NameNode n) throws URISyntaxException {
+ namenode = n;
+
+ // Store the bound ports and override the default FS with the local NN's
+ // RPC
+ rpcPort = n.getNameNodeAddress().getPort();
+ servicePort = n.getServiceRpcAddress().getPort();
+ lifelinePort = n.getServiceRpcAddress().getPort();
+ httpPort = n.getHttpAddress().getPort();
+ fileSystemUri = new URI("hdfs://" + namenode.getHostAndPort());
+ DistributedFileSystem.setDefaultUri(conf, fileSystemUri);
+
+ try {
+ this.fileContext = FileContext.getFileContext(conf);
+ } catch (UnsupportedFileSystemException e) {
+ this.fileContext = null;
+ }
+ }
+
+ public String getRpcAddress() {
+ return namenode.getNameNodeAddress().getHostName() + ":" + rpcPort;
+ }
+
+ public String getServiceAddress() {
+ return namenode.getServiceRpcAddress().getHostName() + ":" + servicePort;
+ }
+
+ public String getLifelineAddress() {
+ return namenode.getServiceRpcAddress().getHostName() + ":" + lifelinePort;
+ }
+
+ public String getHttpAddress() {
+ return namenode.getHttpAddress().getHostName() + ":" + httpPort;
+ }
+
+ public DistributedFileSystem getFileSystem() throws IOException {
+ DistributedFileSystem fs =
+ (DistributedFileSystem) DistributedFileSystem.get(conf);
+ return fs;
+ }
+
+ public void resetClient() {
+ client = null;
+ }
+
+ public DFSClient getClient(UserGroupInformation user)
+ throws IOException, URISyntaxException, InterruptedException {
+
+ LOG.info("Connecting to namenode at " + fileSystemUri);
+ return user.doAs(new PrivilegedExceptionAction<DFSClient>() {
+ @Override
+ public DFSClient run() throws IOException {
+ return new DFSClient(fileSystemUri, conf);
+ }
+ });
+ }
+
+ public DFSClient getClient() throws IOException, URISyntaxException {
+ if (client == null) {
+ LOG.info("Connecting to namenode at " + fileSystemUri);
+ client = new DFSClient(fileSystemUri, conf);
+ }
+ return client;
+ }
+
+ public String getConfSuffix() {
+ String suffix = nameserviceId;
+ if (highAvailability) {
+ suffix += "." + namenodeId;
+ }
+ return suffix;
+ }
+ }
+
+ public static final String NAMENODE1 = "nn0";
+ public static final String NAMENODE2 = "nn1";
+ public static final String NAMENODE3 = "nn2";
+ public static final String TEST_STRING = "teststring";
+ public static final String TEST_DIR = "testdir";
+ public static final String TEST_FILE = "testfile";
+
+ private List<String> nameservices;
+ private List<RouterContext> routers;
+ private List<NamenodeContext> namenodes;
+ private static final Log LOG = LogFactory.getLog(RouterDFSCluster.class);
+ private MiniDFSCluster cluster;
+ private boolean highAvailability;
+
+ protected static final int DEFAULT_HEARTBEAT_INTERVAL = 5;
+ protected static final int DEFAULT_CACHE_INTERVAL_SEC = 5;
+ private Configuration routerOverrides;
+ private Configuration namenodeOverrides;
+
+ private static final String NAMENODES = NAMENODE1 + "," + NAMENODE2;
+
+ public RouterDFSCluster(boolean ha, int numNameservices) {
+ this(ha, numNameservices, 2);
+ }
+
+ public RouterDFSCluster(boolean ha, int numNameservices, int numNamenodes) {
+ this.highAvailability = ha;
+ configureNameservices(numNameservices, numNamenodes);
+ }
+
+ public void addRouterOverrides(Configuration conf) {
+ if (this.routerOverrides == null) {
+ this.routerOverrides = conf;
+ } else {
+ this.routerOverrides.addResource(conf);
+ }
+ }
+
+ public void addNamenodeOverrides(Configuration conf) {
+ if (this.namenodeOverrides == null) {
+ this.namenodeOverrides = conf;
+ } else {
+ this.namenodeOverrides.addResource(conf);
+ }
+ }
+
+ public Configuration generateNamenodeConfiguration(
+ String defaultNameserviceId) {
+ Configuration c = new HdfsConfiguration();
+
+ c.set(DFSConfigKeys.DFS_NAMESERVICES, getNameservicesKey());
+ c.set("fs.defaultFS", "hdfs://" + defaultNameserviceId);
+
+ for (String ns : nameservices) {
+ if (highAvailability) {
+ c.set(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "." + ns, NAMENODES);
+ }
+
+ for (NamenodeContext context : getNamenodes(ns)) {
+ String suffix = context.getConfSuffix();
+
+ c.set(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + suffix,
+ "127.0.0.1:" + context.rpcPort);
+ c.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY + "." + suffix,
+ "127.0.0.1:" + context.httpPort);
+ c.set(DFSConfigKeys.DFS_NAMENODE_RPC_BIND_HOST_KEY + "." + suffix,
+ "0.0.0.0");
+ }
+ }
+
+ if (namenodeOverrides != null) {
+ c.addResource(namenodeOverrides);
+ }
+ return c;
+ }
+
+ public Configuration generateClientConfiguration() {
+ Configuration conf = new HdfsConfiguration();
+ conf.addResource(generateNamenodeConfiguration(getNameservices().get(0)));
+ return conf;
+ }
+
+ public Configuration generateRouterConfiguration(String localNameserviceId,
+ String localNamenodeId) throws IOException {
+ Configuration conf = new HdfsConfiguration();
+ conf.addResource(generateNamenodeConfiguration(localNameserviceId));
+
+ // Use mock resolver classes
+ conf.set(DFSConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS,
+ MockResolver.class.getCanonicalName());
+ conf.set(DFSConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
+ MockResolver.class.getCanonicalName());
+
+ // Set the nameservice ID for the default NN monitor
+ conf.set(DFSConfigKeys.DFS_NAMESERVICE_ID, localNameserviceId);
+
+ if (localNamenodeId != null) {
+ conf.set(DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY, localNamenodeId);
+ }
+
+ StringBuilder routerBuilder = new StringBuilder();
+ for (String ns : nameservices) {
+ for (NamenodeContext context : getNamenodes(ns)) {
+ String suffix = context.getConfSuffix();
+
+ if (routerBuilder.length() != 0) {
+ routerBuilder.append(",");
+ }
+ routerBuilder.append(suffix);
+ }
+ }
+
+ return conf;
+ }
+
+ public void configureNameservices(int numNameservices, int numNamenodes) {
+ nameservices = new ArrayList<String>();
+ for (int i = 0; i < numNameservices; i++) {
+ nameservices.add("ns" + i);
+ }
+ namenodes = new ArrayList<NamenodeContext>();
+ int index = 0;
+ for (String ns : nameservices) {
+ Configuration nnConf = generateNamenodeConfiguration(ns);
+ if (highAvailability) {
+ NamenodeContext context =
+ new NamenodeContext(nnConf, ns, NAMENODE1, index);
+ namenodes.add(context);
+ index++;
+
+ if (numNamenodes > 1) {
+ context = new NamenodeContext(nnConf, ns, NAMENODE2, index + 1);
+ namenodes.add(context);
+ index++;
+ }
+
+ if (numNamenodes > 2) {
+ context = new NamenodeContext(nnConf, ns, NAMENODE3, index + 1);
+ namenodes.add(context);
+ index++;
+ }
+
+ } else {
+ NamenodeContext context = new NamenodeContext(nnConf, ns, null, index);
+ namenodes.add(context);
+ index++;
+ }
+ }
+ }
+
+ public String getNameservicesKey() {
+ StringBuilder ns = new StringBuilder();
+ for (int i = 0; i < nameservices.size(); i++) {
+ if (i > 0) {
+ ns.append(",");
+ }
+ ns.append(nameservices.get(i));
+ }
+ return ns.toString();
+ }
+
+ public String getRandomNameservice() {
+ Random r = new Random();
+ return nameservices.get(r.nextInt(nameservices.size()));
+ }
+
+ public List<String> getNameservices() {
+ return nameservices;
+ }
+
+ public List<NamenodeContext> getNamenodes(String nameservice) {
+ ArrayList<NamenodeContext> nns = new ArrayList<NamenodeContext>();
+ for (NamenodeContext c : namenodes) {
+ if (c.nameserviceId.equals(nameservice)) {
+ nns.add(c);
+ }
+ }
+ return nns;
+ }
+
+ public NamenodeContext getRandomNamenode() {
+ Random rand = new Random();
+ return namenodes.get(rand.nextInt(namenodes.size()));
+ }
+
+ public List<NamenodeContext> getNamenodes() {
+ return namenodes;
+ }
+
+ public boolean isHighAvailability() {
+ return highAvailability;
+ }
+
+ public NamenodeContext getNamenode(String nameservice,
+ String namenode) {
+ for (NamenodeContext c : namenodes) {
+ if (c.nameserviceId.equals(nameservice)) {
+ if (namenode == null || c.namenodeId == null || namenode.isEmpty()
+ || c.namenodeId.isEmpty()) {
+ return c;
+ } else if (c.namenodeId.equals(namenode)) {
+ return c;
+ }
+ }
+ }
+ return null;
+ }
+
+ public List<RouterContext> getRouters(String nameservice) {
+ ArrayList<RouterContext> nns = new ArrayList<RouterContext>();
+ for (RouterContext c : routers) {
+ if (c.nameserviceId.equals(nameservice)) {
+ nns.add(c);
+ }
+ }
+ return nns;
+ }
+
+ public RouterContext getRouterContext(String nameservice,
+ String namenode) {
+ for (RouterContext c : routers) {
+ if (namenode == null) {
+ return c;
+ }
+ if (c.namenodeId.equals(namenode)
+ && c.nameserviceId.equals(nameservice)) {
+ return c;
+ }
+ }
+ return null;
+ }
+
+ public RouterContext getRandomRouter() {
+ Random rand = new Random();
+ return routers.get(rand.nextInt(routers.size()));
+ }
+
+ public List<RouterContext> getRouters() {
+ return routers;
+ }
+
+ public RouterContext buildRouter(String nameservice, String namenode)
+ throws URISyntaxException, IOException {
+ Configuration config = generateRouterConfiguration(nameservice, namenode);
+ RouterContext rc = new RouterContext(config, nameservice, namenode);
+ return rc;
+ }
+
+ public void startCluster() {
+ startCluster(null);
+ }
+
+ public void startCluster(Configuration overrideConf) {
+ try {
+ MiniDFSNNTopology topology = new MiniDFSNNTopology();
+ for (String ns : nameservices) {
+
+ NSConf conf = new MiniDFSNNTopology.NSConf(ns);
+ if (highAvailability) {
+ for(int i = 0; i < namenodes.size()/nameservices.size(); i++) {
+ NNConf nnConf = new MiniDFSNNTopology.NNConf("nn" + i);
+ conf.addNN(nnConf);
+ }
+ } else {
+ NNConf nnConf = new MiniDFSNNTopology.NNConf(null);
+ conf.addNN(nnConf);
+ }
+ topology.addNameservice(conf);
+ }
+ topology.setFederation(true);
+
+ // Start mini DFS cluster
+ Configuration nnConf = generateNamenodeConfiguration(nameservices.get(0));
+ if (overrideConf != null) {
+ nnConf.addResource(overrideConf);
+ }
+ cluster = new MiniDFSCluster.Builder(nnConf).nnTopology(topology).build();
+ cluster.waitActive();
+
+ // Store NN pointers
+ for (int i = 0; i < namenodes.size(); i++) {
+ NameNode nn = cluster.getNameNode(i);
+ namenodes.get(i).setNamenode(nn);
+ }
+
+ } catch (Exception e) {
+ LOG.error("Cannot start Router DFS cluster: " + e.getMessage(), e);
+ cluster.shutdown();
+ }
+ }
+
+ public void startRouters()
+ throws InterruptedException, URISyntaxException, IOException {
+ // Create routers
+ routers = new ArrayList<RouterContext>();
+ for (String ns : nameservices) {
+
+ for (NamenodeContext context : getNamenodes(ns)) {
+ routers.add(buildRouter(ns, context.namenodeId));
+ }
+ }
+
+ // Start all routers
+ for (RouterContext router : routers) {
+ router.router.start();
+ }
+ // Wait until all routers are active and record their ports
+ for (RouterContext router : routers) {
+ waitActive(router);
+ router.initRouter();
+ }
+ }
+
+ public void waitActive(NamenodeContext nn) throws IOException {
+ cluster.waitActive(nn.index);
+ }
+
+ public void waitActive(RouterContext router)
+ throws InterruptedException {
+ for (int loopCount = 0; loopCount < 20; loopCount++) {
+ // Validate connection of routers to NNs
+ if (router.router.getServiceState() == STATE.STARTED) {
+ return;
+ }
+ Thread.sleep(1000);
+ }
+ assertFalse(
+ "Timeout waiting for " + router.router.toString() + " to activate.",
+ true);
+ }
+
+
+ public void registerNamenodes() throws IOException {
+ for (RouterContext r : routers) {
+ ActiveNamenodeResolver resolver = r.router.getNamenodeResolver();
+ for (NamenodeContext nn : namenodes) {
+ // Generate a report
+ NamenodeStatusReport report = new NamenodeStatusReport(nn.nameserviceId,
+ nn.namenodeId, nn.getRpcAddress(), nn.getServiceAddress(),
+ nn.getLifelineAddress(), nn.getHttpAddress());
+ report.setNamespaceInfo(nn.namenode.getNamesystem().getFSImage()
+ .getStorage().getNamespaceInfo());
+
+ // Determine HA state from nn public state string
+ String nnState = nn.namenode.getState();
+ HAServiceState haState = HAServiceState.ACTIVE;
+ for (HAServiceState state : HAServiceState.values()) {
+ if (nnState.equalsIgnoreCase(state.name())) {
+ haState = state;
+ break;
+ }
+ }
+ report.setHAServiceState(haState);
+
+ // Register with the resolver
+ resolver.registerNamenode(report);
+ }
+ }
+ }
+
+ public void waitNamenodeRegistration()
+ throws InterruptedException, IllegalStateException, IOException {
+ for (RouterContext r : routers) {
+ for (NamenodeContext nn : namenodes) {
+ FederationTestUtils.waitNamenodeRegistered(
+ r.router.getNamenodeResolver(), nn.nameserviceId, nn.namenodeId,
+ null);
+ }
+ }
+ }
+
+ public void waitRouterRegistrationQuorum(RouterContext router,
+ FederationNamenodeServiceState state, String nameservice, String namenode)
+ throws InterruptedException, IOException {
+ LOG.info("Waiting for NN - " + nameservice + ":" + namenode
+ + " to transition to state - " + state);
+ FederationTestUtils.waitNamenodeRegistered(
+ router.router.getNamenodeResolver(), nameservice, namenode, state);
+ }
+
+ public String getFederatedPathForNameservice(String ns) {
+ return "/" + ns;
+ }
+
+ public String getNamenodePathForNameservice(String ns) {
+ return "/target-" + ns;
+ }
+
+ /**
+ * @return example:
+ * <ul>
+ * <li>/ns0/testdir which maps to ns0->/target-ns0/testdir
+ * </ul>
+ */
+ public String getFederatedTestDirectoryForNameservice(String ns) {
+ return getFederatedPathForNameservice(ns) + "/" + TEST_DIR;
+ }
+
+ /**
+ * @return example:
+ * <ul>
+ * <li>/target-ns0/testdir
+ * </ul>
+ */
+ public String getNamenodeTestDirectoryForNameservice(String ns) {
+ return getNamenodePathForNameservice(ns) + "/" + TEST_DIR;
+ }
+
+ /**
+ * @return example:
+ * <ul>
+ * <li>/ns0/testfile which maps to ns0->/target-ns0/testfile
+ * </ul>
+ */
+ public String getFederatedTestFileForNameservice(String ns) {
+ return getFederatedPathForNameservice(ns) + "/" + TEST_FILE;
+ }
+
+ /**
+ * @return example:
+ * <ul>
+ * <li>/target-ns0/testfile
+ * </ul>
+ */
+ public String getNamenodeTestFileForNameservice(String ns) {
+ return getNamenodePathForNameservice(ns) + "/" + TEST_FILE;
+ }
+
+ public void shutdown() {
+ cluster.shutdown();
+ if (routers != null) {
+ for (RouterContext context : routers) {
+ stopRouter(context);
+ }
+ }
+ }
+
+ public void stopRouter(RouterContext router) {
+ try {
+
+ router.router.shutDown();
+
+ int loopCount = 0;
+ while (router.router.getServiceState() != STATE.STOPPED) {
+ loopCount++;
+ Thread.sleep(1000);
+ if (loopCount > 20) {
+ LOG.error("Unable to shutdown router - " + router.rpcPort);
+ break;
+ }
+ }
+ } catch (InterruptedException e) {
+ }
+ }
+
+ /////////////////////////////////////////////////////////////////////////////
+ // Namespace Test Fixtures
+ /////////////////////////////////////////////////////////////////////////////
+
+ /**
+ * Creates test directories via the namenode.
+ * 1) /target-ns0/testfile
+ * 2) /target-ns1/testfile
+ * @throws IOException
+ */
+ public void createTestDirectoriesNamenode() throws IOException {
+ // Add a test dir to each NS and verify
+ for (String ns : getNameservices()) {
+ NamenodeContext context = getNamenode(ns, null);
+ if (!createTestDirectoriesNamenode(context)) {
+ throw new IOException("Unable to create test directory for ns - " + ns);
+ }
+ }
+ }
+
+ public boolean createTestDirectoriesNamenode(NamenodeContext nn)
+ throws IOException {
+ return FederationTestUtils.addDirectory(nn.getFileSystem(),
+ getNamenodeTestDirectoryForNameservice(nn.nameserviceId));
+ }
+
+ public void deleteAllFiles() throws IOException {
+ // Delete all files via the NNs and verify
+ for (NamenodeContext context : getNamenodes()) {
+ FileStatus[] status = context.getFileSystem().listStatus(new Path("/"));
+ for(int i = 0; i <status.length; i++) {
+ Path p = status[i].getPath();
+ context.getFileSystem().delete(p, true);
+ }
+ status = context.getFileSystem().listStatus(new Path("/"));
+ assertEquals(status.length, 0);
+ }
+ }
+
+ /////////////////////////////////////////////////////////////////////////////
+ // MockRouterResolver Test Fixtures
+ /////////////////////////////////////////////////////////////////////////////
+
+ /**
+ * <ul>
+ * <li>/ -> [ns0->/].
+ * <li>/nso -> ns0->/target-ns0.
+ * <li>/ns1 -> ns1->/target-ns1.
+ * </ul>
+ */
+ public void installMockLocations() {
+ for (RouterContext r : routers) {
+ MockResolver resolver =
+ (MockResolver) r.router.getSubclusterResolver();
+ // create table entries
+ for (String ns : nameservices) {
+ // Direct path
+ resolver.addLocation(getFederatedPathForNameservice(ns), ns,
+ getNamenodePathForNameservice(ns));
+ }
+
+ // Root path goes to both NS1
+ resolver.addLocation("/", nameservices.get(0), "/");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d710513a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
new file mode 100644
index 0000000..8c720c7
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.MockResolver;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.service.Service.STATE;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * The the safe mode for the {@link Router} controlled by
+ * {@link SafeModeTimer}.
+ */
+public class TestRouter {
+
+ private static Configuration conf;
+
+ @BeforeClass
+ public static void create() throws IOException {
+ // Basic configuration without the state store
+ conf = new Configuration();
+ // Mock resolver classes
+ conf.set(DFSConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS,
+ MockResolver.class.getCanonicalName());
+ conf.set(DFSConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
+ MockResolver.class.getCanonicalName());
+
+ // Simulate a co-located NN
+ conf.set(DFSConfigKeys.DFS_NAMESERVICES, "ns0");
+ conf.set("fs.defaultFS", "hdfs://" + "ns0");
+ conf.set(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + "ns0",
+ "127.0.0.1:0" + 0);
+ conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY + "." + "ns0",
+ "127.0.0.1:" + 0);
+ conf.set(DFSConfigKeys.DFS_NAMENODE_RPC_BIND_HOST_KEY + "." + "ns0",
+ "0.0.0.0");
+ }
+
+ @AfterClass
+ public static void destroy() {
+ }
+
+ @Before
+ public void setup() throws IOException, URISyntaxException {
+ }
+
+ @After
+ public void cleanup() {
+ }
+
+ private static void testRouterStartup(Configuration routerConfig)
+ throws InterruptedException, IOException {
+ Router router = new Router();
+ assertEquals(STATE.NOTINITED, router.getServiceState());
+ router.init(routerConfig);
+ assertEquals(STATE.INITED, router.getServiceState());
+ router.start();
+ assertEquals(STATE.STARTED, router.getServiceState());
+ router.stop();
+ assertEquals(STATE.STOPPED, router.getServiceState());
+ router.close();
+ }
+
+ @Test
+ public void testRouterService() throws InterruptedException, IOException {
+
+ // Run with all services
+ testRouterStartup((new RouterConfigBuilder(conf)).build());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: HDFS-11745. Increase HDFS test
timeouts from 1 second to 10 seconds. Contributed by Eric Badger
Posted by in...@apache.org.
HDFS-11745. Increase HDFS test timeouts from 1 second to 10 seconds. Contributed by Eric Badger
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fab73748
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fab73748
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fab73748
Branch: refs/heads/HDFS-10467
Commit: fab73748a1ce982904d332e6ddd46169f186ad54
Parents: 74a6143
Author: Jason Lowe <jl...@yahoo-inc.com>
Authored: Wed May 10 13:02:31 2017 -0500
Committer: Jason Lowe <jl...@yahoo-inc.com>
Committed: Wed May 10 13:02:31 2017 -0500
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java | 2 +-
.../org/apache/hadoop/fs/TestSymlinkHdfsFileContext.java | 2 +-
.../java/org/apache/hadoop/fs/TestSymlinkHdfsFileSystem.java | 4 ++--
.../src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java | 2 +-
.../hdfs/server/namenode/metrics/TestNameNodeMetrics.java | 2 +-
.../java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java | 8 ++++----
6 files changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fab73748/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java
index 0fabb30..f308763 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java
@@ -749,7 +749,7 @@ public class TestRpcProgramNfs3 {
assertEquals("Incorrect COMMIT3Response:", null, response2);
}
- @Test(timeout=1000)
+ @Test(timeout=10000)
public void testIdempotent() {
Object[][] procedures = {
{ Nfs3Constant.NFSPROC3.NULL, 1 },
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fab73748/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsFileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsFileContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsFileContext.java
index d94a514..2a3e4c3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsFileContext.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsFileContext.java
@@ -34,7 +34,7 @@ public class TestSymlinkHdfsFileContext extends TestSymlinkHdfs {
wrapper = new FileContextTestWrapper(fc, "/tmp/TestSymlinkHdfsFileContext");
}
- @Test(timeout=1000)
+ @Test(timeout=10000)
/** Test access a symlink using AbstractFileSystem */
public void testAccessLinkFromAbstractFileSystem() throws IOException {
Path file = new Path(testBaseDir1(), "file");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fab73748/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsFileSystem.java
index bf42e24..fba9c42 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsFileSystem.java
@@ -35,13 +35,13 @@ public class TestSymlinkHdfsFileSystem extends TestSymlinkHdfs {
@Override
@Ignore("FileSystem adds missing authority in absolute URIs")
- @Test(timeout=1000)
+ @Test(timeout=10000)
public void testCreateWithPartQualPathFails() throws IOException {}
@Ignore("FileSystem#create creates parent directories," +
" so dangling links to directories are created")
@Override
- @Test(timeout=1000)
+ @Test(timeout=10000)
public void testCreateFileViaDanglingLinkParent() throws IOException {}
// Additional tests for DFS-only methods
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fab73748/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
index 14ad6dd..f811d3d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
@@ -860,7 +860,7 @@ public class TestDFSUtil {
DFSUtil.getSpnegoKeytabKey(conf, defaultKey));
}
- @Test(timeout=1000)
+ @Test(timeout=10000)
public void testDurationToString() throws Exception {
assertEquals("000:00:00:00.000", DFSUtil.durationToString(0));
assertEquals("001:01:01:01.000",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fab73748/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
index b9d25b2..3974d0b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
@@ -157,7 +157,7 @@ public class TestNameNodeMetrics {
* Test that capacity metrics are exported and pass
* basic sanity tests.
*/
- @Test (timeout = 1800)
+ @Test (timeout = 10000)
public void testCapacityMetrics() throws Exception {
MetricsRecordBuilder rb = getMetrics(NS_METRICS);
long capacityTotal = MetricsAsserts.getLongGauge("CapacityTotal", rb);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fab73748/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java
index 24de8ac..1862f76 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java
@@ -130,7 +130,7 @@ public class TestWebHdfsTokens {
verify(fs, never()).setDelegationToken(any());
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testGetOpRequireAuth() {
for (HttpOpParam.Op op : GetOpParam.Op.values()) {
boolean expect = (op == GetOpParam.Op.GETDELEGATIONTOKEN);
@@ -138,7 +138,7 @@ public class TestWebHdfsTokens {
}
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testPutOpRequireAuth() {
for (HttpOpParam.Op op : PutOpParam.Op.values()) {
boolean expect = (op == PutOpParam.Op.RENEWDELEGATIONTOKEN || op == PutOpParam.Op.CANCELDELEGATIONTOKEN);
@@ -146,14 +146,14 @@ public class TestWebHdfsTokens {
}
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testPostOpRequireAuth() {
for (HttpOpParam.Op op : PostOpParam.Op.values()) {
assertFalse(op.getRequireAuth());
}
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testDeleteOpRequireAuth() {
for (HttpOpParam.Op op : DeleteOpParam.Op.values()) {
assertFalse(op.getRequireAuth());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: HADOOP-14390. Correct spelling of
'succeed' and variants. Contributed by Dongtao Zhang
Posted by in...@apache.org.
HADOOP-14390. Correct spelling of 'succeed' and variants. Contributed by Dongtao Zhang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e4f34ecb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e4f34ecb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e4f34ecb
Branch: refs/heads/HDFS-10467
Commit: e4f34ecb049a252fb1084c4c7f404d710b221969
Parents: a3954cc
Author: Chris Douglas <cd...@apache.org>
Authored: Fri May 5 12:10:50 2017 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Fri May 5 12:10:50 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java | 6 +++---
.../test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java | 2 +-
.../org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java | 4 ++--
.../src/test/java/org/apache/hadoop/hdfs/NNBenchWithoutMR.java | 6 +++---
.../reservation/planning/TestAlignedPlanner.java | 4 ++--
5 files changed, 11 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f34ecb/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
index 61295b4..68a7560 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
@@ -251,7 +251,7 @@ abstract public class ViewFileSystemBaseTest {
fsTarget.isFile(new Path(targetTestRoot,"user/foo")));
// Delete the created file
- Assert.assertTrue("Delete should suceed",
+ Assert.assertTrue("Delete should succeed",
fsView.delete(new Path("/user/foo"), false));
Assert.assertFalse("File should not exist after delete",
fsView.exists(new Path("/user/foo")));
@@ -266,7 +266,7 @@ abstract public class ViewFileSystemBaseTest {
fsTarget.isFile(new Path(targetTestRoot,"dir2/foo")));
// Delete the created file
- Assert.assertTrue("Delete should suceed",
+ Assert.assertTrue("Delete should succeed",
fsView.delete(new Path("/internalDir/linkToDir2/foo"), false));
Assert.assertFalse("File should not exist after delete",
fsView.exists(new Path("/internalDir/linkToDir2/foo")));
@@ -370,7 +370,7 @@ abstract public class ViewFileSystemBaseTest {
public void testRenameAcrossMounts1() throws IOException {
fileSystemTestHelper.createFile(fsView, "/user/foo");
fsView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar"));
- /* - code if we had wanted this to suceed
+ /* - code if we had wanted this to succeed
Assert.assertFalse(fSys.exists(new Path("/user/foo")));
Assert.assertFalse(fSysLocal.exists(new Path(targetTestRoot,"user/foo")));
Assert.assertTrue(fSys.isFile(FileSystemTestHelper.getTestRootPath(fSys,"/user2/fooBarBar")));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f34ecb/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
index 50237d1..fdc6389 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
@@ -232,7 +232,7 @@ abstract public class ViewFsBaseTest {
isFile(fcTarget, new Path(targetTestRoot,"dir2/foo")));
// Delete the created file
- Assert.assertTrue("Delete should suceed",
+ Assert.assertTrue("Delete should succeed",
fcView.delete(new Path("/internalDir/linkToDir2/foo"),false));
Assert.assertFalse("File should not exist after deletion",
exists(fcView, new Path("/internalDir/linkToDir2/foo")));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f34ecb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
index e643d21..8b48225 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
@@ -509,8 +509,8 @@ public class TestCheckpoint {
Mockito.reset(faultInjector);
secondary.shutdown(); // secondary namenode crash!
- // start new instance of secondary and verify that
- // a new rollEditLog suceedes inspite of the fact that
+ // start new instance of secondary and verify that
+ // a new rollEditLog succeeds inspite of the fact that
// edits.new already exists.
//
secondary = startSecondaryNameNode(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f34ecb/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBenchWithoutMR.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBenchWithoutMR.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBenchWithoutMR.java
index 8cd6f36..9b63010 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBenchWithoutMR.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBenchWithoutMR.java
@@ -93,7 +93,7 @@ public class NNBenchWithoutMR {
/**
* Create and write to a given number of files. Repeat each remote
- * operation until is suceeds (does not throw an exception).
+ * operation until is succeeds (does not throw an exception).
*
* @return the number of exceptions caught
*/
@@ -178,7 +178,7 @@ public class NNBenchWithoutMR {
/**
* Rename a given number of files. Repeat each remote
- * operation until is suceeds (does not throw an exception).
+ * operation until is succeeds (does not throw an exception).
*
* @return the number of exceptions caught
*/
@@ -208,7 +208,7 @@ public class NNBenchWithoutMR {
/**
* Delete a given number of files. Repeat each remote
- * operation until is suceeds (does not throw an exception).
+ * operation until is succeeds (does not throw an exception).
*
* @return the number of exceptions caught
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f34ecb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java
index fd187fc..25ec9c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java
@@ -866,7 +866,7 @@ public class TestAlignedPlanner {
curAlloc.clear();
curAlloc.put(new ReservationInterval(9 * step, 10 * step),
Resource.newInstance(1024, 1));
- assertTrue("validateOrderNoFap() should have suceeded",
+ assertTrue("validateOrderNoFap() should have succeeded",
IterativePlanner.validateOrderNoGap(allocation, curAlloc, false));
// 2. allocateLeft = false, fail when curAlloc has a gap
@@ -888,7 +888,7 @@ public class TestAlignedPlanner {
curAlloc.clear();
curAlloc.put(new ReservationInterval(13 * step, 14 * step),
Resource.newInstance(1024, 1));
- assertTrue("validateOrderNoFap() should have suceeded",
+ assertTrue("validateOrderNoFap() should have succeeded",
IterativePlanner.validateOrderNoGap(allocation, curAlloc, true));
// 5. allocateLeft = true, fail when there is a gap between curAlloc and
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: YARN-3839. Quit throwing
NMNotYetReadyException. Contributed by Manikandan R
Posted by in...@apache.org.
YARN-3839. Quit throwing NMNotYetReadyException. Contributed by Manikandan R
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/424887ec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/424887ec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/424887ec
Branch: refs/heads/HDFS-10467
Commit: 424887ecb7d11a72837f2757ed3ff9e0fe8c5b5d
Parents: cef2815
Author: Jason Lowe <jl...@yahoo-inc.com>
Authored: Mon May 8 17:14:37 2017 -0500
Committer: Jason Lowe <jl...@yahoo-inc.com>
Committed: Mon May 8 17:14:37 2017 -0500
----------------------------------------------------------------------
.../yarn/api/ContainerManagementProtocol.java | 8 +-
.../apache/hadoop/yarn/client/ServerProxy.java | 5 +
.../yarn/server/nodemanager/NodeManager.java | 2 -
.../nodemanager/NodeStatusUpdaterImpl.java | 2 -
.../containermanager/ContainerManager.java | 2 -
.../containermanager/ContainerManagerImpl.java | 28 ----
.../nodemanager/DummyContainerManager.java | 5 -
.../nodemanager/TestNodeManagerResync.java | 164 -------------------
.../BaseContainerManagerTest.java | 13 +-
.../containermanager/TestContainerManager.java | 5 -
.../TestContainerManagerRecovery.java | 11 --
.../containermanager/TestNMProxy.java | 30 +---
.../TestContainerSchedulerQueuing.java | 5 -
.../server/TestContainerManagerSecurity.java | 14 +-
14 files changed, 21 insertions(+), 273 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/424887ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java
index 9077d3b..10708a0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java
@@ -24,10 +24,10 @@ import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.protocolrecords.CommitResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerResponse;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest;
@@ -45,7 +45,6 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.exceptions.NMNotYetReadyException;
import org.apache.hadoop.yarn.exceptions.YarnException;
/**
@@ -101,9 +100,6 @@ public interface ContainerManagementProtocol {
* a allServicesMetaData map.
* @throws YarnException
* @throws IOException
- * @throws NMNotYetReadyException
- * This exception is thrown when NM starts from scratch but has not
- * yet connected with RM.
*/
@Public
@Stable
http://git-wip-us.apache.org/repos/asf/hadoop/blob/424887ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java
index e42435f..6188d6a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java
@@ -80,6 +80,11 @@ public class ServerProxy {
exceptionToPolicyMap.put(ConnectTimeoutException.class, retryPolicy);
exceptionToPolicyMap.put(RetriableException.class, retryPolicy);
exceptionToPolicyMap.put(SocketException.class, retryPolicy);
+
+ /*
+ * Still keeping this to cover case like newer client talking
+ * to an older version of server
+ */
exceptionToPolicyMap.put(NMNotYetReadyException.class, retryPolicy);
return RetryPolicies.retryByException(RetryPolicies.TRY_ONCE_THEN_FAIL,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/424887ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index 55e7b09..3c0e498 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -460,8 +460,6 @@ public class NodeManager extends CompositeService
@Override
public void run() {
try {
- LOG.info("Notifying ContainerManager to block new container-requests");
- containerManager.setBlockNewContainerRequests(true);
if (!rmWorkPreservingRestartEnabled) {
LOG.info("Cleaning up running containers on resync");
containerManager.cleanupContainersOnNMResync();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/424887ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index 4914dc7..dd5b279 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -428,8 +428,6 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
.verifyRMRegistrationResponseForNodeLabels(regNMResponse));
LOG.info(successfullRegistrationMsg);
- LOG.info("Notifying ContainerManager to unblock new container-requests");
- this.context.getContainerManager().setBlockNewContainerRequests(false);
}
private List<ApplicationId> createKeepAliveApplicationList() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/424887ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManager.java
index 066d987..2aeb245 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManager.java
@@ -42,8 +42,6 @@ public interface ContainerManager extends ServiceStateChangeListener,
void updateQueuingLimit(ContainerQueuingLimit queuingLimit);
- void setBlockNewContainerRequests(boolean blockNewContainerRequests);
-
ContainerScheduler getContainerScheduler();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/424887ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index d82c728..f65f1ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -166,7 +166,6 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
@@ -204,7 +203,6 @@ public class ContainerManagerImpl extends CompositeService implements
protected final AsyncDispatcher dispatcher;
private final DeletionService deletionService;
- private AtomicBoolean blockNewContainerRequests = new AtomicBoolean(false);
private boolean serviceStopped = false;
private final ReadLock readLock;
private final WriteLock writeLock;
@@ -550,10 +548,6 @@ public class ContainerManagerImpl extends CompositeService implements
refreshServiceAcls(conf, new NMPolicyProvider());
}
- LOG.info("Blocking new container-requests as container manager rpc" +
- " server is still starting.");
- this.setBlockNewContainerRequests(true);
-
String bindHost = conf.get(YarnConfiguration.NM_BIND_HOST);
String nmAddress = conf.getTrimmed(YarnConfiguration.NM_ADDRESS);
String hostOverride = null;
@@ -617,7 +611,6 @@ public class ContainerManagerImpl extends CompositeService implements
@Override
public void serviceStop() throws Exception {
- setBlockNewContainerRequests(true);
this.writeLock.lock();
try {
serviceStopped = true;
@@ -852,11 +845,6 @@ public class ContainerManagerImpl extends CompositeService implements
@Override
public StartContainersResponse startContainers(
StartContainersRequest requests) throws YarnException, IOException {
- if (blockNewContainerRequests.get()) {
- throw new NMNotYetReadyException(
- "Rejecting new containers as NodeManager has not"
- + " yet connected with ResourceManager");
- }
UserGroupInformation remoteUgi = getRemoteUgi();
NMTokenIdentifier nmTokenIdentifier = selectNMTokenIdentifier(remoteUgi);
authorizeUser(remoteUgi, nmTokenIdentifier);
@@ -1113,11 +1101,6 @@ public class ContainerManagerImpl extends CompositeService implements
public IncreaseContainersResourceResponse increaseContainersResource(
IncreaseContainersResourceRequest requests)
throws YarnException, IOException {
- if (blockNewContainerRequests.get()) {
- throw new NMNotYetReadyException(
- "Rejecting container resource increase as NodeManager has not"
- + " yet connected with ResourceManager");
- }
UserGroupInformation remoteUgi = getRemoteUgi();
NMTokenIdentifier nmTokenIdentifier = selectNMTokenIdentifier(remoteUgi);
authorizeUser(remoteUgi, nmTokenIdentifier);
@@ -1560,17 +1543,6 @@ public class ContainerManagerImpl extends CompositeService implements
}
@Override
- public void setBlockNewContainerRequests(boolean blockNewContainerRequests) {
- this.blockNewContainerRequests.set(blockNewContainerRequests);
- }
-
- @Private
- @VisibleForTesting
- public boolean getBlockNewContainerRequestsStatus() {
- return this.blockNewContainerRequests.get();
- }
-
- @Override
public void stateChanged(Service service) {
// TODO Auto-generated method stub
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/424887ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
index feb6dd6..e520a31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
@@ -191,11 +191,6 @@ public class DummyContainerManager extends ContainerManagerImpl {
}
@Override
- public void setBlockNewContainerRequests(boolean blockNewContainerRequests) {
- // do nothing
- }
-
- @Override
protected void authorizeStartAndResourceIncreaseRequest(
NMTokenIdentifier nmTokenIdentifier,
ContainerTokenIdentifier containerTokenIdentifier,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/424887ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
index 5ab5c37..c5c74d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
@@ -65,7 +65,6 @@ import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.exceptions.NMNotYetReadyException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -87,7 +86,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -188,34 +186,6 @@ public class TestNodeManagerResync {
}
}
- // This test tests new container requests are blocked when NM starts from
- // scratch until it register with RM AND while NM is resyncing with RM
- @SuppressWarnings("unchecked")
- @Test(timeout=60000)
- public void testBlockNewContainerRequestsOnStartAndResync()
- throws IOException, InterruptedException, YarnException {
- NodeManager nm = new TestNodeManager2();
- int port = ServerSocketUtil.getPort(49154, 10);
- YarnConfiguration conf = createNMConfig(port);
- conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, false);
- nm.init(conf);
- nm.start();
-
- // Start the container in running state
- ContainerId cId = TestNodeManagerShutdown.createContainerId();
- TestNodeManagerShutdown.startContainer(nm, cId, localFS, tmpDir,
- processStartFile, port);
-
- nm.getNMDispatcher().getEventHandler()
- .handle(new NodeManagerEvent(NodeManagerEventType.RESYNC));
- try {
- syncBarrier.await();
- } catch (BrokenBarrierException e) {
- }
- Assert.assertFalse(assertionFailedInThread.get());
- nm.stop();
- }
-
@SuppressWarnings("unchecked")
@Test(timeout=10000)
public void testNMshutdownWhenResyncThrowException() throws IOException,
@@ -493,135 +463,6 @@ public class TestNodeManagerResync {
}
}
- class TestNodeManager2 extends NodeManager {
-
- Thread launchContainersThread = null;
- @Override
- protected NodeStatusUpdater createNodeStatusUpdater(Context context,
- Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
- return new TestNodeStatusUpdaterImpl2(context, dispatcher,
- healthChecker, metrics);
- }
-
- @Override
- protected ContainerManagerImpl createContainerManager(Context context,
- ContainerExecutor exec, DeletionService del,
- NodeStatusUpdater nodeStatusUpdater, ApplicationACLsManager aclsManager,
- LocalDirsHandlerService dirsHandler) {
- return new ContainerManagerImpl(context, exec, del, nodeStatusUpdater,
- metrics, dirsHandler){
- @Override
- public void setBlockNewContainerRequests(
- boolean blockNewContainerRequests) {
- if (blockNewContainerRequests) {
- // start test thread right after blockNewContainerRequests is set
- // true
- super.setBlockNewContainerRequests(blockNewContainerRequests);
- launchContainersThread = new RejectedContainersLauncherThread();
- launchContainersThread.start();
- } else {
- // join the test thread right before blockNewContainerRequests is
- // reset
- try {
- // stop the test thread
- ((RejectedContainersLauncherThread) launchContainersThread)
- .setStopThreadFlag(true);
- launchContainersThread.join();
- ((RejectedContainersLauncherThread) launchContainersThread)
- .setStopThreadFlag(false);
- super.setBlockNewContainerRequests(blockNewContainerRequests);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
- };
- }
-
- class TestNodeStatusUpdaterImpl2 extends MockNodeStatusUpdater {
-
- public TestNodeStatusUpdaterImpl2(Context context, Dispatcher dispatcher,
- NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) {
- super(context, dispatcher, healthChecker, metrics);
- }
-
- @Override
- protected void rebootNodeStatusUpdaterAndRegisterWithRM() {
- ConcurrentMap<ContainerId, org.apache.hadoop.yarn.server.nodemanager
- .containermanager.container.Container> containers =
- getNMContext().getContainers();
-
- try {
- // ensure that containers are empty before restart nodeStatusUpdater
- if (!containers.isEmpty()) {
- for (Container container: containers.values()) {
- Assert.assertEquals(ContainerState.COMPLETE,
- container.cloneAndGetContainerStatus().getState());
- }
- }
- super.rebootNodeStatusUpdaterAndRegisterWithRM();
- // After this point new containers are free to be launched, except
- // containers from previous RM
- // Wait here so as to sync with the main test thread.
- syncBarrier.await();
- } catch (InterruptedException e) {
- } catch (BrokenBarrierException e) {
- } catch (AssertionError ae) {
- ae.printStackTrace();
- assertionFailedInThread.set(true);
- }
- }
- }
-
- class RejectedContainersLauncherThread extends Thread {
-
- boolean isStopped = false;
- public void setStopThreadFlag(boolean isStopped) {
- this.isStopped = isStopped;
- }
-
- @Override
- public void run() {
- int numContainers = 0;
- int numContainersRejected = 0;
- ContainerLaunchContext containerLaunchContext =
- recordFactory.newRecordInstance(ContainerLaunchContext.class);
- try {
- while (!isStopped && numContainers < 10) {
- StartContainerRequest scRequest =
- StartContainerRequest.newInstance(containerLaunchContext,
- null);
- List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
- list.add(scRequest);
- StartContainersRequest allRequests =
- StartContainersRequest.newInstance(list);
- System.out.println("no. of containers to be launched: "
- + numContainers);
- numContainers++;
- try {
- getContainerManager().startContainers(allRequests);
- } catch (YarnException e) {
- numContainersRejected++;
- Assert.assertTrue(e.getMessage().contains(
- "Rejecting new containers as NodeManager has not" +
- " yet connected with ResourceManager"));
- Assert.assertEquals(NMNotYetReadyException.class.getName(), e
- .getClass().getName());
- } catch (IOException e) {
- e.printStackTrace();
- assertionFailedInThread.set(true);
- }
- }
- // no. of containers to be launched should equal to no. of
- // containers rejected
- Assert.assertEquals(numContainers, numContainersRejected);
- } catch (AssertionError ae) {
- assertionFailedInThread.set(true);
- }
- }
- }
- }
-
class TestNodeManager3 extends NodeManager {
private int registrationCount = 0;
@@ -681,11 +522,6 @@ public class TestNodeManagerResync {
LocalDirsHandlerService dirsHandler) {
return new ContainerManagerImpl(context, exec, del, nodeStatusUpdater,
metrics, dirsHandler){
- @Override
- public void
- setBlockNewContainerRequests(boolean blockNewContainerRequests) {
- // do nothing
- }
@Override
protected void authorizeGetAndStopContainerRequest(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/424887ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
index ad0a831..2991c0c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager;
+import static org.mockito.Mockito.spy;
+
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -28,8 +30,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import org.apache.hadoop.yarn.server.nodemanager.executor.DeletionAsUserContext;
-import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -75,6 +75,7 @@ import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.executor.DeletionAsUserContext;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
@@ -82,10 +83,9 @@ import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerIn
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
-import static org.mockito.Mockito.spy;
-
public abstract class BaseContainerManagerTest {
protected static RecordFactory recordFactory = RecordFactoryProvider
@@ -214,11 +214,6 @@ public abstract class BaseContainerManagerTest {
return new ContainerManagerImpl(context, exec, delSrvc, nodeStatusUpdater,
metrics, dirsHandler) {
- @Override
- public void
- setBlockNewContainerRequests(boolean blockNewContainerRequests) {
- // do nothing
- }
@Override
protected void authorizeGetAndStopContainerRequest(ContainerId containerId,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/424887ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
index 6fead7e..60df7cb8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
@@ -127,11 +127,6 @@ public class TestContainerManager extends BaseContainerManagerTest {
createContainerManager(DeletionService delSrvc) {
return new ContainerManagerImpl(context, exec, delSrvc, nodeStatusUpdater,
metrics, dirsHandler) {
- @Override
- public void
- setBlockNewContainerRequests(boolean blockNewContainerRequests) {
- // do nothing
- }
@Override
protected UserGroupInformation getRemoteUgi() throws YarnException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/424887ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
index ef60c68..633bb6d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
@@ -545,11 +545,6 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest {
return new ContainerManagerImpl(context, exec, delSrvc,
mock(NodeStatusUpdater.class), metrics, dirsHandler) {
@Override
- public void
- setBlockNewContainerRequests(boolean blockNewContainerRequests) {
- // do nothing
- }
- @Override
protected void authorizeGetAndStopContainerRequest(
ContainerId containerId, Container container,
boolean stopRequest, NMTokenIdentifier identifier)
@@ -757,12 +752,6 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest {
}
@Override
- public void setBlockNewContainerRequests(
- boolean blockNewContainerRequests) {
- // do nothing
- }
-
- @Override
public NMTimelinePublisher
createNMTimelinePublisher(Context context) {
return null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/424887ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
index 46b32de..e6a7a02 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
@@ -56,12 +56,10 @@ public class TestNMProxy extends BaseContainerManagerTest {
}
int retryCount = 0;
- boolean shouldThrowNMNotYetReadyException = false;
@Before
public void setUp() throws Exception {
containerManager.start();
- containerManager.setBlockNewContainerRequests(false);
}
@Override
@@ -75,21 +73,13 @@ public class TestNMProxy extends BaseContainerManagerTest {
StartContainersRequest requests) throws YarnException, IOException {
if (retryCount < 5) {
retryCount++;
- if (shouldThrowNMNotYetReadyException) {
- // This causes super to throw an NMNotYetReadyException
- containerManager.setBlockNewContainerRequests(true);
+ if (isRetryPolicyRetryForEver()) {
+ // Throw non network exception
+ throw new IOException(
+ new UnreliableInterface.UnreliableException());
} else {
- if (isRetryPolicyRetryForEver()) {
- // Throw non network exception
- throw new IOException(
- new UnreliableInterface.UnreliableException());
- } else {
- throw new java.net.ConnectException("start container exception");
- }
+ throw new java.net.ConnectException("start container exception");
}
- } else {
- // This stops super from throwing an NMNotYetReadyException
- containerManager.setBlockNewContainerRequests(false);
}
return super.startContainers(requests);
}
@@ -131,26 +121,17 @@ public class TestNMProxy extends BaseContainerManagerTest {
ContainerManagementProtocol proxy = getNMProxy(conf);
- retryCount = 0;
- shouldThrowNMNotYetReadyException = false;
proxy.startContainers(allRequests);
Assert.assertEquals(5, retryCount);
retryCount = 0;
- shouldThrowNMNotYetReadyException = false;
proxy.stopContainers(Records.newRecord(StopContainersRequest.class));
Assert.assertEquals(5, retryCount);
retryCount = 0;
- shouldThrowNMNotYetReadyException = false;
proxy.getContainerStatuses(Records
.newRecord(GetContainerStatusesRequest.class));
Assert.assertEquals(5, retryCount);
-
- retryCount = 0;
- shouldThrowNMNotYetReadyException = true;
- proxy.startContainers(allRequests);
- Assert.assertEquals(5, retryCount);
}
@Test(timeout = 20000, expected = IOException.class)
@@ -162,7 +143,6 @@ public class TestNMProxy extends BaseContainerManagerTest {
ContainerManagementProtocol proxy = getNMProxy(conf);
- shouldThrowNMNotYetReadyException = false;
retryCount = 0;
proxy.startContainers(allRequests);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/424887ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
index a98a341..8264f2e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
@@ -77,11 +77,6 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
DeletionService delSrvc) {
return new ContainerManagerImpl(context, exec, delSrvc,
nodeStatusUpdater, metrics, dirsHandler) {
- @Override
- public void
- setBlockNewContainerRequests(boolean blockNewContainerRequests) {
- // do nothing
- }
@Override
protected UserGroupInformation getRemoteUgi() throws YarnException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/424887ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
index 98cb365..9626b35 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
@@ -66,7 +66,6 @@ import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
@@ -178,7 +177,7 @@ public class TestContainerManagerSecurity extends KerberosSecurityTestcase {
NodeManager nm = yarnCluster.getNodeManager(0);
- waitForNMToReceiveNMTokenKey(nmTokenSecretManagerNM, nm);
+ waitForNMToReceiveNMTokenKey(nmTokenSecretManagerNM);
// Both id should be equal.
Assert.assertEquals(nmTokenSecretManagerNM.getCurrentKey().getKeyId(),
@@ -412,13 +411,10 @@ public class TestContainerManagerSecurity extends KerberosSecurityTestcase {
}
protected void waitForNMToReceiveNMTokenKey(
- NMTokenSecretManagerInNM nmTokenSecretManagerNM, NodeManager nm)
+ NMTokenSecretManagerInNM nmTokenSecretManagerNM)
throws InterruptedException {
int attempt = 60;
- ContainerManagerImpl cm =
- ((ContainerManagerImpl) nm.getNMContext().getContainerManager());
- while ((cm.getBlockNewContainerRequestsStatus() || nmTokenSecretManagerNM
- .getNodeId() == null) && attempt-- > 0) {
+ while (nmTokenSecretManagerNM.getNodeId() == null && attempt-- > 0) {
Thread.sleep(2000);
}
}
@@ -627,7 +623,7 @@ public class TestContainerManagerSecurity extends KerberosSecurityTestcase {
nm.getNMContext().getNMTokenSecretManager();
String user = "test";
- waitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM, nm);
+ waitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM);
NodeId nodeId = nm.getNMContext().getNodeId();
@@ -722,7 +718,7 @@ public class TestContainerManagerSecurity extends KerberosSecurityTestcase {
nm.getNMContext().getNMTokenSecretManager();
String user = "test";
- waitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM, nm);
+ waitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM);
NodeId nodeId = nm.getNMContext().getNodeId();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: HADOOP-14377. Increase Common test
timeouts from 1 second to 10 seconds. Contributed by Eric Badger
Posted by in...@apache.org.
HADOOP-14377. Increase Common test timeouts from 1 second to 10 seconds. Contributed by Eric Badger
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f7faac8e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f7faac8e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f7faac8e
Branch: refs/heads/HDFS-10467
Commit: f7faac8e90efaed4fbe9975c97e1b861c32cc3a3
Parents: a2f6804
Author: Jason Lowe <jl...@yahoo-inc.com>
Authored: Tue May 9 16:08:34 2017 -0500
Committer: Jason Lowe <jl...@yahoo-inc.com>
Committed: Tue May 9 16:08:34 2017 -0500
----------------------------------------------------------------------
.../crypto/TestCryptoStreamsForLocalFS.java | 10 +--
.../hadoop/crypto/TestCryptoStreamsNormal.java | 16 ++---
.../org/apache/hadoop/fs/TestGlobPattern.java | 2 +-
.../hadoop/fs/TestSymlinkLocalFSFileSystem.java | 8 +--
.../apache/hadoop/io/TestSortedMapWritable.java | 2 +-
.../hadoop/io/compress/TestCodecPool.java | 12 ++--
.../security/TestUserGroupInformation.java | 2 +-
.../hadoop/service/TestCompositeService.java | 66 ++++++++++----------
.../org/apache/hadoop/util/TestClassUtil.java | 2 +-
.../org/apache/hadoop/portmap/TestPortmap.java | 4 +-
10 files changed, 62 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7faac8e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsForLocalFS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsForLocalFS.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsForLocalFS.java
index 235e9c2..1ef6f3c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsForLocalFS.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsForLocalFS.java
@@ -88,28 +88,28 @@ public class TestCryptoStreamsForLocalFS extends CryptoStreamsTestBase {
@Ignore("ChecksumFSInputChecker doesn't support ByteBuffer read")
@Override
- @Test(timeout=1000)
+ @Test(timeout=10000)
public void testByteBufferRead() throws Exception {}
@Ignore("ChecksumFSOutputSummer doesn't support Syncable")
@Override
- @Test(timeout=1000)
+ @Test(timeout=10000)
public void testSyncable() throws IOException {}
@Ignore("ChecksumFSInputChecker doesn't support ByteBuffer read")
@Override
- @Test(timeout=1000)
+ @Test(timeout=10000)
public void testCombinedOp() throws Exception {}
@Ignore("ChecksumFSInputChecker doesn't support enhanced ByteBuffer access")
@Override
- @Test(timeout=1000)
+ @Test(timeout=10000)
public void testHasEnhancedByteBufferAccess() throws Exception {
}
@Ignore("ChecksumFSInputChecker doesn't support seekToNewSource")
@Override
- @Test(timeout=1000)
+ @Test(timeout=10000)
public void testSeekToNewSource() throws Exception {
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7faac8e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsNormal.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsNormal.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsNormal.java
index e9c313f..b5382c1 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsNormal.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsNormal.java
@@ -83,41 +83,41 @@ public class TestCryptoStreamsNormal extends CryptoStreamsTestBase {
@Ignore("Wrapped stream doesn't support Syncable")
@Override
- @Test(timeout=1000)
+ @Test(timeout=10000)
public void testSyncable() throws IOException {}
@Ignore("Wrapped stream doesn't support PositionedRead")
@Override
- @Test(timeout=1000)
+ @Test(timeout=10000)
public void testPositionedRead() throws IOException {}
@Ignore("Wrapped stream doesn't support ReadFully")
@Override
- @Test(timeout=1000)
+ @Test(timeout=10000)
public void testReadFully() throws IOException {}
@Ignore("Wrapped stream doesn't support Seek")
@Override
- @Test(timeout=1000)
+ @Test(timeout=10000)
public void testSeek() throws IOException {}
@Ignore("Wrapped stream doesn't support ByteBufferRead")
@Override
- @Test(timeout=1000)
+ @Test(timeout=10000)
public void testByteBufferRead() throws IOException {}
@Ignore("Wrapped stream doesn't support ByteBufferRead, Seek")
@Override
- @Test(timeout=1000)
+ @Test(timeout=10000)
public void testCombinedOp() throws IOException {}
@Ignore("Wrapped stream doesn't support SeekToNewSource")
@Override
- @Test(timeout=1000)
+ @Test(timeout=10000)
public void testSeekToNewSource() throws IOException {}
@Ignore("Wrapped stream doesn't support HasEnhancedByteBufferAccess")
@Override
- @Test(timeout=1000)
+ @Test(timeout=10000)
public void testHasEnhancedByteBufferAccess() throws IOException {}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7faac8e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestGlobPattern.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestGlobPattern.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestGlobPattern.java
index 99cb645..b409a8f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestGlobPattern.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestGlobPattern.java
@@ -72,7 +72,7 @@ public class TestGlobPattern {
shouldThrow("[", "[[]]", "{", "\\");
}
- @Test(timeout=1000) public void testPathologicalPatterns() {
+ @Test(timeout=10000) public void testPathologicalPatterns() {
String badFilename = "job_1429571161900_4222-1430338332599-tda%2D%2D+******************************+++...%270%27%28Stage-1430338580443-39-2000-SUCCEEDED-production%2Dhigh-1430338340360.jhist";
assertMatch(true, badFilename, badFilename);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7faac8e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestSymlinkLocalFSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestSymlinkLocalFSFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestSymlinkLocalFSFileSystem.java
index 6fc2d62..9844949 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestSymlinkLocalFSFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestSymlinkLocalFSFileSystem.java
@@ -41,24 +41,24 @@ public class TestSymlinkLocalFSFileSystem extends TestSymlinkLocalFS {
@Ignore("RawLocalFileSystem#mkdir does not treat existence of directory" +
" as an error")
@Override
- @Test(timeout=1000)
+ @Test(timeout=10000)
public void testMkdirExistingLink() throws IOException {}
@Ignore("FileSystem#create defaults to creating parents," +
" throwing an IOException instead of FileNotFoundException")
@Override
- @Test(timeout=1000)
+ @Test(timeout=10000)
public void testCreateFileViaDanglingLinkParent() throws IOException {}
@Ignore("RawLocalFileSystem does not throw an exception if the path" +
" already exists")
@Override
- @Test(timeout=1000)
+ @Test(timeout=10000)
public void testCreateFileDirExistingLink() throws IOException {}
@Ignore("ChecksumFileSystem does not support append")
@Override
- @Test(timeout=1000)
+ @Test(timeout=10000)
public void testAccessFileViaInterSymlinkAbsTarget() throws IOException {}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7faac8e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSortedMapWritable.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSortedMapWritable.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSortedMapWritable.java
index 8c98808..3d5bb1e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSortedMapWritable.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSortedMapWritable.java
@@ -166,7 +166,7 @@ public class TestSortedMapWritable {
assertTrue(failureReason, !mapB.equals(mapA));
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testPutAll() {
SortedMapWritable<Text> map1 = new SortedMapWritable<Text>();
SortedMapWritable<Text> map2 = new SortedMapWritable<Text>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7faac8e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java
index c889a59..1fb25cb 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java
@@ -43,7 +43,7 @@ public class TestCodecPool {
this.codec.setConf(new Configuration());
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testCompressorPoolCounts() {
// Get two compressors and return them
Compressor comp1 = CodecPool.getCompressor(codec);
@@ -64,7 +64,7 @@ public class TestCodecPool {
CodecPool.getLeasedCompressorsCount(codec));
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testCompressorNotReturnSameInstance() {
Compressor comp = CodecPool.getCompressor(codec);
CodecPool.returnCompressor(comp);
@@ -79,7 +79,7 @@ public class TestCodecPool {
}
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testDecompressorPoolCounts() {
// Get two decompressors and return them
Decompressor decomp1 = CodecPool.getDecompressor(codec);
@@ -100,7 +100,7 @@ public class TestCodecPool {
CodecPool.getLeasedCompressorsCount(codec));
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testMultiThreadedCompressorPool() throws InterruptedException {
final int iterations = 4;
ExecutorService threadpool = Executors.newFixedThreadPool(3);
@@ -137,7 +137,7 @@ public class TestCodecPool {
assertEquals(LEASE_COUNT_ERR, 0, CodecPool.getLeasedCompressorsCount(codec));
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testMultiThreadedDecompressorPool() throws InterruptedException {
final int iterations = 4;
ExecutorService threadpool = Executors.newFixedThreadPool(3);
@@ -175,7 +175,7 @@ public class TestCodecPool {
CodecPool.getLeasedDecompressorsCount(codec));
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testDecompressorNotReturnSameInstance() {
Decompressor decomp = CodecPool.getDecompressor(codec);
CodecPool.returnDecompressor(decomp);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7faac8e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
index 510987e..00062c0 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
@@ -878,7 +878,7 @@ public class TestUserGroupInformation {
method.setAccessible(false);
}
- @Test(timeout=1000)
+ @Test(timeout=10000)
public void testSetLoginUser() throws IOException {
UserGroupInformation ugi = UserGroupInformation.createRemoteUser("test-user");
UserGroupInformation.setLoginUser(ugi);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7faac8e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/TestCompositeService.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/TestCompositeService.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/TestCompositeService.java
index 9493740..6189c0e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/TestCompositeService.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/TestCompositeService.java
@@ -312,7 +312,7 @@ public class TestCompositeService {
assertInState(STATE.INITED, child);
}
- @Test (timeout = 1000)
+ @Test (timeout = 10000)
public void testAddIfService() {
CompositeService testService = new CompositeService("TestService") {
Service service;
@@ -363,7 +363,7 @@ public class TestCompositeService {
// Tests for adding child service to parent
//
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddUninitedChildBeforeInit() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService child = new BreakableService();
@@ -376,7 +376,7 @@ public class TestCompositeService {
assertInState(STATE.STOPPED, child);
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddUninitedChildInInit() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService child = new BreakableService();
@@ -396,7 +396,7 @@ public class TestCompositeService {
1, parent.getServices().size());
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddUninitedChildInStart() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService child = new BreakableService();
@@ -408,7 +408,7 @@ public class TestCompositeService {
assertInState(STATE.NOTINITED, child);
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddUninitedChildInStop() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService child = new BreakableService();
@@ -419,7 +419,7 @@ public class TestCompositeService {
assertInState(STATE.NOTINITED, child);
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddInitedChildBeforeInit() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService child = new BreakableService();
@@ -433,7 +433,7 @@ public class TestCompositeService {
assertInState(STATE.STOPPED, child);
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddInitedChildInInit() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService child = new BreakableService();
@@ -446,7 +446,7 @@ public class TestCompositeService {
assertInState(STATE.STOPPED, child);
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddInitedChildInStart() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService child = new BreakableService();
@@ -459,7 +459,7 @@ public class TestCompositeService {
assertInState(STATE.STOPPED, child);
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddInitedChildInStop() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService child = new BreakableService();
@@ -471,7 +471,7 @@ public class TestCompositeService {
assertInState(STATE.INITED, child);
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddStartedChildBeforeInit() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService child = new BreakableService();
@@ -489,7 +489,7 @@ public class TestCompositeService {
1, parent.getServices().size());
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddStartedChildInInit() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService child = new BreakableService();
@@ -503,7 +503,7 @@ public class TestCompositeService {
assertInState(STATE.STOPPED, child);
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddStartedChildInStart() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService child = new BreakableService();
@@ -517,7 +517,7 @@ public class TestCompositeService {
assertInState(STATE.STOPPED, child);
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddStartedChildInStop() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService child = new BreakableService();
@@ -530,7 +530,7 @@ public class TestCompositeService {
assertInState(STATE.STARTED, child);
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddStoppedChildBeforeInit() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService child = new BreakableService();
@@ -549,7 +549,7 @@ public class TestCompositeService {
1, parent.getServices().size());
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddStoppedChildInInit() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService child = new BreakableService();
@@ -570,7 +570,7 @@ public class TestCompositeService {
1, parent.getServices().size());
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddStoppedChildInStart() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService child = new BreakableService();
@@ -583,7 +583,7 @@ public class TestCompositeService {
parent.stop();
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddStoppedChildInStop() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService child = new BreakableService();
@@ -600,7 +600,7 @@ public class TestCompositeService {
// Tests for adding sibling service to parent
//
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddUninitedSiblingBeforeInit() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService sibling = new BreakableService();
@@ -617,7 +617,7 @@ public class TestCompositeService {
1, parent.getServices().size());
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddUninitedSiblingInInit() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService sibling = new BreakableService();
@@ -636,7 +636,7 @@ public class TestCompositeService {
2, parent.getServices().size());
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddUninitedSiblingInStart() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService sibling = new BreakableService();
@@ -653,7 +653,7 @@ public class TestCompositeService {
2, parent.getServices().size());
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddUninitedSiblingInStop() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService sibling = new BreakableService();
@@ -670,7 +670,7 @@ public class TestCompositeService {
2, parent.getServices().size());
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddInitedSiblingBeforeInit() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService sibling = new BreakableService();
@@ -688,7 +688,7 @@ public class TestCompositeService {
1, parent.getServices().size());
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddInitedSiblingInInit() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService sibling = new BreakableService();
@@ -706,7 +706,7 @@ public class TestCompositeService {
2, parent.getServices().size());
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddInitedSiblingInStart() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService sibling = new BreakableService();
@@ -724,7 +724,7 @@ public class TestCompositeService {
2, parent.getServices().size());
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddInitedSiblingInStop() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService sibling = new BreakableService();
@@ -735,7 +735,7 @@ public class TestCompositeService {
parent.init(new Configuration());
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddStartedSiblingBeforeInit() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService sibling = new BreakableService();
@@ -754,7 +754,7 @@ public class TestCompositeService {
1, parent.getServices().size());
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddStartedSiblingInInit() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService sibling = new BreakableService();
@@ -774,7 +774,7 @@ public class TestCompositeService {
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddStartedSiblingInStart() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService sibling = new BreakableService();
@@ -793,7 +793,7 @@ public class TestCompositeService {
2, parent.getServices().size());
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddStartedSiblingInStop() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService sibling = new BreakableService();
@@ -812,7 +812,7 @@ public class TestCompositeService {
2, parent.getServices().size());
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddStoppedSiblingBeforeInit() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService sibling = new BreakableService();
@@ -832,7 +832,7 @@ public class TestCompositeService {
1, parent.getServices().size());
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddStoppedSiblingInInit() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService sibling = new BreakableService();
@@ -856,7 +856,7 @@ public class TestCompositeService {
2, parent.getServices().size());
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddStoppedSiblingInStart() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService sibling = new BreakableService();
@@ -876,7 +876,7 @@ public class TestCompositeService {
2, parent.getServices().size());
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testAddStoppedSiblingInStop() throws Throwable {
CompositeService parent = new CompositeService("parent");
BreakableService sibling = new BreakableService();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7faac8e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClassUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClassUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClassUtil.java
index 4bc36e2..98e1822 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClassUtil.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClassUtil.java
@@ -26,7 +26,7 @@ import org.apache.log4j.Logger;
import org.junit.Test;
public class TestClassUtil {
- @Test(timeout=1000)
+ @Test(timeout=10000)
public void testFindContainingJar() {
String containingJar = ClassUtil.findContainingJar(Logger.class);
Assert.assertNotNull("Containing jar not found for Logger",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7faac8e/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java
index f217e84..ba132bb 100644
--- a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java
+++ b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java
@@ -53,7 +53,7 @@ public class TestPortmap {
pm.shutdown();
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testIdle() throws InterruptedException, IOException {
Socket s = new Socket();
try {
@@ -75,7 +75,7 @@ public class TestPortmap {
}
}
- @Test(timeout = 1000)
+ @Test(timeout = 10000)
public void testRegistration() throws IOException, InterruptedException {
XDR req = new XDR();
RpcCall.getInstance(++xid, RpcProgramPortmap.PROGRAM,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: YARN-6234. Support multiple
attempts on the node when AMRMProxy is enabled. (Giovanni Matteo Fumarola via
Subru).
Posted by in...@apache.org.
YARN-6234. Support multiple attempts on the node when AMRMProxy is enabled. (Giovanni Matteo Fumarola via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cd9ff27f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cd9ff27f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cd9ff27f
Branch: refs/heads/HDFS-10467
Commit: cd9ff27ffc9369820d0c39200a11bf00e6a767c8
Parents: 1769b12
Author: Subru Krishnan <su...@apache.org>
Authored: Mon May 8 16:41:30 2017 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Mon May 8 16:41:30 2017 -0700
----------------------------------------------------------------------
.../nodemanager/amrmproxy/AMRMProxyService.java | 32 ++++++++++++++---
.../amrmproxy/TestAMRMProxyService.java | 36 ++++++++++++++++++++
2 files changed, 63 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd9ff27f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
index 9f2d9a1..2696bca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
@@ -270,18 +270,40 @@ public class AMRMProxyService extends AbstractService implements
* @param user
* @param amrmToken
*/
- protected void initializePipeline(
- ApplicationAttemptId applicationAttemptId, String user,
- Token<AMRMTokenIdentifier> amrmToken,
+ protected void initializePipeline(ApplicationAttemptId applicationAttemptId,
+ String user, Token<AMRMTokenIdentifier> amrmToken,
Token<AMRMTokenIdentifier> localToken) {
RequestInterceptorChainWrapper chainWrapper = null;
synchronized (applPipelineMap) {
- if (applPipelineMap.containsKey(applicationAttemptId.getApplicationId())) {
+ if (applPipelineMap
+ .containsKey(applicationAttemptId.getApplicationId())) {
LOG.warn("Request to start an already existing appId was received. "
+ " This can happen if an application failed and a new attempt "
+ "was created on this machine. ApplicationId: "
+ applicationAttemptId.toString());
- return;
+
+ RequestInterceptorChainWrapper chainWrapperBackup =
+ this.applPipelineMap.get(applicationAttemptId.getApplicationId());
+ if (chainWrapperBackup != null
+ && chainWrapperBackup.getApplicationAttemptId() != null
+ && !chainWrapperBackup.getApplicationAttemptId()
+ .equals(applicationAttemptId)) {
+ // Remove the existing pipeline
+ LOG.info("Remove the previous pipeline for ApplicationId: "
+ + applicationAttemptId.toString());
+ RequestInterceptorChainWrapper pipeline =
+ applPipelineMap.remove(applicationAttemptId.getApplicationId());
+ try {
+ pipeline.getRootInterceptor().shutdown();
+ } catch (Throwable ex) {
+ LOG.warn(
+ "Failed to shutdown the request processing pipeline for app:"
+ + applicationAttemptId.getApplicationId(),
+ ex);
+ }
+ } else {
+ return;
+ }
}
chainWrapper = new RequestInterceptorChainWrapper();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd9ff27f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java
index 7fffddf..837278c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.nodemanager.amrmproxy;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -27,10 +28,14 @@ import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.nodemanager.amrmproxy.AMRMProxyService.RequestInterceptorChainWrapper;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Assert;
import org.junit.Test;
@@ -380,6 +385,37 @@ public class TestAMRMProxyService extends BaseAMRMProxyTest {
}
}
+ @Test
+ public void testMultipleAttemptsSameNode()
+ throws YarnException, IOException, Exception {
+
+ String user = "hadoop";
+ ApplicationId appId = ApplicationId.newInstance(1, 1);
+ ApplicationAttemptId applicationAttemptId;
+
+ // First Attempt
+
+ RegisterApplicationMasterResponse response1 =
+ registerApplicationMaster(appId.getId());
+ Assert.assertNotNull(response1);
+
+ AllocateResponse allocateResponse = allocate(appId.getId());
+ Assert.assertNotNull(allocateResponse);
+
+ // Second Attempt
+
+ applicationAttemptId = ApplicationAttemptId.newInstance(appId, 2);
+ getAMRMProxyService().initializePipeline(applicationAttemptId, user, null,
+ null);
+
+ RequestInterceptorChainWrapper chain2 =
+ getAMRMProxyService().getPipelines().get(appId);
+ Assert.assertEquals(applicationAttemptId, chain2.getApplicationAttemptId());
+
+ allocateResponse = allocate(appId.getId());
+ Assert.assertNotNull(allocateResponse);
+ }
+
private List<Container> getContainersAndAssert(int appId,
int numberOfResourceRequests) throws Exception {
AllocateRequest allocateRequest =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: YARN-6435. [ATSv2] Can't retrieve
more than 1000 versions of metrics in time series. (Rohith Sharma K S via
Haibo Chen)
Posted by in...@apache.org.
YARN-6435. [ATSv2] Can't retrieve more than 1000 versions of metrics in time series. (Rohith Sharma K S via Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/461ee44d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/461ee44d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/461ee44d
Branch: refs/heads/HDFS-10467
Commit: 461ee44d287b1fcf0bf15d662aebd3e6f2b83a72
Parents: 166be0e
Author: Haibo Chen <ha...@cloudera.com>
Authored: Tue May 9 21:10:18 2017 -0700
Committer: Haibo Chen <ha...@cloudera.com>
Committed: Tue May 9 21:12:57 2017 -0700
----------------------------------------------------------------------
.../storage/application/ApplicationTable.java | 12 ++++++++++--
.../timelineservice/storage/entity/EntityTable.java | 12 ++++++++++--
2 files changed, 20 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/461ee44d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
index a02f768..cb4fc92 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
@@ -82,6 +82,13 @@ public class ApplicationTable extends BaseTable<ApplicationTable> {
private static final String METRICS_TTL_CONF_NAME = PREFIX
+ ".table.metrics.ttl";
+ /**
+ * config param name that specifies max-versions for metrics column family in
+ * entity table.
+ */
+ private static final String METRICS_MAX_VERSIONS =
+ PREFIX + ".table.metrics.max-versions";
+
/** default value for application table name. */
private static final String DEFAULT_TABLE_NAME =
"timelineservice.application";
@@ -90,7 +97,7 @@ public class ApplicationTable extends BaseTable<ApplicationTable> {
private static final int DEFAULT_METRICS_TTL = 2592000;
/** default max number of versions. */
- private static final int DEFAULT_METRICS_MAX_VERSIONS = 1000;
+ private static final int DEFAULT_METRICS_MAX_VERSIONS = 10000;
private static final Log LOG = LogFactory.getLog(ApplicationTable.class);
@@ -136,7 +143,8 @@ public class ApplicationTable extends BaseTable<ApplicationTable> {
metricsCF.setBlockCacheEnabled(true);
// always keep 1 version (the latest)
metricsCF.setMinVersions(1);
- metricsCF.setMaxVersions(DEFAULT_METRICS_MAX_VERSIONS);
+ metricsCF.setMaxVersions(
+ hbaseConf.getInt(METRICS_MAX_VERSIONS, DEFAULT_METRICS_MAX_VERSIONS));
metricsCF.setTimeToLive(hbaseConf.getInt(METRICS_TTL_CONF_NAME,
DEFAULT_METRICS_TTL));
applicationTableDescp.setRegionSplitPolicyClassName(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/461ee44d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
index b194f07..ddf0406 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
@@ -83,6 +83,13 @@ public class EntityTable extends BaseTable<EntityTable> {
private static final String METRICS_TTL_CONF_NAME = PREFIX
+ ".table.metrics.ttl";
+ /**
+ * config param name that specifies max-versions for metrics column family in
+ * entity table.
+ */
+ private static final String METRICS_MAX_VERSIONS =
+ PREFIX + ".table.metrics.max-versions";
+
/** default value for entity table name. */
public static final String DEFAULT_TABLE_NAME = "timelineservice.entity";
@@ -90,7 +97,7 @@ public class EntityTable extends BaseTable<EntityTable> {
private static final int DEFAULT_METRICS_TTL = 2592000;
/** default max number of versions. */
- private static final int DEFAULT_METRICS_MAX_VERSIONS = 1000;
+ private static final int DEFAULT_METRICS_MAX_VERSIONS = 10000;
private static final Log LOG = LogFactory.getLog(EntityTable.class);
@@ -136,7 +143,8 @@ public class EntityTable extends BaseTable<EntityTable> {
metricsCF.setBlockCacheEnabled(true);
// always keep 1 version (the latest)
metricsCF.setMinVersions(1);
- metricsCF.setMaxVersions(DEFAULT_METRICS_MAX_VERSIONS);
+ metricsCF.setMaxVersions(
+ hbaseConf.getInt(METRICS_MAX_VERSIONS, DEFAULT_METRICS_MAX_VERSIONS));
metricsCF.setTimeToLive(hbaseConf.getInt(METRICS_TTL_CONF_NAME,
DEFAULT_METRICS_TTL));
entityTableDescp.setRegionSplitPolicyClassName(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: HADOOP-14405. Fix performance
regression due to incorrect use of DataChecksum. Contributed by LiXin Ge.
Posted by in...@apache.org.
HADOOP-14405. Fix performance regression due to incorrect use of DataChecksum. Contributed by LiXin Ge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/06ffeb8d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/06ffeb8d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/06ffeb8d
Branch: refs/heads/HDFS-10467
Commit: 06ffeb8d4d705d14e8755364099286ac4ed3bb83
Parents: ab2bb93
Author: Masatake Iwasaki <iw...@apache.org>
Authored: Wed May 10 14:14:20 2017 +0900
Committer: Masatake Iwasaki <iw...@apache.org>
Committed: Wed May 10 14:16:41 2017 +0900
----------------------------------------------------------------------
.../java/org/apache/hadoop/util/DataChecksum.java | 14 +++++++++++---
1 file changed, 11 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06ffeb8d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
index 6982a92..03946af 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
@@ -296,12 +296,20 @@ public class DataChecksum implements Checksum {
public void verifyChunkedSums(ByteBuffer data, ByteBuffer checksums,
String fileName, long basePos) throws ChecksumException {
if (type.size == 0) return;
-
+
if (data.hasArray() && checksums.hasArray()) {
final int dataOffset = data.arrayOffset() + data.position();
final int crcsOffset = checksums.arrayOffset() + checksums.position();
- verifyChunked(type, summer, data.array(), dataOffset, data.remaining(),
- bytesPerChecksum, checksums.array(), crcsOffset, fileName, basePos);
+
+ if (NativeCrc32.isAvailable()) {
+ NativeCrc32.verifyChunkedSumsByteArray(bytesPerChecksum, type.id,
+ checksums.array(), crcsOffset, data.array(), dataOffset,
+ data.remaining(), fileName, basePos);
+ } else {
+ verifyChunked(type, summer, data.array(), dataOffset, data.remaining(),
+ bytesPerChecksum, checksums.array(), crcsOffset, fileName,
+ basePos);
+ }
return;
}
if (NativeCrc32.isAvailable() && data.isDirect()) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: HADOOP-14374. License error in
GridmixTestUtils.java. Contributed by lixinglong.
Posted by in...@apache.org.
HADOOP-14374. License error in GridmixTestUtils.java. Contributed by lixinglong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c60164fb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c60164fb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c60164fb
Branch: refs/heads/HDFS-10467
Commit: c60164fb60d3d8afb0fb8410936ccc231c4593f9
Parents: 54fd0e4
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue May 9 05:37:13 2017 -0500
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue May 9 05:37:43 2017 -0500
----------------------------------------------------------------------
.../hadoop/mapred/gridmix/GridmixTestUtils.java | 31 +++++++++++---------
1 file changed, 17 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c60164fb/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/GridmixTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/GridmixTestUtils.java b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/GridmixTestUtils.java
index 7052540..3da5d9d 100644
--- a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/GridmixTestUtils.java
+++ b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/GridmixTestUtils.java
@@ -1,3 +1,19 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
package org.apache.hadoop.mapred.gridmix;
import org.apache.commons.logging.Log;
@@ -16,20 +32,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.C
import java.io.IOException;
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
+ * This is a test class.
*/
public class GridmixTestUtils {
private static final Log LOG = LogFactory.getLog(GridmixTestUtils.class);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: HDFS-9342. Erasure coding: client
should update and commit block based on acknowledged size. Contributed by
SammiChen.
Posted by in...@apache.org.
HDFS-9342. Erasure coding: client should update and commit block based on acknowledged size. Contributed by SammiChen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a9a3d219
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a9a3d219
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a9a3d219
Branch: refs/heads/HDFS-10467
Commit: a9a3d219fed2dd9d7bb84c228f6b8d97eadbe1f6
Parents: 8065129
Author: Andrew Wang <wa...@apache.org>
Authored: Sun May 7 14:45:26 2017 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Sun May 7 14:45:26 2017 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/DFSStripedOutputStream.java | 28 ++++++++++++++++++++
1 file changed, 28 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9a3d219/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
index 3dd07f7..0fdae8c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
@@ -772,9 +772,37 @@ public class DFSStripedOutputStream extends DFSOutputStream {
newStorageIDs[i] = "";
}
}
+
+ // should update the block group length based on the acked length
+ final long sentBytes = currentBlockGroup.getNumBytes();
+ final long ackedBytes = getNumAckedStripes() * cellSize * numDataBlocks;
+ Preconditions.checkState(ackedBytes <= sentBytes);
+ currentBlockGroup.setNumBytes(ackedBytes);
+ newBG.setNumBytes(ackedBytes);
dfsClient.namenode.updatePipeline(dfsClient.clientName, currentBlockGroup,
newBG, newNodes, newStorageIDs);
currentBlockGroup = newBG;
+ currentBlockGroup.setNumBytes(sentBytes);
+ }
+
+ /**
+ * Get the number of acked stripes. An acked stripe means at least data block
+ * number size cells of the stripe were acked.
+ */
+ private long getNumAckedStripes() {
+ int minStripeNum = Integer.MAX_VALUE;
+ for (int i = 0; i < numAllBlocks; i++) {
+ final StripedDataStreamer streamer = getStripedDataStreamer(i);
+ if (streamer.isHealthy()) {
+ int curStripeNum = 0;
+ if (streamer.getBlock() != null) {
+ curStripeNum = (int) (streamer.getBlock().getNumBytes() / cellSize);
+ }
+ minStripeNum = Math.min(curStripeNum, minStripeNum);
+ }
+ }
+ assert minStripeNum != Integer.MAX_VALUE;
+ return minStripeNum;
}
private int stripeDataSize() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: HDFS-11546. Federation Router RPC
server. Contributed by Jason Kace and Inigo Goiri.
Posted by in...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6632c5a4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
index ee6f57d..2875750 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.util.Time;
/**
* In-memory cache/mock of a namenode and file resolver. Stores the most
- * recently updated NN information for each nameservice and block pool. Also
+ * recently updated NN information for each nameservice and block pool. It also
* stores a virtual mount table for resolving global namespace paths to local NN
* paths.
*/
@@ -51,82 +51,93 @@ public class MockResolver
implements ActiveNamenodeResolver, FileSubclusterResolver {
private Map<String, List<? extends FederationNamenodeContext>> resolver =
- new HashMap<String, List<? extends FederationNamenodeContext>>();
- private Map<String, List<RemoteLocation>> locations =
- new HashMap<String, List<RemoteLocation>>();
- private Set<FederationNamespaceInfo> namespaces =
- new HashSet<FederationNamespaceInfo>();
+ new HashMap<>();
+ private Map<String, List<RemoteLocation>> locations = new HashMap<>();
+ private Set<FederationNamespaceInfo> namespaces = new HashSet<>();
private String defaultNamespace = null;
+
public MockResolver(Configuration conf, StateStoreService store) {
this.cleanRegistrations();
}
- public void addLocation(String mount, String nameservice, String location) {
- RemoteLocation remoteLocation = new RemoteLocation(nameservice, location);
- List<RemoteLocation> locationsList = locations.get(mount);
+ public void addLocation(String mount, String nsId, String location) {
+ List<RemoteLocation> locationsList = this.locations.get(mount);
if (locationsList == null) {
- locationsList = new LinkedList<RemoteLocation>();
- locations.put(mount, locationsList);
+ locationsList = new LinkedList<>();
+ this.locations.put(mount, locationsList);
}
+
+ final RemoteLocation remoteLocation = new RemoteLocation(nsId, location);
if (!locationsList.contains(remoteLocation)) {
locationsList.add(remoteLocation);
}
if (this.defaultNamespace == null) {
- this.defaultNamespace = nameservice;
+ this.defaultNamespace = nsId;
}
}
public synchronized void cleanRegistrations() {
- this.resolver =
- new HashMap<String, List<? extends FederationNamenodeContext>>();
- this.namespaces = new HashSet<FederationNamespaceInfo>();
+ this.resolver = new HashMap<>();
+ this.namespaces = new HashSet<>();
}
@Override
public void updateActiveNamenode(
- String ns, InetSocketAddress successfulAddress) {
+ String nsId, InetSocketAddress successfulAddress) {
String address = successfulAddress.getHostName() + ":" +
successfulAddress.getPort();
- String key = ns;
+ String key = nsId;
if (key != null) {
// Update the active entry
@SuppressWarnings("unchecked")
- List<FederationNamenodeContext> iterator =
- (List<FederationNamenodeContext>) resolver.get(key);
- for (FederationNamenodeContext namenode : iterator) {
+ List<FederationNamenodeContext> namenodes =
+ (List<FederationNamenodeContext>) this.resolver.get(key);
+ for (FederationNamenodeContext namenode : namenodes) {
if (namenode.getRpcAddress().equals(address)) {
MockNamenodeContext nn = (MockNamenodeContext) namenode;
nn.setState(FederationNamenodeServiceState.ACTIVE);
break;
}
}
- Collections.sort(iterator, new NamenodePriorityComparator());
+ // This operation modifies the list so we need to be careful
+ synchronized(namenodes) {
+ Collections.sort(namenodes, new NamenodePriorityComparator());
+ }
}
}
@Override
public List<? extends FederationNamenodeContext>
getNamenodesForNameserviceId(String nameserviceId) {
- return resolver.get(nameserviceId);
+ // Return a copy of the list because it is updated periodically
+ List<? extends FederationNamenodeContext> namenodes =
+ this.resolver.get(nameserviceId);
+ return Collections.unmodifiableList(new ArrayList<>(namenodes));
}
@Override
public List<? extends FederationNamenodeContext> getNamenodesForBlockPoolId(
String blockPoolId) {
- return resolver.get(blockPoolId);
+ // Return a copy of the list because it is updated periodically
+ List<? extends FederationNamenodeContext> namenodes =
+ this.resolver.get(blockPoolId);
+ return Collections.unmodifiableList(new ArrayList<>(namenodes));
}
private static class MockNamenodeContext
implements FederationNamenodeContext {
+
+ private String namenodeId;
+ private String nameserviceId;
+
private String webAddress;
private String rpcAddress;
private String serviceAddress;
private String lifelineAddress;
- private String namenodeId;
- private String nameserviceId;
+
private FederationNamenodeServiceState state;
private long dateModified;
@@ -197,6 +208,7 @@ public class MockResolver
@Override
public synchronized boolean registerNamenode(NamenodeStatusReport report)
throws IOException {
+
MockNamenodeContext context = new MockNamenodeContext(
report.getRpcAddress(), report.getServiceAddress(),
report.getLifelineAddress(), report.getWebAddress(),
@@ -205,13 +217,14 @@ public class MockResolver
String nsId = report.getNameserviceId();
String bpId = report.getBlockPoolId();
String cId = report.getClusterId();
+
@SuppressWarnings("unchecked")
List<MockNamenodeContext> existingItems =
- (List<MockNamenodeContext>) resolver.get(nsId);
+ (List<MockNamenodeContext>) this.resolver.get(nsId);
if (existingItems == null) {
- existingItems = new ArrayList<MockNamenodeContext>();
- resolver.put(bpId, existingItems);
- resolver.put(nsId, existingItems);
+ existingItems = new ArrayList<>();
+ this.resolver.put(bpId, existingItems);
+ this.resolver.put(nsId, existingItems);
}
boolean added = false;
for (int i=0; i<existingItems.size() && !added; i++) {
@@ -227,7 +240,7 @@ public class MockResolver
Collections.sort(existingItems, new NamenodePriorityComparator());
FederationNamespaceInfo info = new FederationNamespaceInfo(bpId, cId, nsId);
- namespaces.add(info);
+ this.namespaces.add(info);
return true;
}
@@ -238,16 +251,13 @@ public class MockResolver
@Override
public PathLocation getDestinationForPath(String path) throws IOException {
- String finalPath = null;
- String nameservice = null;
- Set<String> namespaceSet = new HashSet<String>();
- LinkedList<RemoteLocation> remoteLocations =
- new LinkedList<RemoteLocation>();
- for(String key : this.locations.keySet()) {
- if(path.startsWith(key)) {
+ Set<String> namespaceSet = new HashSet<>();
+ List<RemoteLocation> remoteLocations = new LinkedList<>();
+ for (String key : this.locations.keySet()) {
+ if (path.startsWith(key)) {
for (RemoteLocation location : this.locations.get(key)) {
- finalPath = location.getDest() + path.substring(key.length());
- nameservice = location.getNameserviceId();
+ String finalPath = location.getDest() + path.substring(key.length());
+ String nameservice = location.getNameserviceId();
RemoteLocation remoteLocation =
new RemoteLocation(nameservice, finalPath);
remoteLocations.add(remoteLocation);
@@ -265,7 +275,7 @@ public class MockResolver
@Override
public List<String> getMountPoints(String path) throws IOException {
- List<String> mounts = new ArrayList<String>();
+ List<String> mounts = new ArrayList<>();
if (path.equals("/")) {
// Mounts only supported under root level
for (String mount : this.locations.keySet()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6632c5a4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
index 16d624c..39fcf7a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.federation;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
/**
* Constructs a router configuration with individual features enabled/disabled.
@@ -26,15 +27,32 @@ public class RouterConfigBuilder {
private Configuration conf;
+ private boolean enableRpcServer = false;
+
public RouterConfigBuilder(Configuration configuration) {
this.conf = configuration;
}
public RouterConfigBuilder() {
- this.conf = new Configuration();
+ this.conf = new Configuration(false);
+ }
+
+ public RouterConfigBuilder all() {
+ this.enableRpcServer = true;
+ return this;
+ }
+
+ public RouterConfigBuilder rpc(boolean enable) {
+ this.enableRpcServer = enable;
+ return this;
+ }
+
+ public RouterConfigBuilder rpc() {
+ return this.rpc(true);
}
public Configuration build() {
+ conf.setBoolean(DFSConfigKeys.DFS_ROUTER_RPC_ENABLE, this.enableRpcServer);
return conf;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6632c5a4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
index 55d04ad..4031b7f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
@@ -17,27 +17,44 @@
*/
package org.apache.hadoop.hdfs.server.federation;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_BIND_HOST_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HANDLER_COUNT_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_RPC_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_RPC_BIND_HOST_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMENODES;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.addDirectory;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.waitNamenodeRegistered;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
import java.io.IOException;
+import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map.Entry;
import java.util.Random;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.hdfs.DFSClient;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -46,16 +63,49 @@ import org.apache.hadoop.hdfs.MiniDFSNNTopology.NNConf;
import org.apache.hadoop.hdfs.MiniDFSNNTopology.NSConf;
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport;
import org.apache.hadoop.hdfs.server.federation.router.Router;
+import org.apache.hadoop.hdfs.server.namenode.FSImage;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.Service.STATE;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
- * Test utility to mimic a federated HDFS cluster with a router.
+ * Test utility to mimic a federated HDFS cluster with multiple routers.
*/
public class RouterDFSCluster {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RouterDFSCluster.class);
+
+ public static final String TEST_STRING = "teststring";
+ public static final String TEST_DIR = "testdir";
+ public static final String TEST_FILE = "testfile";
+
+
+ /** Nameservices in the federated cluster. */
+ private List<String> nameservices;
+ /** Namenodes in the federated cluster. */
+ private List<NamenodeContext> namenodes;
+ /** Routers in the federated cluster. */
+ private List<RouterContext> routers;
+ /** If the Namenodes are in high availability.*/
+ private boolean highAvailability;
+
+ /** Mini cluster. */
+ private MiniDFSCluster cluster;
+
+ /** Router configuration overrides. */
+ private Configuration routerOverrides;
+ /** Namenode configuration overrides. */
+ private Configuration namenodeOverrides;
+
+
/**
* Router context.
*/
@@ -69,13 +119,14 @@ public class RouterDFSCluster {
private Configuration conf;
private URI fileSystemUri;
- public RouterContext(Configuration conf, String ns, String nn)
+ public RouterContext(Configuration conf, String nsId, String nnId)
throws URISyntaxException {
- this.namenodeId = nn;
- this.nameserviceId = ns;
this.conf = conf;
- router = new Router();
- router.init(conf);
+ this.nameserviceId = nsId;
+ this.namenodeId = nnId;
+
+ this.router = new Router();
+ this.router.init(conf);
}
public Router getRouter() {
@@ -99,18 +150,30 @@ public class RouterDFSCluster {
}
public void initRouter() throws URISyntaxException {
+ // Store the bound points for the router interfaces
+ InetSocketAddress rpcAddress = router.getRpcServerAddress();
+ if (rpcAddress != null) {
+ this.rpcPort = rpcAddress.getPort();
+ this.fileSystemUri =
+ URI.create("hdfs://" + NetUtils.getHostPortString(rpcAddress));
+ // Override the default FS to point to the router RPC
+ DistributedFileSystem.setDefaultUri(conf, fileSystemUri);
+ try {
+ this.fileContext = FileContext.getFileContext(conf);
+ } catch (UnsupportedFileSystemException e) {
+ this.fileContext = null;
+ }
+ }
}
- public DistributedFileSystem getFileSystem() throws IOException {
- DistributedFileSystem fs =
- (DistributedFileSystem) DistributedFileSystem.get(conf);
- return fs;
+ public FileSystem getFileSystem() throws IOException {
+ return DistributedFileSystem.get(conf);
}
public DFSClient getClient(UserGroupInformation user)
throws IOException, URISyntaxException, InterruptedException {
- LOG.info("Connecting to router at " + fileSystemUri);
+ LOG.info("Connecting to router at {}", fileSystemUri);
return user.doAs(new PrivilegedExceptionAction<DFSClient>() {
@Override
public DFSClient run() throws IOException {
@@ -120,9 +183,8 @@ public class RouterDFSCluster {
}
public DFSClient getClient() throws IOException, URISyntaxException {
-
if (client == null) {
- LOG.info("Connecting to router at " + fileSystemUri);
+ LOG.info("Connecting to router at {}", fileSystemUri);
client = new DFSClient(fileSystemUri, conf);
}
return client;
@@ -130,9 +192,10 @@ public class RouterDFSCluster {
}
/**
- * Namenode context.
+ * Namenode context in the federated cluster.
*/
public class NamenodeContext {
+ private Configuration conf;
private NameNode namenode;
private String nameserviceId;
private String namenodeId;
@@ -143,14 +206,13 @@ public class RouterDFSCluster {
private int httpPort;
private URI fileSystemUri;
private int index;
- private Configuration conf;
private DFSClient client;
- public NamenodeContext(Configuration conf, String ns, String nn,
- int index) {
+ public NamenodeContext(
+ Configuration conf, String nsId, String nnId, int index) {
this.conf = conf;
- this.namenodeId = nn;
- this.nameserviceId = ns;
+ this.nameserviceId = nsId;
+ this.namenodeId = nnId;
this.index = index;
}
@@ -170,20 +232,19 @@ public class RouterDFSCluster {
return this.fileContext;
}
- public void setNamenode(NameNode n) throws URISyntaxException {
- namenode = n;
+ public void setNamenode(NameNode nn) throws URISyntaxException {
+ this.namenode = nn;
- // Store the bound ports and override the default FS with the local NN's
- // RPC
- rpcPort = n.getNameNodeAddress().getPort();
- servicePort = n.getServiceRpcAddress().getPort();
- lifelinePort = n.getServiceRpcAddress().getPort();
- httpPort = n.getHttpAddress().getPort();
- fileSystemUri = new URI("hdfs://" + namenode.getHostAndPort());
- DistributedFileSystem.setDefaultUri(conf, fileSystemUri);
+ // Store the bound ports and override the default FS with the local NN RPC
+ this.rpcPort = nn.getNameNodeAddress().getPort();
+ this.servicePort = nn.getServiceRpcAddress().getPort();
+ this.lifelinePort = nn.getServiceRpcAddress().getPort();
+ this.httpPort = nn.getHttpAddress().getPort();
+ this.fileSystemUri = new URI("hdfs://" + namenode.getHostAndPort());
+ DistributedFileSystem.setDefaultUri(this.conf, this.fileSystemUri);
try {
- this.fileContext = FileContext.getFileContext(conf);
+ this.fileContext = FileContext.getFileContext(this.conf);
} catch (UnsupportedFileSystemException e) {
this.fileContext = null;
}
@@ -205,10 +266,8 @@ public class RouterDFSCluster {
return namenode.getHttpAddress().getHostName() + ":" + httpPort;
}
- public DistributedFileSystem getFileSystem() throws IOException {
- DistributedFileSystem fs =
- (DistributedFileSystem) DistributedFileSystem.get(conf);
- return fs;
+ public FileSystem getFileSystem() throws IOException {
+ return DistributedFileSystem.get(conf);
}
public void resetClient() {
@@ -218,7 +277,7 @@ public class RouterDFSCluster {
public DFSClient getClient(UserGroupInformation user)
throws IOException, URISyntaxException, InterruptedException {
- LOG.info("Connecting to namenode at " + fileSystemUri);
+ LOG.info("Connecting to namenode at {}", fileSystemUri);
return user.doAs(new PrivilegedExceptionAction<DFSClient>() {
@Override
public DFSClient run() throws IOException {
@@ -229,7 +288,7 @@ public class RouterDFSCluster {
public DFSClient getClient() throws IOException, URISyntaxException {
if (client == null) {
- LOG.info("Connecting to namenode at " + fileSystemUri);
+ LOG.info("Connecting to namenode at {}", fileSystemUri);
client = new DFSClient(fileSystemUri, conf);
}
return client;
@@ -244,36 +303,20 @@ public class RouterDFSCluster {
}
}
- public static final String NAMENODE1 = "nn0";
- public static final String NAMENODE2 = "nn1";
- public static final String NAMENODE3 = "nn2";
- public static final String TEST_STRING = "teststring";
- public static final String TEST_DIR = "testdir";
- public static final String TEST_FILE = "testfile";
-
- private List<String> nameservices;
- private List<RouterContext> routers;
- private List<NamenodeContext> namenodes;
- private static final Log LOG = LogFactory.getLog(RouterDFSCluster.class);
- private MiniDFSCluster cluster;
- private boolean highAvailability;
-
- protected static final int DEFAULT_HEARTBEAT_INTERVAL = 5;
- protected static final int DEFAULT_CACHE_INTERVAL_SEC = 5;
- private Configuration routerOverrides;
- private Configuration namenodeOverrides;
-
- private static final String NAMENODES = NAMENODE1 + "," + NAMENODE2;
-
- public RouterDFSCluster(boolean ha, int numNameservices) {
- this(ha, numNameservices, 2);
- }
-
public RouterDFSCluster(boolean ha, int numNameservices, int numNamenodes) {
this.highAvailability = ha;
configureNameservices(numNameservices, numNamenodes);
}
+ public RouterDFSCluster(boolean ha, int numNameservices) {
+ this(ha, numNameservices, 2);
+ }
+
+ /**
+ * Add configuration settings to override default Router settings.
+ *
+ * @param conf Router configuration overrides.
+ */
public void addRouterOverrides(Configuration conf) {
if (this.routerOverrides == null) {
this.routerOverrides = conf;
@@ -282,6 +325,11 @@ public class RouterDFSCluster {
}
}
+ /**
+ * Add configuration settings to override default Namenode settings.
+ *
+ * @param conf Namenode configuration overrides.
+ */
public void addNamenodeOverrides(Configuration conf) {
if (this.namenodeOverrides == null) {
this.namenodeOverrides = conf;
@@ -290,124 +338,134 @@ public class RouterDFSCluster {
}
}
- public Configuration generateNamenodeConfiguration(
- String defaultNameserviceId) {
- Configuration c = new HdfsConfiguration();
+ /**
+ * Generate the configuration for a client.
+ *
+ * @param nsId Nameservice identifier.
+ * @return New namenode configuration.
+ */
+ public Configuration generateNamenodeConfiguration(String nsId) {
+ Configuration conf = new HdfsConfiguration();
- c.set(DFSConfigKeys.DFS_NAMESERVICES, getNameservicesKey());
- c.set("fs.defaultFS", "hdfs://" + defaultNameserviceId);
+ conf.set(DFS_NAMESERVICES, getNameservicesKey());
+ conf.set(FS_DEFAULT_NAME_KEY, "hdfs://" + nsId);
for (String ns : nameservices) {
if (highAvailability) {
- c.set(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "." + ns, NAMENODES);
+ conf.set(
+ DFS_HA_NAMENODES_KEY_PREFIX + "." + ns,
+ NAMENODES[0] + "," + NAMENODES[1]);
}
for (NamenodeContext context : getNamenodes(ns)) {
String suffix = context.getConfSuffix();
- c.set(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + suffix,
+ conf.set(DFS_NAMENODE_RPC_ADDRESS_KEY + "." + suffix,
"127.0.0.1:" + context.rpcPort);
- c.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY + "." + suffix,
+ conf.set(DFS_NAMENODE_HTTP_ADDRESS_KEY + "." + suffix,
"127.0.0.1:" + context.httpPort);
- c.set(DFSConfigKeys.DFS_NAMENODE_RPC_BIND_HOST_KEY + "." + suffix,
+ conf.set(DFS_NAMENODE_RPC_BIND_HOST_KEY + "." + suffix,
"0.0.0.0");
}
}
- if (namenodeOverrides != null) {
- c.addResource(namenodeOverrides);
+ if (this.namenodeOverrides != null) {
+ conf.addResource(this.namenodeOverrides);
}
- return c;
+ return conf;
}
+ /**
+ * Generate the configuration for a client.
+ *
+ * @return New configuration for a client.
+ */
public Configuration generateClientConfiguration() {
- Configuration conf = new HdfsConfiguration();
- conf.addResource(generateNamenodeConfiguration(getNameservices().get(0)));
+ Configuration conf = new HdfsConfiguration(false);
+ String ns0 = getNameservices().get(0);
+ conf.addResource(generateNamenodeConfiguration(ns0));
return conf;
}
- public Configuration generateRouterConfiguration(String localNameserviceId,
- String localNamenodeId) throws IOException {
- Configuration conf = new HdfsConfiguration();
- conf.addResource(generateNamenodeConfiguration(localNameserviceId));
+ /**
+ * Generate the configuration for a Router.
+ *
+ * @param nsId Nameservice identifier.
+ * @param nnId Namenode identifier.
+ * @return New configuration for a Router.
+ */
+ public Configuration generateRouterConfiguration(String nsId, String nnId) {
+
+ Configuration conf = new HdfsConfiguration(false);
+ conf.addResource(generateNamenodeConfiguration(nsId));
+
+ conf.setInt(DFS_ROUTER_HANDLER_COUNT_KEY, 10);
+ conf.set(DFS_ROUTER_RPC_ADDRESS_KEY, "127.0.0.1:0");
+ conf.set(DFS_ROUTER_RPC_BIND_HOST_KEY, "0.0.0.0");
+
+ conf.set(DFS_ROUTER_DEFAULT_NAMESERVICE, nameservices.get(0));
// Use mock resolver classes
- conf.set(DFSConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS,
- MockResolver.class.getCanonicalName());
- conf.set(DFSConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
- MockResolver.class.getCanonicalName());
+ conf.setClass(FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS,
+ MockResolver.class, ActiveNamenodeResolver.class);
+ conf.setClass(FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
+ MockResolver.class, FileSubclusterResolver.class);
// Set the nameservice ID for the default NN monitor
- conf.set(DFSConfigKeys.DFS_NAMESERVICE_ID, localNameserviceId);
-
- if (localNamenodeId != null) {
- conf.set(DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY, localNamenodeId);
+ conf.set(DFS_NAMESERVICE_ID, nsId);
+ if (nnId != null) {
+ conf.set(DFS_HA_NAMENODE_ID_KEY, nnId);
}
- StringBuilder routerBuilder = new StringBuilder();
- for (String ns : nameservices) {
- for (NamenodeContext context : getNamenodes(ns)) {
- String suffix = context.getConfSuffix();
-
- if (routerBuilder.length() != 0) {
- routerBuilder.append(",");
- }
- routerBuilder.append(suffix);
+ // Add custom overrides if available
+ if (this.routerOverrides != null) {
+ for (Entry<String, String> entry : this.routerOverrides) {
+ String confKey = entry.getKey();
+ String confValue = entry.getValue();
+ conf.set(confKey, confValue);
}
}
-
return conf;
}
public void configureNameservices(int numNameservices, int numNamenodes) {
- nameservices = new ArrayList<String>();
- for (int i = 0; i < numNameservices; i++) {
- nameservices.add("ns" + i);
- }
- namenodes = new ArrayList<NamenodeContext>();
- int index = 0;
- for (String ns : nameservices) {
- Configuration nnConf = generateNamenodeConfiguration(ns);
- if (highAvailability) {
- NamenodeContext context =
- new NamenodeContext(nnConf, ns, NAMENODE1, index);
- namenodes.add(context);
- index++;
-
- if (numNamenodes > 1) {
- context = new NamenodeContext(nnConf, ns, NAMENODE2, index + 1);
- namenodes.add(context);
- index++;
- }
+ this.nameservices = new ArrayList<>();
+ this.namenodes = new ArrayList<>();
- if (numNamenodes > 2) {
- context = new NamenodeContext(nnConf, ns, NAMENODE3, index + 1);
- namenodes.add(context);
- index++;
- }
+ NamenodeContext context = null;
+ int nnIndex = 0;
+ for (int i=0; i<numNameservices; i++) {
+ String ns = "ns" + i;
+ this.nameservices.add("ns" + i);
+ Configuration nnConf = generateNamenodeConfiguration(ns);
+ if (!highAvailability) {
+ context = new NamenodeContext(nnConf, ns, null, nnIndex++);
+ this.namenodes.add(context);
} else {
- NamenodeContext context = new NamenodeContext(nnConf, ns, null, index);
- namenodes.add(context);
- index++;
+ for (int j=0; j<numNamenodes; j++) {
+ context = new NamenodeContext(nnConf, ns, NAMENODES[j], nnIndex++);
+ this.namenodes.add(context);
+ }
}
}
}
public String getNameservicesKey() {
- StringBuilder ns = new StringBuilder();
- for (int i = 0; i < nameservices.size(); i++) {
- if (i > 0) {
- ns.append(",");
+ StringBuilder sb = new StringBuilder();
+ for (String nsId : this.nameservices) {
+ if (sb.length() > 0) {
+ sb.append(",");
}
- ns.append(nameservices.get(i));
+ sb.append(nsId);
}
- return ns.toString();
+ return sb.toString();
}
public String getRandomNameservice() {
Random r = new Random();
- return nameservices.get(r.nextInt(nameservices.size()));
+ int randIndex = r.nextInt(nameservices.size());
+ return nameservices.get(randIndex);
}
public List<String> getNameservices() {
@@ -415,7 +473,7 @@ public class RouterDFSCluster {
}
public List<NamenodeContext> getNamenodes(String nameservice) {
- ArrayList<NamenodeContext> nns = new ArrayList<NamenodeContext>();
+ List<NamenodeContext> nns = new ArrayList<>();
for (NamenodeContext c : namenodes) {
if (c.nameserviceId.equals(nameservice)) {
nns.add(c);
@@ -426,23 +484,23 @@ public class RouterDFSCluster {
public NamenodeContext getRandomNamenode() {
Random rand = new Random();
- return namenodes.get(rand.nextInt(namenodes.size()));
+ int i = rand.nextInt(this.namenodes.size());
+ return this.namenodes.get(i);
}
public List<NamenodeContext> getNamenodes() {
- return namenodes;
+ return this.namenodes;
}
public boolean isHighAvailability() {
return highAvailability;
}
- public NamenodeContext getNamenode(String nameservice,
- String namenode) {
- for (NamenodeContext c : namenodes) {
+ public NamenodeContext getNamenode(String nameservice, String namenode) {
+ for (NamenodeContext c : this.namenodes) {
if (c.nameserviceId.equals(nameservice)) {
- if (namenode == null || c.namenodeId == null || namenode.isEmpty()
- || c.namenodeId.isEmpty()) {
+ if (namenode == null || namenode.isEmpty() ||
+ c.namenodeId == null || c.namenodeId.isEmpty()) {
return c;
} else if (c.namenodeId.equals(namenode)) {
return c;
@@ -453,7 +511,7 @@ public class RouterDFSCluster {
}
public List<RouterContext> getRouters(String nameservice) {
- ArrayList<RouterContext> nns = new ArrayList<RouterContext>();
+ List<RouterContext> nns = new ArrayList<>();
for (RouterContext c : routers) {
if (c.nameserviceId.equals(nameservice)) {
nns.add(c);
@@ -462,14 +520,13 @@ public class RouterDFSCluster {
return nns;
}
- public RouterContext getRouterContext(String nameservice,
- String namenode) {
+ public RouterContext getRouterContext(String nsId, String nnId) {
for (RouterContext c : routers) {
- if (namenode == null) {
+ if (nnId == null) {
return c;
}
- if (c.namenodeId.equals(namenode)
- && c.nameserviceId.equals(nameservice)) {
+ if (c.namenodeId.equals(nnId) &&
+ c.nameserviceId.equals(nsId)) {
return c;
}
}
@@ -485,10 +542,10 @@ public class RouterDFSCluster {
return routers;
}
- public RouterContext buildRouter(String nameservice, String namenode)
+ public RouterContext buildRouter(String nsId, String nnId)
throws URISyntaxException, IOException {
- Configuration config = generateRouterConfiguration(nameservice, namenode);
- RouterContext rc = new RouterContext(config, nameservice, namenode);
+ Configuration config = generateRouterConfiguration(nsId, nnId);
+ RouterContext rc = new RouterContext(config, nsId, nnId);
return rc;
}
@@ -500,10 +557,9 @@ public class RouterDFSCluster {
try {
MiniDFSNNTopology topology = new MiniDFSNNTopology();
for (String ns : nameservices) {
-
NSConf conf = new MiniDFSNNTopology.NSConf(ns);
if (highAvailability) {
- for(int i = 0; i < namenodes.size()/nameservices.size(); i++) {
+ for (int i=0; i<namenodes.size()/nameservices.size(); i++) {
NNConf nnConf = new MiniDFSNNTopology.NNConf("nn" + i);
conf.addNN(nnConf);
}
@@ -516,11 +572,15 @@ public class RouterDFSCluster {
topology.setFederation(true);
// Start mini DFS cluster
- Configuration nnConf = generateNamenodeConfiguration(nameservices.get(0));
+ String ns0 = nameservices.get(0);
+ Configuration nnConf = generateNamenodeConfiguration(ns0);
if (overrideConf != null) {
nnConf.addResource(overrideConf);
}
- cluster = new MiniDFSCluster.Builder(nnConf).nnTopology(topology).build();
+ cluster = new MiniDFSCluster.Builder(nnConf)
+ .numDataNodes(nameservices.size()*2)
+ .nnTopology(topology)
+ .build();
cluster.waitActive();
// Store NN pointers
@@ -530,28 +590,32 @@ public class RouterDFSCluster {
}
} catch (Exception e) {
- LOG.error("Cannot start Router DFS cluster: " + e.getMessage(), e);
- cluster.shutdown();
+ LOG.error("Cannot start Router DFS cluster: {}", e.getMessage(), e);
+ if (cluster != null) {
+ cluster.shutdown();
+ }
}
}
public void startRouters()
throws InterruptedException, URISyntaxException, IOException {
- // Create routers
- routers = new ArrayList<RouterContext>();
- for (String ns : nameservices) {
+ // Create one router per nameservice
+ this.routers = new ArrayList<>();
+ for (String ns : this.nameservices) {
for (NamenodeContext context : getNamenodes(ns)) {
- routers.add(buildRouter(ns, context.namenodeId));
+ RouterContext router = buildRouter(ns, context.namenodeId);
+ this.routers.add(router);
}
}
// Start all routers
- for (RouterContext router : routers) {
+ for (RouterContext router : this.routers) {
router.router.start();
}
+
// Wait until all routers are active and record their ports
- for (RouterContext router : routers) {
+ for (RouterContext router : this.routers) {
waitActive(router);
router.initRouter();
}
@@ -570,22 +634,21 @@ public class RouterDFSCluster {
}
Thread.sleep(1000);
}
- assertFalse(
- "Timeout waiting for " + router.router.toString() + " to activate.",
- true);
+ fail("Timeout waiting for " + router.router + " to activate");
}
-
public void registerNamenodes() throws IOException {
- for (RouterContext r : routers) {
+ for (RouterContext r : this.routers) {
ActiveNamenodeResolver resolver = r.router.getNamenodeResolver();
- for (NamenodeContext nn : namenodes) {
+ for (NamenodeContext nn : this.namenodes) {
// Generate a report
- NamenodeStatusReport report = new NamenodeStatusReport(nn.nameserviceId,
- nn.namenodeId, nn.getRpcAddress(), nn.getServiceAddress(),
+ NamenodeStatusReport report = new NamenodeStatusReport(
+ nn.nameserviceId, nn.namenodeId,
+ nn.getRpcAddress(), nn.getServiceAddress(),
nn.getLifelineAddress(), nn.getHttpAddress());
- report.setNamespaceInfo(nn.namenode.getNamesystem().getFSImage()
- .getStorage().getNamespaceInfo());
+ FSImage fsImage = nn.namenode.getNamesystem().getFSImage();
+ NamespaceInfo nsInfo = fsImage.getStorage().getNamespaceInfo();
+ report.setNamespaceInfo(nsInfo);
// Determine HA state from nn public state string
String nnState = nn.namenode.getState();
@@ -606,74 +669,97 @@ public class RouterDFSCluster {
public void waitNamenodeRegistration()
throws InterruptedException, IllegalStateException, IOException {
- for (RouterContext r : routers) {
- for (NamenodeContext nn : namenodes) {
- FederationTestUtils.waitNamenodeRegistered(
- r.router.getNamenodeResolver(), nn.nameserviceId, nn.namenodeId,
- null);
+ for (RouterContext r : this.routers) {
+ Router router = r.router;
+ for (NamenodeContext nn : this.namenodes) {
+ ActiveNamenodeResolver nnResolver = router.getNamenodeResolver();
+ waitNamenodeRegistered(
+ nnResolver, nn.nameserviceId, nn.namenodeId, null);
}
}
}
public void waitRouterRegistrationQuorum(RouterContext router,
- FederationNamenodeServiceState state, String nameservice, String namenode)
+ FederationNamenodeServiceState state, String nsId, String nnId)
throws InterruptedException, IOException {
- LOG.info("Waiting for NN - " + nameservice + ":" + namenode
- + " to transition to state - " + state);
- FederationTestUtils.waitNamenodeRegistered(
- router.router.getNamenodeResolver(), nameservice, namenode, state);
+ LOG.info("Waiting for NN {} {} to transition to {}", nsId, nnId, state);
+ ActiveNamenodeResolver nnResolver = router.router.getNamenodeResolver();
+ waitNamenodeRegistered(nnResolver, nsId, nnId, state);
}
- public String getFederatedPathForNameservice(String ns) {
- return "/" + ns;
+ /**
+ * Get the federated path for a nameservice.
+ * @param nsId Nameservice identifier.
+ * @return Path in the Router.
+ */
+ public String getFederatedPathForNS(String nsId) {
+ return "/" + nsId;
}
- public String getNamenodePathForNameservice(String ns) {
- return "/target-" + ns;
+ /**
+ * Get the namenode path for a nameservice.
+ * @param nsId Nameservice identifier.
+ * @return Path in the Namenode.
+ */
+ public String getNamenodePathForNS(String nsId) {
+ return "/target-" + nsId;
}
/**
- * @return example:
+ * Get the federated test directory for a nameservice.
+ * @param nsId Nameservice identifier.
+ * @return Example:
* <ul>
* <li>/ns0/testdir which maps to ns0->/target-ns0/testdir
* </ul>
*/
- public String getFederatedTestDirectoryForNameservice(String ns) {
- return getFederatedPathForNameservice(ns) + "/" + TEST_DIR;
+ public String getFederatedTestDirectoryForNS(String nsId) {
+ return getFederatedPathForNS(nsId) + "/" + TEST_DIR;
}
/**
+ * Get the namenode test directory for a nameservice.
+ * @param nsId Nameservice identifier.
* @return example:
* <ul>
* <li>/target-ns0/testdir
* </ul>
*/
- public String getNamenodeTestDirectoryForNameservice(String ns) {
- return getNamenodePathForNameservice(ns) + "/" + TEST_DIR;
+ public String getNamenodeTestDirectoryForNS(String nsId) {
+ return getNamenodePathForNS(nsId) + "/" + TEST_DIR;
}
/**
+ * Get the federated test file for a nameservice.
+ * @param nsId Nameservice identifier.
* @return example:
* <ul>
* <li>/ns0/testfile which maps to ns0->/target-ns0/testfile
* </ul>
*/
- public String getFederatedTestFileForNameservice(String ns) {
- return getFederatedPathForNameservice(ns) + "/" + TEST_FILE;
+ public String getFederatedTestFileForNS(String nsId) {
+ return getFederatedPathForNS(nsId) + "/" + TEST_FILE;
}
/**
+ * Get the namenode test file for a nameservice.
+ * @param nsId Nameservice identifier.
* @return example:
* <ul>
* <li>/target-ns0/testfile
* </ul>
*/
- public String getNamenodeTestFileForNameservice(String ns) {
- return getNamenodePathForNameservice(ns) + "/" + TEST_FILE;
+ public String getNamenodeTestFileForNS(String nsId) {
+ return getNamenodePathForNS(nsId) + "/" + TEST_FILE;
}
+ /**
+ * Stop the federated HDFS cluster.
+ */
public void shutdown() {
- cluster.shutdown();
+ if (cluster != null) {
+ cluster.shutdown();
+ }
if (routers != null) {
for (RouterContext context : routers) {
stopRouter(context);
@@ -681,9 +767,12 @@ public class RouterDFSCluster {
}
}
+ /**
+ * Stop a router.
+ * @param router Router context.
+ */
public void stopRouter(RouterContext router) {
try {
-
router.router.shutDown();
int loopCount = 0;
@@ -691,7 +780,7 @@ public class RouterDFSCluster {
loopCount++;
Thread.sleep(1000);
if (loopCount > 20) {
- LOG.error("Unable to shutdown router - " + router.rpcPort);
+ LOG.error("Cannot shutdown router {}", router.rpcPort);
break;
}
}
@@ -714,26 +803,28 @@ public class RouterDFSCluster {
for (String ns : getNameservices()) {
NamenodeContext context = getNamenode(ns, null);
if (!createTestDirectoriesNamenode(context)) {
- throw new IOException("Unable to create test directory for ns - " + ns);
+ throw new IOException("Cannot create test directory for ns " + ns);
}
}
}
public boolean createTestDirectoriesNamenode(NamenodeContext nn)
throws IOException {
- return FederationTestUtils.addDirectory(nn.getFileSystem(),
- getNamenodeTestDirectoryForNameservice(nn.nameserviceId));
+ FileSystem fs = nn.getFileSystem();
+ String testDir = getNamenodeTestDirectoryForNS(nn.nameserviceId);
+ return addDirectory(fs, testDir);
}
public void deleteAllFiles() throws IOException {
// Delete all files via the NNs and verify
for (NamenodeContext context : getNamenodes()) {
- FileStatus[] status = context.getFileSystem().listStatus(new Path("/"));
- for(int i = 0; i <status.length; i++) {
+ FileSystem fs = context.getFileSystem();
+ FileStatus[] status = fs.listStatus(new Path("/"));
+ for (int i = 0; i <status.length; i++) {
Path p = status[i].getPath();
- context.getFileSystem().delete(p, true);
+ fs.delete(p, true);
}
- status = context.getFileSystem().listStatus(new Path("/"));
+ status = fs.listStatus(new Path("/"));
assertEquals(status.length, 0);
}
}
@@ -754,14 +845,34 @@ public class RouterDFSCluster {
MockResolver resolver =
(MockResolver) r.router.getSubclusterResolver();
// create table entries
- for (String ns : nameservices) {
+ for (String nsId : nameservices) {
// Direct path
- resolver.addLocation(getFederatedPathForNameservice(ns), ns,
- getNamenodePathForNameservice(ns));
+ String routerPath = getFederatedPathForNS(nsId);
+ String nnPath = getNamenodePathForNS(nsId);
+ resolver.addLocation(routerPath, nsId, nnPath);
}
- // Root path goes to both NS1
- resolver.addLocation("/", nameservices.get(0), "/");
+ // Root path points to both first nameservice
+ String ns0 = nameservices.get(0);
+ resolver.addLocation("/", ns0, "/");
+ }
+ }
+
+ public MiniDFSCluster getCluster() {
+ return cluster;
+ }
+
+ /**
+ * Wait until the federated cluster is up and ready.
+ * @throws IOException If we cannot wait for the cluster to be up.
+ */
+ public void waitClusterUp() throws IOException {
+ cluster.waitClusterUp();
+ registerNamenodes();
+ try {
+ waitNamenodeRegistration();
+ } catch (Exception e) {
+ throw new IOException("Cannot wait for the namenodes", e);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6632c5a4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
index 8c720c7..d8afb39 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.federation.router;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import java.io.IOException;
import java.net.URISyntaxException;
@@ -51,6 +52,10 @@ public class TestRouter {
conf.set(DFSConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
MockResolver.class.getCanonicalName());
+ // Bind to any available port
+ conf.set(DFSConfigKeys.DFS_ROUTER_RPC_BIND_HOST_KEY, "0.0.0.0");
+ conf.set(DFSConfigKeys.DFS_ROUTER_RPC_ADDRESS_KEY, "127.0.0.1:0");
+
// Simulate a co-located NN
conf.set(DFSConfigKeys.DFS_NAMESERVICES, "ns0");
conf.set("fs.defaultFS", "hdfs://" + "ns0");
@@ -90,7 +95,31 @@ public class TestRouter {
@Test
public void testRouterService() throws InterruptedException, IOException {
+ // Rpc only
+ testRouterStartup(new RouterConfigBuilder(conf).rpc().build());
+
// Run with all services
- testRouterStartup((new RouterConfigBuilder(conf)).build());
+ testRouterStartup(new RouterConfigBuilder(conf).all().build());
+ }
+
+ @Test
+ public void testRouterRestartRpcService() throws IOException {
+
+ // Start
+ Router router = new Router();
+ router.init(new RouterConfigBuilder(conf).rpc().build());
+ router.start();
+
+ // Verify RPC server is running
+ assertNotNull(router.getRpcServerAddress());
+ RouterRpcServer rpcServer = router.getRpcServer();
+ assertNotNull(rpcServer);
+ assertEquals(STATE.STARTED, rpcServer.getServiceState());
+
+ // Stop router and RPC server
+ router.stop();
+ assertEquals(STATE.STOPPED, rpcServer.getServiceState());
+
+ router.close();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6632c5a4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
new file mode 100644
index 0000000..af506c9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
@@ -0,0 +1,869 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.addDirectory;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.countContents;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createFile;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.deleteFile;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.getFileStatus;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyFileExists;
+import static org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.TEST_STRING;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.CryptoProtocolVersion;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Options;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
+import org.apache.hadoop.io.EnumSetWritable;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.Service.STATE;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * The the RPC interface of the {@link Router} implemented by
+ * {@link RouterRpcServer}.
+ */
+public class TestRouterRpc {
+
+ /** Federated HDFS cluster. */
+ private static RouterDFSCluster cluster;
+
+ /** Random Router for this federated cluster. */
+ private RouterContext router;
+
+ /** Random nameservice in the federated cluster. */
+ private String ns;
+ /** First namenode in the nameservice. */
+ private NamenodeContext namenode;
+
+ /** Client interface to the Router. */
+ private ClientProtocol routerProtocol;
+ /** Client interface to the Namenode. */
+ private ClientProtocol nnProtocol;
+
+ /** Filesystem interface to the Router. */
+ private FileSystem routerFS;
+ /** Filesystem interface to the Namenode. */
+ private FileSystem nnFS;
+
+ /** File in the Router. */
+ private String routerFile;
+ /** File in the Namenode. */
+ private String nnFile;
+
+
+ @BeforeClass
+ public static void globalSetUp() throws Exception {
+ cluster = new RouterDFSCluster(false, 2);
+
+ // Start NNs and DNs and wait until ready
+ cluster.startCluster();
+
+ // Start routers with only an RPC service
+ cluster.addRouterOverrides((new RouterConfigBuilder()).rpc().build());
+ cluster.startRouters();
+
+ // Register and verify all NNs with all routers
+ cluster.registerNamenodes();
+ cluster.waitNamenodeRegistration();
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ cluster.shutdown();
+ }
+
+ @Before
+ public void testSetup() throws Exception {
+
+ // Create mock locations
+ cluster.installMockLocations();
+
+ // Delete all files via the NNs and verify
+ cluster.deleteAllFiles();
+
+ // Create test fixtures on NN
+ cluster.createTestDirectoriesNamenode();
+
+ // Wait to ensure NN has fully created its test directories
+ Thread.sleep(100);
+
+ // Pick a NS, namenode and router for this test
+ this.router = cluster.getRandomRouter();
+ this.ns = cluster.getRandomNameservice();
+ this.namenode = cluster.getNamenode(ns, null);
+
+ // Handles to the ClientProtocol interface
+ this.routerProtocol = router.getClient().getNamenode();
+ this.nnProtocol = namenode.getClient().getNamenode();
+
+ // Handles to the filesystem client
+ this.nnFS = namenode.getFileSystem();
+ this.routerFS = router.getFileSystem();
+
+ // Create a test file on the NN
+ Random r = new Random();
+ String randomFile = "testfile-" + r.nextInt();
+ this.nnFile =
+ cluster.getNamenodeTestDirectoryForNS(ns) + "/" + randomFile;
+ this.routerFile =
+ cluster.getFederatedTestDirectoryForNS(ns) + "/" + randomFile;
+
+ createFile(nnFS, nnFile, 32);
+ verifyFileExists(nnFS, nnFile);
+ }
+
+ @Test
+ public void testRpcService() throws IOException {
+ Router testRouter = new Router();
+ List<String> nss = cluster.getNameservices();
+ String ns0 = nss.get(0);
+ Configuration routerConfig = cluster.generateRouterConfiguration(ns0, null);
+ RouterRpcServer server = new RouterRpcServer(routerConfig, testRouter,
+ testRouter.getNamenodeResolver(), testRouter.getSubclusterResolver());
+ server.init(routerConfig);
+ assertEquals(STATE.INITED, server.getServiceState());
+ server.start();
+ assertEquals(STATE.STARTED, server.getServiceState());
+ server.stop();
+ assertEquals(STATE.STOPPED, server.getServiceState());
+ server.close();
+ testRouter.close();
+ }
+
+ protected RouterDFSCluster getCluster() {
+ return TestRouterRpc.cluster;
+ }
+
+ protected RouterContext getRouterContext() {
+ return this.router;
+ }
+
+ protected void setRouter(RouterContext r)
+ throws IOException, URISyntaxException {
+ this.router = r;
+ this.routerProtocol = r.getClient().getNamenode();
+ this.routerFS = r.getFileSystem();
+ }
+
+ protected FileSystem getRouterFileSystem() {
+ return this.routerFS;
+ }
+
+ protected FileSystem getNamenodeFileSystem() {
+ return this.nnFS;
+ }
+
+ protected ClientProtocol getRouterProtocol() {
+ return this.routerProtocol;
+ }
+
+ protected ClientProtocol getNamenodeProtocol() {
+ return this.nnProtocol;
+ }
+
+ protected NamenodeContext getNamenode() {
+ return this.namenode;
+ }
+
+ protected void setNamenodeFile(String filename) {
+ this.nnFile = filename;
+ }
+
+ protected String getNamenodeFile() {
+ return this.nnFile;
+ }
+
+ protected void setRouterFile(String filename) {
+ this.routerFile = filename;
+ }
+
+ protected String getRouterFile() {
+ return this.routerFile;
+ }
+
+ protected void setNamenode(NamenodeContext nn)
+ throws IOException, URISyntaxException {
+ this.namenode = nn;
+ this.nnProtocol = nn.getClient().getNamenode();
+ this.nnFS = nn.getFileSystem();
+ }
+
+ protected String getNs() {
+ return this.ns;
+ }
+
+ protected void setNs(String nameservice) {
+ this.ns = nameservice;
+ }
+
+ protected static void compareResponses(
+ ClientProtocol protocol1, ClientProtocol protocol2,
+ Method m, Object[] paramList) {
+
+ Object return1 = null;
+ Exception exception1 = null;
+ try {
+ return1 = m.invoke(protocol1, paramList);
+ } catch (Exception ex) {
+ exception1 = ex;
+ }
+
+ Object return2 = null;
+ Exception exception2 = null;
+ try {
+ return2 = m.invoke(protocol2, paramList);
+ } catch (Exception ex) {
+ exception2 = ex;
+ }
+
+ assertEquals(return1, return2);
+ if (exception1 == null && exception2 == null) {
+ return;
+ }
+
+ assertEquals(
+ exception1.getCause().getClass(),
+ exception2.getCause().getClass());
+ }
+
+ @Test
+ public void testProxyListFiles() throws IOException, InterruptedException,
+ URISyntaxException, NoSuchMethodException, SecurityException {
+
+ // Verify that the root listing is a union of the mount table destinations
+ // and the files stored at all nameservices mounted at the root (ns0 + ns1)
+ //
+ // / -->
+ // /ns0 (from mount table)
+ // /ns1 (from mount table)
+ // all items in / of ns0 (default NS)
+
+ // Collect the mount table entries from the root mount point
+ Set<String> requiredPaths = new TreeSet<>();
+ FileSubclusterResolver fileResolver =
+ router.getRouter().getSubclusterResolver();
+ for (String mount : fileResolver.getMountPoints("/")) {
+ requiredPaths.add(mount);
+ }
+
+ // Collect all files/dirs on the root path of the default NS
+ String defaultNs = cluster.getNameservices().get(0);
+ NamenodeContext nn = cluster.getNamenode(defaultNs, null);
+ FileStatus[] iterator = nn.getFileSystem().listStatus(new Path("/"));
+ for (FileStatus file : iterator) {
+ requiredPaths.add(file.getPath().getName());
+ }
+
+ // Fetch listing
+ DirectoryListing listing =
+ routerProtocol.getListing("/", HdfsFileStatus.EMPTY_NAME, false);
+ Iterator<String> requiredPathsIterator = requiredPaths.iterator();
+ // Match each path returned and verify order returned
+ for(HdfsFileStatus f : listing.getPartialListing()) {
+ String fileName = requiredPathsIterator.next();
+ String currentFile = f.getFullPath(new Path("/")).getName();
+ assertEquals(currentFile, fileName);
+ }
+
+ // Verify the total number of results found/matched
+ assertEquals(requiredPaths.size(), listing.getPartialListing().length);
+
+ // List a path that doesn't exist and validate error response with NN
+ // behavior.
+ Method m = ClientProtocol.class.getMethod(
+ "getListing", String.class, byte[].class, boolean.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(routerProtocol, nnProtocol, m,
+ new Object[] {badPath, HdfsFileStatus.EMPTY_NAME, false});
+ }
+
+ @Test
+ public void testProxyListFilesWithConflict()
+ throws IOException, InterruptedException {
+
+ // Add a directory to the namespace that conflicts with a mount point
+ NamenodeContext nn = cluster.getNamenode(ns, null);
+ FileSystem nnFs = nn.getFileSystem();
+ addDirectory(nnFs, cluster.getFederatedTestDirectoryForNS(ns));
+
+ FileSystem routerFs = router.getFileSystem();
+ int initialCount = countContents(routerFs, "/");
+
+ // Root file system now for NS X:
+ // / ->
+ // /ns0 (mount table)
+ // /ns1 (mount table)
+ // /target-ns0 (the target folder for the NS0 mapped to /
+ // /nsX (local directory that duplicates mount table)
+ int newCount = countContents(routerFs, "/");
+ assertEquals(initialCount, newCount);
+
+ // Verify that each root path is readable and contains one test directory
+ assertEquals(1, countContents(routerFs, cluster.getFederatedPathForNS(ns)));
+
+ // Verify that real folder for the ns contains a single test directory
+ assertEquals(1, countContents(nnFs, cluster.getNamenodePathForNS(ns)));
+
+ }
+
+ protected void testRename(RouterContext testRouter, String filename,
+ String renamedFile, boolean exceptionExpected) throws IOException {
+
+ createFile(testRouter.getFileSystem(), filename, 32);
+ // verify
+ verifyFileExists(testRouter.getFileSystem(), filename);
+ // rename
+ boolean exceptionThrown = false;
+ try {
+ DFSClient client = testRouter.getClient();
+ ClientProtocol clientProtocol = client.getNamenode();
+ clientProtocol.rename(filename, renamedFile);
+ } catch (Exception ex) {
+ exceptionThrown = true;
+ }
+ if (exceptionExpected) {
+ // Error was expected
+ assertTrue(exceptionThrown);
+ FileContext fileContext = testRouter.getFileContext();
+ assertTrue(fileContext.delete(new Path(filename), true));
+ } else {
+ // No error was expected
+ assertFalse(exceptionThrown);
+ // verify
+ assertTrue(verifyFileExists(testRouter.getFileSystem(), renamedFile));
+ // delete
+ FileContext fileContext = testRouter.getFileContext();
+ assertTrue(fileContext.delete(new Path(renamedFile), true));
+ }
+ }
+
+ protected void testRename2(RouterContext testRouter, String filename,
+ String renamedFile, boolean exceptionExpected) throws IOException {
+ createFile(testRouter.getFileSystem(), filename, 32);
+ // verify
+ verifyFileExists(testRouter.getFileSystem(), filename);
+ // rename
+ boolean exceptionThrown = false;
+ try {
+ DFSClient client = testRouter.getClient();
+ ClientProtocol clientProtocol = client.getNamenode();
+ clientProtocol.rename2(filename, renamedFile, new Options.Rename[] {});
+ } catch (Exception ex) {
+ exceptionThrown = true;
+ }
+ assertEquals(exceptionExpected, exceptionThrown);
+ if (exceptionExpected) {
+ // Error was expected
+ FileContext fileContext = testRouter.getFileContext();
+ assertTrue(fileContext.delete(new Path(filename), true));
+ } else {
+ // verify
+ assertTrue(verifyFileExists(testRouter.getFileSystem(), renamedFile));
+ // delete
+ FileContext fileContext = testRouter.getFileContext();
+ assertTrue(fileContext.delete(new Path(renamedFile), true));
+ }
+ }
+
+ @Test
+ public void testProxyRenameFiles() throws IOException, InterruptedException {
+
+ Thread.sleep(5000);
+ List<String> nss = cluster.getNameservices();
+ String ns0 = nss.get(0);
+ String ns1 = nss.get(1);
+
+ // Rename within the same namespace
+ // /ns0/testdir/testrename -> /ns0/testdir/testrename-append
+ String filename =
+ cluster.getFederatedTestDirectoryForNS(ns0) + "/testrename";
+ String renamedFile = filename + "-append";
+ testRename(router, filename, renamedFile, false);
+ testRename2(router, filename, renamedFile, false);
+
+ // Rename a file to a destination that is in a different namespace (fails)
+ filename = cluster.getFederatedTestDirectoryForNS(ns0) + "/testrename";
+ renamedFile = cluster.getFederatedTestDirectoryForNS(ns1) + "/testrename";
+ testRename(router, filename, renamedFile, true);
+ testRename2(router, filename, renamedFile, true);
+ }
+
+ @Test
+ public void testProxyChownFiles() throws Exception {
+
+ String newUsername = "TestUser";
+ String newGroup = "TestGroup";
+
+ // change owner
+ routerProtocol.setOwner(routerFile, newUsername, newGroup);
+
+ // Verify with NN
+ FileStatus file = getFileStatus(namenode.getFileSystem(), nnFile);
+ assertEquals(file.getOwner(), newUsername);
+ assertEquals(file.getGroup(), newGroup);
+
+ // Bad request and validate router response matches NN response.
+ Method m = ClientProtocol.class.getMethod("setOwner", String.class,
+ String.class, String.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(routerProtocol, nnProtocol, m,
+ new Object[] {badPath, newUsername, newGroup});
+ }
+
+ @Test
+ public void testProxyGetStats() throws Exception {
+
+ long[] combinedData = routerProtocol.getStats();
+
+ long[] individualData = new long[10];
+ for (String nameservice : cluster.getNameservices()) {
+ NamenodeContext n = cluster.getNamenode(nameservice, null);
+ DFSClient client = n.getClient();
+ ClientProtocol clientProtocol = client.getNamenode();
+ long[] data = clientProtocol.getStats();
+ for (int i = 0; i < data.length; i++) {
+ individualData[i] += data[i];
+ }
+ assert(data.length == combinedData.length);
+ }
+
+ for (int i = 0; i < combinedData.length && i < individualData.length; i++) {
+ if (i == ClientProtocol.GET_STATS_REMAINING_IDX) {
+ // Skip available storage as this fluctuates in mini cluster
+ continue;
+ }
+ assertEquals(combinedData[i], individualData[i]);
+ }
+ }
+
+ @Test
+ public void testProxyGetDatanodeReport() throws Exception {
+
+ DatanodeInfo[] combinedData =
+ routerProtocol.getDatanodeReport(DatanodeReportType.ALL);
+
+ Set<Integer> individualData = new HashSet<Integer>();
+ for (String nameservice : cluster.getNameservices()) {
+ NamenodeContext n = cluster.getNamenode(nameservice, null);
+ DFSClient client = n.getClient();
+ ClientProtocol clientProtocol = client.getNamenode();
+ DatanodeInfo[] data =
+ clientProtocol.getDatanodeReport(DatanodeReportType.ALL);
+ for (int i = 0; i < data.length; i++) {
+ // Collect unique DNs based on their xfer port
+ DatanodeInfo info = data[i];
+ individualData.add(info.getXferPort());
+ }
+ }
+ assertEquals(combinedData.length, individualData.size());
+ }
+
+ @Test
+ public void testProxyGetDatanodeStorageReport()
+ throws IOException, InterruptedException, URISyntaxException {
+
+ DatanodeStorageReport[] combinedData =
+ routerProtocol.getDatanodeStorageReport(DatanodeReportType.ALL);
+
+ Set<String> individualData = new HashSet<>();
+ for (String nameservice : cluster.getNameservices()) {
+ NamenodeContext n = cluster.getNamenode(nameservice, null);
+ DFSClient client = n.getClient();
+ ClientProtocol clientProtocol = client.getNamenode();
+ DatanodeStorageReport[] data =
+ clientProtocol.getDatanodeStorageReport(DatanodeReportType.ALL);
+ for (DatanodeStorageReport report : data) {
+ // Determine unique DN instances
+ DatanodeInfo dn = report.getDatanodeInfo();
+ individualData.add(dn.toString());
+ }
+ }
+ assertEquals(combinedData.length, individualData.size());
+ }
+
+ @Test
+ public void testProxyMkdir() throws Exception {
+
+ // Check the initial folders
+ FileStatus[] filesInitial = routerFS.listStatus(new Path("/"));
+
+ // Create a directory via the router at the root level
+ String dirPath = "/testdir";
+ FsPermission permission = new FsPermission("705");
+ routerProtocol.mkdirs(dirPath, permission, false);
+
+ // Verify the root listing has the item via the router
+ FileStatus[] files = routerFS.listStatus(new Path("/"));
+ assertEquals(Arrays.toString(files) + " should be " +
+ Arrays.toString(filesInitial) + " + " + dirPath,
+ filesInitial.length + 1, files.length);
+ assertTrue(verifyFileExists(routerFS, dirPath));
+
+ // Verify the directory is present in only 1 Namenode
+ int foundCount = 0;
+ for (NamenodeContext n : cluster.getNamenodes()) {
+ if (verifyFileExists(n.getFileSystem(), dirPath)) {
+ foundCount++;
+ }
+ }
+ assertEquals(1, foundCount);
+ assertTrue(deleteFile(routerFS, dirPath));
+
+ // Validate router failure response matches NN failure response.
+ Method m = ClientProtocol.class.getMethod("mkdirs", String.class,
+ FsPermission.class, boolean.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(routerProtocol, nnProtocol, m,
+ new Object[] {badPath, permission, false});
+ }
+
+ @Test
+ public void testProxyChmodFiles() throws Exception {
+
+ FsPermission permission = new FsPermission("444");
+
+ // change permissions
+ routerProtocol.setPermission(routerFile, permission);
+
+ // Validate permissions NN
+ FileStatus file = getFileStatus(namenode.getFileSystem(), nnFile);
+ assertEquals(permission, file.getPermission());
+
+ // Validate router failure response matches NN failure response.
+ Method m = ClientProtocol.class.getMethod(
+ "setPermission", String.class, FsPermission.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(routerProtocol, nnProtocol, m,
+ new Object[] {badPath, permission});
+ }
+
+ @Test
+ public void testProxySetReplication() throws Exception {
+
+ // Check current replication via NN
+ FileStatus file = getFileStatus(nnFS, nnFile);
+ assertEquals(1, file.getReplication());
+
+ // increment replication via router
+ routerProtocol.setReplication(routerFile, (short) 2);
+
+ // Verify via NN
+ file = getFileStatus(nnFS, nnFile);
+ assertEquals(2, file.getReplication());
+
+ // Validate router failure response matches NN failure response.
+ Method m = ClientProtocol.class.getMethod(
+ "setReplication", String.class, short.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(routerProtocol, nnProtocol, m,
+ new Object[] {badPath, (short) 2});
+ }
+
+ @Test
+ public void testProxyTruncateFile() throws Exception {
+
+ // Check file size via NN
+ FileStatus file = getFileStatus(nnFS, nnFile);
+ assertTrue(file.getLen() > 0);
+
+ // Truncate to 0 bytes via router
+ routerProtocol.truncate(routerFile, 0, "testclient");
+
+ // Verify via NN
+ file = getFileStatus(nnFS, nnFile);
+ assertEquals(0, file.getLen());
+
+ // Validate router failure response matches NN failure response.
+ Method m = ClientProtocol.class.getMethod(
+ "truncate", String.class, long.class, String.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(routerProtocol, nnProtocol, m,
+ new Object[] {badPath, (long) 0, "testclient"});
+ }
+
+ @Test
+ public void testProxyGetBlockLocations() throws Exception {
+
+ // Fetch block locations via router
+ LocatedBlocks locations =
+ routerProtocol.getBlockLocations(routerFile, 0, 1024);
+ assertEquals(1, locations.getLocatedBlocks().size());
+
+ // Validate router failure response matches NN failure response.
+ Method m = ClientProtocol.class.getMethod(
+ "getBlockLocations", String.class, long.class, long.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(routerProtocol, nnProtocol,
+ m, new Object[] {badPath, (long) 0, (long) 0});
+ }
+
+ @Test
+ public void testProxyStoragePolicy() throws Exception {
+
+ // Query initial policy via NN
+ HdfsFileStatus status = namenode.getClient().getFileInfo(nnFile);
+
+ // Set a random policy via router
+ BlockStoragePolicy[] policies = namenode.getClient().getStoragePolicies();
+ BlockStoragePolicy policy = policies[0];
+
+ while (policy.isCopyOnCreateFile()) {
+ // Pick a non copy on create policy
+ Random rand = new Random();
+ int randIndex = rand.nextInt(policies.length);
+ policy = policies[randIndex];
+ }
+ routerProtocol.setStoragePolicy(routerFile, policy.getName());
+
+ // Verify policy via NN
+ HdfsFileStatus newStatus = namenode.getClient().getFileInfo(nnFile);
+ assertTrue(newStatus.getStoragePolicy() == policy.getId());
+ assertTrue(newStatus.getStoragePolicy() != status.getStoragePolicy());
+
+ // Validate router failure response matches NN failure response.
+ Method m = ClientProtocol.class.getMethod("setStoragePolicy", String.class,
+ String.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(routerProtocol, nnProtocol,
+ m, new Object[] {badPath, "badpolicy"});
+ }
+
+ @Test
+ public void testProxyGetPreferedBlockSize() throws Exception {
+
+ // Query via NN and Router and verify
+ long namenodeSize = nnProtocol.getPreferredBlockSize(nnFile);
+ long routerSize = routerProtocol.getPreferredBlockSize(routerFile);
+ assertEquals(routerSize, namenodeSize);
+
+ // Validate router failure response matches NN failure response.
+ Method m = ClientProtocol.class.getMethod(
+ "getPreferredBlockSize", String.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(
+ routerProtocol, nnProtocol, m, new Object[] {badPath});
+ }
+
+ private void testConcat(
+ String source, String target, boolean failureExpected) {
+ boolean failure = false;
+ try {
+ // Concat test file with fill block length file via router
+ routerProtocol.concat(target, new String[] {source});
+ } catch (IOException ex) {
+ failure = true;
+ }
+ assertEquals(failureExpected, failure);
+ }
+
+ @Test
+ public void testProxyConcatFile() throws Exception {
+
+ // Create a stub file in the primary ns
+ String sameNameservice = ns;
+ String existingFile =
+ cluster.getFederatedTestDirectoryForNS(sameNameservice) +
+ "_concatfile";
+ int existingFileSize = 32;
+ createFile(routerFS, existingFile, existingFileSize);
+
+ // Identify an alternate nameservice that doesn't match the existing file
+ String alternateNameservice = null;
+ for (String n : cluster.getNameservices()) {
+ if (!n.equals(sameNameservice)) {
+ alternateNameservice = n;
+ break;
+ }
+ }
+
+ // Create new files, must be a full block to use concat. One file is in the
+ // same namespace as the target file, the other is in a different namespace.
+ String altRouterFile =
+ cluster.getFederatedTestDirectoryForNS(alternateNameservice) +
+ "_newfile";
+ String sameRouterFile =
+ cluster.getFederatedTestDirectoryForNS(sameNameservice) +
+ "_newfile";
+ createFile(routerFS, altRouterFile, DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
+ createFile(routerFS, sameRouterFile, DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
+
+ // Concat in different namespaces, fails
+ testConcat(existingFile, altRouterFile, true);
+
+ // Concat in same namespaces, succeeds
+ testConcat(existingFile, sameRouterFile, false);
+
+ // Check target file length
+ FileStatus status = getFileStatus(routerFS, sameRouterFile);
+ assertEquals(
+ existingFileSize + DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT,
+ status.getLen());
+
+ // Validate router failure response matches NN failure response.
+ Method m = ClientProtocol.class.getMethod(
+ "concat", String.class, String[].class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(routerProtocol, nnProtocol, m,
+ new Object[] {badPath, new String[] {routerFile}});
+ }
+
+ @Test
+ public void testProxyAppend() throws Exception {
+
+ // Append a test string via router
+ EnumSet<CreateFlag> createFlag = EnumSet.of(CreateFlag.APPEND);
+ DFSClient routerClient = getRouterContext().getClient();
+ HdfsDataOutputStream stream =
+ routerClient.append(routerFile, 1024, createFlag, null, null);
+ stream.writeBytes(TEST_STRING);
+ stream.close();
+
+ // Verify file size via NN
+ FileStatus status = getFileStatus(nnFS, nnFile);
+ assertTrue(status.getLen() > TEST_STRING.length());
+
+ // Validate router failure response matches NN failure response.
+ Method m = ClientProtocol.class.getMethod("append", String.class,
+ String.class, EnumSetWritable.class);
+ String badPath = "/unknownlocation/unknowndir";
+ EnumSetWritable<CreateFlag> createFlagWritable =
+ new EnumSetWritable<CreateFlag>(createFlag);
+ compareResponses(routerProtocol, nnProtocol, m,
+ new Object[] {badPath, "testClient", createFlagWritable});
+ }
+
+ @Test
+ public void testProxyGetAdditionalDatanode()
+ throws IOException, InterruptedException, URISyntaxException {
+
+ // Use primitive APIs to open a file, add a block, and get datanode location
+ EnumSet<CreateFlag> createFlag = EnumSet.of(CreateFlag.CREATE);
+ String clientName = getRouterContext().getClient().getClientName();
+ String newRouterFile = routerFile + "_additionalDatanode";
+ HdfsFileStatus status = routerProtocol.create(
+ newRouterFile, new FsPermission("777"), clientName,
+ new EnumSetWritable<CreateFlag>(createFlag), true, (short) 1,
+ (long) 1024, CryptoProtocolVersion.supported(), null);
+
+ // Add a block via router (requires client to have same lease)
+ LocatedBlock block = routerProtocol.addBlock(
+ newRouterFile, clientName, null, null,
+ status.getFileId(), null, null);
+
+ DatanodeInfo[] exclusions = new DatanodeInfo[0];
+ LocatedBlock newBlock = routerProtocol.getAdditionalDatanode(
+ newRouterFile, status.getFileId(), block.getBlock(),
+ block.getLocations(), block.getStorageIDs(), exclusions, 1, clientName);
+ assertNotNull(newBlock);
+ }
+
+ @Test
+ public void testProxyCreateFileAlternateUser()
+ throws IOException, URISyntaxException, InterruptedException {
+
+ // Create via Router
+ String routerDir = cluster.getFederatedTestDirectoryForNS(ns);
+ String namenodeDir = cluster.getNamenodeTestDirectoryForNS(ns);
+ String newRouterFile = routerDir + "/unknownuser";
+ String newNamenodeFile = namenodeDir + "/unknownuser";
+ String username = "unknownuser";
+
+ // Allow all user access to dir
+ namenode.getFileContext().setPermission(
+ new Path(namenodeDir), new FsPermission("777"));
+
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser(username);
+ DFSClient client = getRouterContext().getClient(ugi);
+ client.create(newRouterFile, true);
+
+ // Fetch via NN and check user
+ FileStatus status = getFileStatus(nnFS, newNamenodeFile);
+ assertEquals(status.getOwner(), username);
+ }
+
+ @Test
+ public void testProxyGetFileInfoAcessException() throws IOException {
+
+ UserGroupInformation ugi =
+ UserGroupInformation.createRemoteUser("unknownuser");
+
+ // List files from the NN and trap the exception
+ Exception nnFailure = null;
+ try {
+ String testFile = cluster.getNamenodeTestFileForNS(ns);
+ namenode.getClient(ugi).getLocatedBlocks(testFile, 0);
+ } catch (Exception e) {
+ nnFailure = e;
+ }
+ assertNotNull(nnFailure);
+
+ // List files from the router and trap the exception
+ Exception routerFailure = null;
+ try {
+ String testFile = cluster.getFederatedTestFileForNS(ns);
+ getRouterContext().getClient(ugi).getLocatedBlocks(testFile, 0);
+ } catch (Exception e) {
+ routerFailure = e;
+ }
+ assertNotNull(routerFailure);
+
+ assertEquals(routerFailure.getClass(), nnFailure.getClass());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6632c5a4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
new file mode 100644
index 0000000..5489691
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createFile;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyFileExists;
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.server.federation.MockResolver;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+
+/**
+ * The the RPC interface of the {@link getRouter()} implemented by
+ * {@link RouterRpcServer}.
+ */
+public class TestRouterRpcMultiDestination extends TestRouterRpc {
+
+ @Override
+ public void testSetup() throws Exception {
+
+ RouterDFSCluster cluster = getCluster();
+
+ // Create mock locations
+ getCluster().installMockLocations();
+ List<RouterContext> routers = cluster.getRouters();
+
+ // Add extra location to the root mount / such that the root mount points:
+ // /
+ // ns0 -> /
+ // ns1 -> /
+ for (RouterContext rc : routers) {
+ Router router = rc.getRouter();
+ MockResolver resolver = (MockResolver) router.getSubclusterResolver();
+ resolver.addLocation("/", cluster.getNameservices().get(1), "/");
+ }
+
+ // Create a mount that points to 2 dirs in the same ns:
+ // /same
+ // ns0 -> /
+ // ns0 -> /target-ns0
+ for (RouterContext rc : routers) {
+ Router router = rc.getRouter();
+ MockResolver resolver = (MockResolver) router.getSubclusterResolver();
+ List<String> nss = cluster.getNameservices();
+ String ns0 = nss.get(0);
+ resolver.addLocation("/same", ns0, "/");
+ resolver.addLocation("/same", ns0, cluster.getNamenodePathForNS(ns0));
+ }
+
+ // Delete all files via the NNs and verify
+ cluster.deleteAllFiles();
+
+ // Create test fixtures on NN
+ cluster.createTestDirectoriesNamenode();
+
+ // Wait to ensure NN has fully created its test directories
+ Thread.sleep(100);
+
+ // Pick a NS, namenode and getRouter() for this test
+ RouterContext router = cluster.getRandomRouter();
+ this.setRouter(router);
+
+ String ns = cluster.getRandomNameservice();
+ this.setNs(ns);
+ this.setNamenode(cluster.getNamenode(ns, null));
+
+ // Create a test file on a single NN that is accessed via a getRouter() path
+ // with 2 destinations. All tests should failover to the alternate
+ // destination if the wrong NN is attempted first.
+ Random r = new Random();
+ String randomString = "testfile-" + r.nextInt();
+ setNamenodeFile("/" + randomString);
+ setRouterFile("/" + randomString);
+
+ FileSystem nnFs = getNamenodeFileSystem();
+ FileSystem routerFs = getRouterFileSystem();
+ createFile(nnFs, getNamenodeFile(), 32);
+
+ verifyFileExists(nnFs, getNamenodeFile());
+ verifyFileExists(routerFs, getRouterFile());
+ }
+
+ private void testListing(String path) throws IOException {
+
+ // Collect the mount table entries for this path
+ Set<String> requiredPaths = new TreeSet<>();
+ RouterContext rc = getRouterContext();
+ Router router = rc.getRouter();
+ FileSubclusterResolver subclusterResolver = router.getSubclusterResolver();
+ for (String mount : subclusterResolver.getMountPoints(path)) {
+ requiredPaths.add(mount);
+ }
+
+ // Get files/dirs from the Namenodes
+ PathLocation location = subclusterResolver.getDestinationForPath(path);
+ for (RemoteLocation loc : location.getDestinations()) {
+ String nsId = loc.getNameserviceId();
+ String dest = loc.getDest();
+ NamenodeContext nn = getCluster().getNamenode(nsId, null);
+ FileSystem fs = nn.getFileSystem();
+ FileStatus[] files = fs.listStatus(new Path(dest));
+ for (FileStatus file : files) {
+ String pathName = file.getPath().getName();
+ requiredPaths.add(pathName);
+ }
+ }
+
+ // Get files/dirs from the Router
+ DirectoryListing listing =
+ getRouterProtocol().getListing(path, HdfsFileStatus.EMPTY_NAME, false);
+ Iterator<String> requiredPathsIterator = requiredPaths.iterator();
+
+ // Match each path returned and verify order returned
+ HdfsFileStatus[] partialListing = listing.getPartialListing();
+ for (HdfsFileStatus fileStatus : listing.getPartialListing()) {
+ String fileName = requiredPathsIterator.next();
+ String currentFile = fileStatus.getFullPath(new Path(path)).getName();
+ assertEquals(currentFile, fileName);
+ }
+
+ // Verify the total number of results found/matched
+ assertEquals(
+ requiredPaths + " doesn't match " + Arrays.toString(partialListing),
+ requiredPaths.size(), partialListing.length);
+ }
+
+ @Override
+ public void testProxyListFiles() throws IOException, InterruptedException,
+ URISyntaxException, NoSuchMethodException, SecurityException {
+
+ // Verify that the root listing is a union of the mount table destinations
+ // and the files stored at all nameservices mounted at the root (ns0 + ns1)
+ // / -->
+ // /ns0 (from mount table)
+ // /ns1 (from mount table)
+ // /same (from the mount table)
+ // all items in / of ns0 from mapping of / -> ns0:::/)
+ // all items in / of ns1 from mapping of / -> ns1:::/)
+ testListing("/");
+
+ // Verify that the "/same" mount point lists the contents of both dirs in
+ // the same ns
+ // /same -->
+ // /target-ns0 (from root of ns0)
+ // /testdir (from contents of /target-ns0)
+ testListing("/same");
+
+ // List a non-existing path and validate error response with NN behavior
+ ClientProtocol namenodeProtocol =
+ getCluster().getRandomNamenode().getClient().getNamenode();
+ Method m = ClientProtocol.class.getMethod(
+ "getListing", String.class, byte[].class, boolean.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(getRouterProtocol(), namenodeProtocol, m,
+ new Object[] {badPath, HdfsFileStatus.EMPTY_NAME, false});
+ }
+
+ @Override
+ public void testProxyRenameFiles() throws IOException, InterruptedException {
+
+ super.testProxyRenameFiles();
+
+ List<String> nss = getCluster().getNameservices();
+ String ns0 = nss.get(0);
+ String ns1 = nss.get(1);
+
+ // Rename a file from ns0 into the root (mapped to both ns0 and ns1)
+ String testDir0 = getCluster().getFederatedTestDirectoryForNS(ns0);
+ String filename0 = testDir0 + "/testrename";
+ String renamedFile = "/testrename";
+ testRename(getRouterContext(), filename0, renamedFile, false);
+ testRename2(getRouterContext(), filename0, renamedFile, false);
+
+ // Rename a file from ns1 into the root (mapped to both ns0 and ns1)
+ String testDir1 = getCluster().getFederatedTestDirectoryForNS(ns1);
+ String filename1 = testDir1 + "/testrename";
+ testRename(getRouterContext(), filename1, renamedFile, false);
+ testRename2(getRouterContext(), filename1, renamedFile, false);
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: HDFS-11755. Underconstruction
blocks can be considered missing. Contributed by Nathan Roberts.
Posted by in...@apache.org.
HDFS-11755. Underconstruction blocks can be considered missing. Contributed by Nathan Roberts.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ad1e3e4d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ad1e3e4d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ad1e3e4d
Branch: refs/heads/HDFS-10467
Commit: ad1e3e4d9f105fac246ce1bdae80e92e013b8ba5
Parents: fab7374
Author: Kihwal Lee <ki...@apache.org>
Authored: Wed May 10 14:15:57 2017 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Wed May 10 14:15:57 2017 -0500
----------------------------------------------------------------------
.../server/blockmanagement/BlockManager.java | 2 +-
.../hdfs/server/namenode/CacheManager.java | 2 +-
.../apache/hadoop/hdfs/TestFileCorruption.java | 1 +
.../blockmanagement/TestBlockManager.java | 72 ++++++++++++++++++--
4 files changed, 68 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1e3e4d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 8f58e25..41662a4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -4092,7 +4092,7 @@ public class BlockManager implements BlockStatsMXBean {
final int curReplicasDelta, int expectedReplicasDelta) {
namesystem.writeLock();
try {
- if (!isPopulatingReplQueues()) {
+ if (!isPopulatingReplQueues() || !block.isComplete()) {
return;
}
NumberReplicas repl = countNodes(block);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1e3e4d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java
index 515a363..ab026f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java
@@ -100,7 +100,7 @@ import com.google.common.collect.Lists;
* caching directives, we will schedule caching and uncaching work.
*/
@InterfaceAudience.LimitedPrivate({"HDFS"})
-public final class CacheManager {
+public class CacheManager {
public static final Logger LOG = LoggerFactory.getLogger(CacheManager.class);
private static final float MIN_CACHED_BLOCKS_PERCENT = 0.001f;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1e3e4d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java
index 1f62414..1d9d402 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java
@@ -275,6 +275,7 @@ public class TestFileCorruption {
out.write(outBuffer, 0, bufferSize);
out.close();
dfs.setReplication(filePath, (short) 10);
+ cluster.triggerBlockReports();
// underreplicated Blocks should be one after setrep
GenericTestUtils.waitFor(new Supplier<Boolean>() {
@Override public Boolean get() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad1e3e4d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 36dafa5..beaef4a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -83,11 +83,15 @@ import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils;
import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
import org.apache.hadoop.hdfs.server.datanode.ReplicaBeingWritten;
+import org.apache.hadoop.hdfs.server.namenode.CacheManager;
+import org.apache.hadoop.hdfs.server.namenode.CachedBlock;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.INodeId;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.namenode.TestINodeFile;
+import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
+import org.apache.hadoop.hdfs.server.namenode.ha.HAState;
import org.apache.hadoop.hdfs.server.protocol.BlockReportContext;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
@@ -102,6 +106,8 @@ import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.MetricsAsserts;
+import org.apache.hadoop.util.GSet;
+import org.apache.hadoop.util.LightWeightGSet;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.Assert;
@@ -146,7 +152,20 @@ public class TestBlockManager {
Mockito.doReturn(true).when(fsn).hasWriteLock();
Mockito.doReturn(true).when(fsn).hasReadLock();
Mockito.doReturn(true).when(fsn).isRunning();
+ //Make shouldPopulaeReplQueues return true
+ HAContext haContext = Mockito.mock(HAContext.class);
+ HAState haState = Mockito.mock(HAState.class);
+ Mockito.when(haContext.getState()).thenReturn(haState);
+ Mockito.when(haState.shouldPopulateReplQueues()).thenReturn(true);
+ Mockito.when(fsn.getHAContext()).thenReturn(haContext);
bm = new BlockManager(fsn, false, conf);
+ bm.setInitializedReplQueues(true);
+ CacheManager cm = Mockito.mock(CacheManager.class);
+ Mockito.doReturn(cm).when(fsn).getCacheManager();
+ GSet<CachedBlock, CachedBlock> cb =
+ new LightWeightGSet<CachedBlock, CachedBlock>(1);
+ Mockito.when(cm.getCachedBlocks()).thenReturn(cb);
+
final String[] racks = {
"/rackA",
"/rackA",
@@ -522,7 +541,7 @@ public class TestBlockManager {
}
return ret;
}
-
+
private List<DatanodeDescriptor> startDecommission(int ... indexes) {
List<DatanodeDescriptor> nodes = getNodes(indexes);
for (DatanodeDescriptor node : nodes) {
@@ -918,6 +937,42 @@ public class TestBlockManager {
return builder.build();
}
+ @Test
+ public void testUCBlockNotConsideredMissing() throws Exception {
+ DatanodeDescriptor node = nodes.get(0);
+ DatanodeStorageInfo ds = node.getStorageInfos()[0];
+ node.setAlive(true);
+ DatanodeRegistration nodeReg =
+ new DatanodeRegistration(node, null, null, "");
+
+ // register new node
+ bm.getDatanodeManager().registerDatanode(nodeReg);
+ bm.getDatanodeManager().addDatanode(node);
+
+ // Build an incremental report
+ List<ReceivedDeletedBlockInfo> rdbiList = new ArrayList<>();
+
+ // blk_42 is under construction, finalizes on one node and is
+ // immediately deleted on same node
+ long blockId = 42; // arbitrary
+ BlockInfo receivedBlock = addUcBlockToBM(blockId);
+
+ rdbiList.add(new ReceivedDeletedBlockInfo(new Block(receivedBlock),
+ ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null));
+ rdbiList.add(new ReceivedDeletedBlockInfo(
+ new Block(blockId),
+ ReceivedDeletedBlockInfo.BlockStatus.DELETED_BLOCK, null));
+
+ // process IBR
+ StorageReceivedDeletedBlocks srdb =
+ new StorageReceivedDeletedBlocks(new DatanodeStorage(ds.getStorageID()),
+ rdbiList.toArray(new ReceivedDeletedBlockInfo[rdbiList.size()]));
+ bm.processIncrementalBlockReport(node, srdb);
+ // Needed replications should still be 0.
+ assertEquals("UC block was incorrectly added to needed Replications",
+ 0, bm.neededReconstruction.size());
+ }
+
private BlockInfo addBlockToBM(long blkId) {
Block block = new Block(blkId);
BlockInfo blockInfo = new BlockInfoContiguous(block, (short) 3);
@@ -1250,14 +1305,17 @@ public class TestBlockManager {
FileInputStream fstream = new FileInputStream(file);
DataInputStream in = new DataInputStream(fstream);
BufferedReader reader = new BufferedReader(new InputStreamReader(in));
+ String corruptBlocksLine;
+ Boolean foundIt = false;
try {
- for(int i =0;i<6;i++) {
- reader.readLine();
+ while ((corruptBlocksLine = reader.readLine()) != null) {
+ if (corruptBlocksLine.compareTo("Corrupt Blocks:") == 0) {
+ foundIt = true;
+ break;
+ }
}
- String corruptBlocksLine = reader.readLine();
- assertEquals("Unexpected text in metasave," +
- "was expecting corrupt blocks section!", 0,
- corruptBlocksLine.compareTo("Corrupt Blocks:"));
+ assertTrue("Unexpected text in metasave," +
+ "was expecting corrupt blocks section!", foundIt);
corruptBlocksLine = reader.readLine();
String regex = "Block=[0-9]+\\tNode=.*\\tStorageID=.*StorageState.*" +
"TotalReplicas=.*Reason=GENSTAMP_MISMATCH";
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: HDFS-11702. Remove indefinite
caching of key provider uri in DFSClient. Contributed by Rushabh S Shah.
Posted by in...@apache.org.
HDFS-11702. Remove indefinite caching of key provider uri in DFSClient. Contributed by Rushabh S Shah.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cef2815c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cef2815c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cef2815c
Branch: refs/heads/HDFS-10467
Commit: cef2815cf48154fe82f44082dcbdce6373c81284
Parents: a9a3d21
Author: Kihwal Lee <ki...@apache.org>
Authored: Mon May 8 08:27:37 2017 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Mon May 8 08:27:37 2017 -0500
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hdfs/DFSClient.java | 14 +-------------
.../org/apache/hadoop/hdfs/TestEncryptionZones.java | 9 ---------
2 files changed, 1 insertion(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef2815c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 3f1f3ea..d21b9b4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -232,7 +232,6 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
private static ThreadPoolExecutor HEDGED_READ_THREAD_POOL;
private static volatile ThreadPoolExecutor STRIPED_READ_THREAD_POOL;
private final int smallBufferSize;
- private URI keyProviderUri = null;
public DfsClientConf getConf() {
return dfsClientConf;
@@ -2901,10 +2900,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
* @throws IOException
*/
URI getKeyProviderUri() throws IOException {
- if (keyProviderUri != null) {
- return keyProviderUri;
- }
-
+ URI keyProviderUri = null;
// Lookup the secret in credentials object for namenodeuri.
Credentials credentials = ugi.getCredentials();
byte[] keyProviderUriBytes = credentials.getSecretKey(getKeyProviderMapKey());
@@ -2936,14 +2932,6 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
return clientContext.getKeyProviderCache().get(conf, getKeyProviderUri());
}
- /*
- * Should be used only for testing.
- */
- @VisibleForTesting
- public void setKeyProviderUri(URI providerUri) {
- this.keyProviderUri = providerUri;
- }
-
@VisibleForTesting
public void setKeyProvider(KeyProvider provider) {
clientContext.getKeyProviderCache().setKeyProvider(conf, provider);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef2815c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index 093d516..8eb3b7b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -1702,7 +1702,6 @@ public class TestEncryptionZones {
credentials.addSecretKey(lookUpKey,
DFSUtilClient.string2Bytes(dummyKeyProvider));
client.ugi.addCredentials(credentials);
- client.setKeyProviderUri(null);
Assert.assertEquals("Client Key provider is different from provider in "
+ "credentials map", dummyKeyProvider,
client.getKeyProviderUri().toString());
@@ -1724,7 +1723,6 @@ public class TestEncryptionZones {
CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
dummyKeyProviderUri1);
DFSClient mockClient = Mockito.spy(cluster.getFileSystem().getClient());
- mockClient.setKeyProviderUri(null);
// Namenode returning null as keyProviderUri in FSServerDefaults.
FsServerDefaults serverDefaultsWithKeyProviderNull =
getTestServerDefaults(null);
@@ -1736,7 +1734,6 @@ public class TestEncryptionZones {
Mockito.verify(mockClient, Mockito.times(1)).getServerDefaults();
String dummyKeyProviderUri2 = "dummy://foo:bar@test_provider2";
- mockClient.setKeyProviderUri(null);
FsServerDefaults serverDefaultsWithDummyKeyProvider =
getTestServerDefaults(dummyKeyProviderUri2);
// Namenode returning dummyKeyProvider2 in serverDefaults.
@@ -1765,8 +1762,6 @@ public class TestEncryptionZones {
// Unset the provider path in conf
clusterConf.unset(
CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH);
- // Nullify the cached value for key provider uri on client
- cluster.getFileSystem().getClient().setKeyProviderUri(null);
// Even after unsetting the local conf, the client key provider should be
// the same as namenode's provider.
Assert.assertEquals("Key Provider for client and namenode are different",
@@ -1777,8 +1772,6 @@ public class TestEncryptionZones {
clusterConf.set(
CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
"dummy://foo:bar@test_provider1");
- // Nullify the cached value for key provider uri on client
- cluster.getFileSystem().getClient().setKeyProviderUri(null);
// Even after pointing the conf to some dummy provider, the client key
// provider should be the same as namenode's provider.
Assert.assertEquals("Key Provider for client and namenode are different",
@@ -1813,8 +1806,6 @@ public class TestEncryptionZones {
// Creating a fake serverdefaults so that we can simulate namenode not
// being upgraded.
DFSClient spyClient = Mockito.spy(cluster.getFileSystem().getClient());
- // Clear the cache value of keyProviderUri on client side.
- spyClient.setKeyProviderUri(null);
Mockito.doReturn(spyServerDefaults).when(spyClient).getServerDefaults();
// Since FsServerDefaults#keyProviderUri is null, the client
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: HDFS-10630. Federation State Store
FS Implementation. Contributed by Jason Kace and Inigo Goiri.
Posted by in...@apache.org.
HDFS-10630. Federation State Store FS Implementation. Contributed by Jason Kace and Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/57ff72c5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/57ff72c5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/57ff72c5
Branch: refs/heads/HDFS-10467
Commit: 57ff72c5ba8121dd6a2947831182b20b209f7eb2
Parents: 515ab55
Author: Inigo Goiri <in...@apache.org>
Authored: Tue May 2 15:49:53 2017 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu May 11 09:54:08 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 14 +
.../federation/router/PeriodicService.java | 198 ++++++++
.../StateStoreConnectionMonitorService.java | 67 +++
.../federation/store/StateStoreService.java | 152 +++++-
.../federation/store/StateStoreUtils.java | 51 +-
.../store/driver/StateStoreDriver.java | 31 +-
.../driver/StateStoreRecordOperations.java | 17 +-
.../store/driver/impl/StateStoreBaseImpl.java | 31 +-
.../driver/impl/StateStoreFileBaseImpl.java | 429 ++++++++++++++++
.../store/driver/impl/StateStoreFileImpl.java | 161 +++++++
.../driver/impl/StateStoreFileSystemImpl.java | 178 +++++++
.../driver/impl/StateStoreSerializableImpl.java | 77 +++
.../federation/store/records/BaseRecord.java | 20 +-
.../server/federation/store/records/Query.java | 66 +++
.../src/main/resources/hdfs-default.xml | 16 +
.../store/FederationStateStoreTestUtils.java | 232 +++++++++
.../store/driver/TestStateStoreDriverBase.java | 483 +++++++++++++++++++
.../store/driver/TestStateStoreFile.java | 64 +++
.../store/driver/TestStateStoreFileSystem.java | 88 ++++
19 files changed, 2329 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57ff72c5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index f55be20..4045912 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.hdfs;
+import java.util.concurrent.TimeUnit;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
@@ -25,6 +27,8 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRackFaultTolerant;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.RamDiskReplicaLruTracker;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl;
import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreSerializerPBImpl;
import org.apache.hadoop.http.HttpConfig;
@@ -1117,6 +1121,16 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
FEDERATION_STORE_SERIALIZER_CLASS_DEFAULT =
StateStoreSerializerPBImpl.class;
+ public static final String FEDERATION_STORE_DRIVER_CLASS =
+ FEDERATION_STORE_PREFIX + "driver.class";
+ public static final Class<? extends StateStoreDriver>
+ FEDERATION_STORE_DRIVER_CLASS_DEFAULT = StateStoreFileImpl.class;
+
+ public static final String FEDERATION_STORE_CONNECTION_TEST_MS =
+ FEDERATION_STORE_PREFIX + "connection.test";
+ public static final long FEDERATION_STORE_CONNECTION_TEST_MS_DEFAULT =
+ TimeUnit.MINUTES.toMillis(1);
+
// dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry
@Deprecated
public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57ff72c5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java
new file mode 100644
index 0000000..5e12222
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java
@@ -0,0 +1,198 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.service.ServiceStateException;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+/**
+ * Service to periodically execute a runnable.
+ */
+public abstract class PeriodicService extends AbstractService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(PeriodicService.class);
+
+ /** Default interval in milliseconds for the periodic service. */
+ private static final long DEFAULT_INTERVAL_MS = TimeUnit.MINUTES.toMillis(1);
+
+
+ /** Interval for running the periodic service in milliseconds. */
+ private long intervalMs;
+ /** Name of the service. */
+ private final String serviceName;
+
+ /** Scheduler for the periodic service. */
+ private final ScheduledExecutorService scheduler;
+
+ /** If the service is running. */
+ private volatile boolean isRunning = false;
+
+ /** How many times we run. */
+ private long runCount;
+ /** How many errors we got. */
+ private long errorCount;
+ /** When was the last time we executed this service successfully. */
+ private long lastRun;
+
+ /**
+ * Create a new periodic update service.
+ *
+ * @param name Name of the service.
+ */
+ public PeriodicService(String name) {
+ this(name, DEFAULT_INTERVAL_MS);
+ }
+
+ /**
+ * Create a new periodic update service.
+ *
+ * @param name Name of the service.
+ * @param interval Interval for the periodic service in milliseconds.
+ */
+ public PeriodicService(String name, long interval) {
+ super(name);
+ this.serviceName = name;
+ this.intervalMs = interval;
+
+ ThreadFactory threadFactory = new ThreadFactoryBuilder()
+ .setNameFormat(this.getName() + "-%d")
+ .build();
+ this.scheduler = Executors.newScheduledThreadPool(1, threadFactory);
+ }
+
+ /**
+ * Set the interval for the periodic service.
+ *
+ * @param interval Interval in milliseconds.
+ */
+ protected void setIntervalMs(long interval) {
+ if (getServiceState() == STATE.STARTED) {
+ throw new ServiceStateException("Periodic service already started");
+ } else {
+ this.intervalMs = interval;
+ }
+ }
+
+ /**
+ * Get the interval for the periodic service.
+ *
+ * @return Interval in milliseconds.
+ */
+ protected long getIntervalMs() {
+ return this.intervalMs;
+ }
+
+ /**
+ * Get how many times we failed to run the periodic service.
+ *
+ * @return Times we failed to run the periodic service.
+ */
+ protected long getErrorCount() {
+ return this.errorCount;
+ }
+
+ /**
+ * Get how many times we run the periodic service.
+ *
+ * @return Times we run the periodic service.
+ */
+ protected long getRunCount() {
+ return this.runCount;
+ }
+
+ /**
+ * Get the last time the periodic service was executed.
+ *
+ * @return Last time the periodic service was executed.
+ */
+ protected long getLastUpdate() {
+ return this.lastRun;
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ super.serviceStart();
+ LOG.info("Starting periodic service {}", this.serviceName);
+ startPeriodic();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ stopPeriodic();
+ LOG.info("Stopping periodic service {}", this.serviceName);
+ super.serviceStop();
+ }
+
+ /**
+ * Stop the periodic task.
+ */
+ protected synchronized void stopPeriodic() {
+ if (this.isRunning) {
+ LOG.info("{} is shutting down", this.serviceName);
+ this.isRunning = false;
+ this.scheduler.shutdownNow();
+ }
+ }
+
+ /**
+ * Start the periodic execution.
+ */
+ protected synchronized void startPeriodic() {
+ stopPeriodic();
+
+ // Create the runnable service
+ Runnable updateRunnable = new Runnable() {
+ @Override
+ public void run() {
+ LOG.debug("Running {} update task", serviceName);
+ try {
+ if (!isRunning) {
+ return;
+ }
+ periodicInvoke();
+ runCount++;
+ lastRun = Time.now();
+ } catch (Exception ex) {
+ errorCount++;
+ LOG.warn(serviceName + " service threw an exception", ex);
+ }
+ }
+ };
+
+ // Start the execution of the periodic service
+ this.isRunning = true;
+ this.scheduler.scheduleWithFixedDelay(
+ updateRunnable, 0, this.intervalMs, TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Method that the service will run periodically.
+ */
+ protected abstract void periodicInvoke();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57ff72c5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreConnectionMonitorService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreConnectionMonitorService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreConnectionMonitorService.java
new file mode 100644
index 0000000..4d279c5
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreConnectionMonitorService.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.router.PeriodicService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Service to periodically monitor the connection of the StateStore
+ * {@link StateStoreService} data store and to re-open the connection
+ * to the data store if required.
+ */
+public class StateStoreConnectionMonitorService extends PeriodicService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StateStoreConnectionMonitorService.class);
+
+ /** Service that maintains the State Store connection. */
+ private final StateStoreService stateStore;
+
+
+ /**
+ * Create a new service to monitor the connectivity of the state store driver.
+ *
+ * @param store Instance of the state store to be monitored.
+ */
+ public StateStoreConnectionMonitorService(StateStoreService store) {
+ super(StateStoreConnectionMonitorService.class.getSimpleName());
+ this.stateStore = store;
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ this.setIntervalMs(conf.getLong(
+ DFSConfigKeys.FEDERATION_STORE_CONNECTION_TEST_MS,
+ DFSConfigKeys.FEDERATION_STORE_CONNECTION_TEST_MS_DEFAULT));
+
+ super.serviceInit(conf);
+ }
+
+ @Override
+ public void periodicInvoke() {
+ LOG.debug("Checking state store connection");
+ if (!stateStore.isDriverReady()) {
+ LOG.info("Attempting to open state store driver.");
+ stateStore.loadDriver();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57ff72c5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
index 866daa3..df207e0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
@@ -15,45 +15,168 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.hdfs.server.federation.store;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.LinkedList;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
/**
* A service to initialize a
* {@link org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver
- * StateStoreDriver} and maintain the connection to the data store. There
- * are multiple state store driver connections supported:
+ * StateStoreDriver} and maintain the connection to the data store. There are
+ * multiple state store driver connections supported:
* <ul>
- * <li>File {@link org.apache.hadoop.hdfs.server.federation.store.driver.impl.
+ * <li>File
+ * {@link org.apache.hadoop.hdfs.server.federation.store.driver.impl.
* StateStoreFileImpl StateStoreFileImpl}
- * <li>ZooKeeper {@link org.apache.hadoop.hdfs.server.federation.store.driver.
- * impl.StateStoreZooKeeperImpl StateStoreZooKeeperImpl}
+ * <li>ZooKeeper
+ * {@link org.apache.hadoop.hdfs.server.federation.store.driver.impl.
+ * StateStoreZooKeeperImpl StateStoreZooKeeperImpl}
* </ul>
* <p>
- * The service also supports the dynamic registration of data interfaces such as
- * the following:
+ * The service also supports the dynamic registration of record stores like:
* <ul>
- * <li>{@link MembershipStateStore}: state of the Namenodes in the
+ * <li>{@link MembershipStore}: state of the Namenodes in the
* federation.
* <li>{@link MountTableStore}: Mount table between to subclusters.
* See {@link org.apache.hadoop.fs.viewfs.ViewFs ViewFs}.
- * <li>{@link RouterStateStore}: State of the routers in the federation.
* </ul>
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class StateStoreService extends CompositeService {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StateStoreService.class);
+
+
+ /** State Store configuration. */
+ private Configuration conf;
+
/** Identifier for the service. */
private String identifier;
- // Stub class
- public StateStoreService(String name) {
- super(name);
+ /** Driver for the back end connection. */
+ private StateStoreDriver driver;
+
+ /** Service to maintain data store connection. */
+ private StateStoreConnectionMonitorService monitorService;
+
+
+ public StateStoreService() {
+ super(StateStoreService.class.getName());
+ }
+
+ /**
+ * Initialize the State Store and the connection to the backend.
+ *
+ * @param config Configuration for the State Store.
+ * @throws IOException
+ */
+ @Override
+ protected void serviceInit(Configuration config) throws Exception {
+ this.conf = config;
+
+ // Create implementation of State Store
+ Class<? extends StateStoreDriver> driverClass = this.conf.getClass(
+ DFSConfigKeys.FEDERATION_STORE_DRIVER_CLASS,
+ DFSConfigKeys.FEDERATION_STORE_DRIVER_CLASS_DEFAULT,
+ StateStoreDriver.class);
+ this.driver = ReflectionUtils.newInstance(driverClass, this.conf);
+
+ if (this.driver == null) {
+ throw new IOException("Cannot create driver for the State Store");
+ }
+
+ // Check the connection to the State Store periodically
+ this.monitorService = new StateStoreConnectionMonitorService(this);
+ this.addService(monitorService);
+
+ super.serviceInit(this.conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ loadDriver();
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ closeDriver();
+
+ super.serviceStop();
+ }
+
+ /**
+ * List of records supported by this State Store.
+ *
+ * @return List of supported record classes.
+ */
+ public Collection<Class<? extends BaseRecord>> getSupportedRecords() {
+ // TODO add list of records
+ return new LinkedList<>();
+ }
+
+ /**
+ * Load the State Store driver. If successful, refresh cached data tables.
+ */
+ public void loadDriver() {
+ synchronized (this.driver) {
+ if (!isDriverReady()) {
+ String driverName = this.driver.getClass().getSimpleName();
+ if (this.driver.init(conf, getIdentifier(), getSupportedRecords())) {
+ LOG.info("Connection to the State Store driver {} is open and ready",
+ driverName);
+ } else {
+ LOG.error("Cannot initialize State Store driver {}", driverName);
+ }
+ }
+ }
+ }
+
+ /**
+ * Check if the driver is ready to be used.
+ *
+ * @return If the driver is ready.
+ */
+ public boolean isDriverReady() {
+ return this.driver.isDriverReady();
+ }
+
+ /**
+ * Manually shuts down the driver.
+ *
+ * @throws Exception If the driver cannot be closed.
+ */
+ @VisibleForTesting
+ public void closeDriver() throws Exception {
+ if (this.driver != null) {
+ this.driver.close();
+ }
+ }
+
+ /**
+ * Get the state store driver.
+ *
+ * @return State store driver.
+ */
+ public StateStoreDriver getDriver() {
+ return this.driver;
}
/**
@@ -74,4 +197,5 @@ public class StateStoreService extends CompositeService {
public void setIdentifier(String id) {
this.identifier = id;
}
-}
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57ff72c5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUtils.java
index 8c681df..0a36619 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUtils.java
@@ -17,17 +17,22 @@
*/
package org.apache.hadoop.hdfs.server.federation.store;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.Query;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
- * Set of utility functions used to query, create, update and delete data
- * records in the state store.
+ * Set of utility functions used to work with the State Store.
*/
public final class StateStoreUtils {
- private static final Log LOG = LogFactory.getLog(StateStoreUtils.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StateStoreUtils.class);
+
private StateStoreUtils() {
// Utility class
@@ -52,7 +57,7 @@ public final class StateStoreUtils {
// Check if we went too far
if (actualClazz.equals(BaseRecord.class)) {
- LOG.error("We went too far (" + actualClazz + ") with " + clazz);
+ LOG.error("We went too far ({}) with {}", actualClazz, clazz);
actualClazz = clazz;
}
return actualClazz;
@@ -69,4 +74,36 @@ public final class StateStoreUtils {
Class<? extends BaseRecord> getRecordClass(final T record) {
return getRecordClass(record.getClass());
}
-}
+
+ /**
+ * Get the base class name for a record. If we get an implementation of a
+ * record we will return the real parent record class.
+ *
+ * @param clazz Class of the data record to check.
+ * @return Name of the base class for the record.
+ */
+ public static <T extends BaseRecord> String getRecordName(
+ final Class<T> clazz) {
+ return getRecordClass(clazz).getSimpleName();
+ }
+
+ /**
+ * Filters a list of records to find all records matching the query.
+ *
+ * @param query Map of field names and objects to use to filter results.
+ * @param records List of data records to filter.
+ * @return List of all records matching the query (or empty list if none
+ * match), null if the data set could not be filtered.
+ */
+ public static <T extends BaseRecord> List<T> filterMultiple(
+ final Query<T> query, final Iterable<T> records) {
+
+ List<T> matchingList = new ArrayList<>();
+ for (T record : records) {
+ if (query.matches(record)) {
+ matchingList.add(record);
+ }
+ }
+ return matchingList;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57ff72c5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java
index a1527df..90111bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java
@@ -18,15 +18,16 @@
package org.apache.hadoop.hdfs.server.federation.store.driver;
import java.net.InetAddress;
-import java.util.List;
+import java.util.Collection;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
import org.apache.hadoop.hdfs.server.federation.store.StateStoreUnavailableException;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils;
import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Driver class for an implementation of a {@link StateStoreService}
@@ -35,7 +36,8 @@ import org.apache.hadoop.util.Time;
*/
public abstract class StateStoreDriver implements StateStoreRecordOperations {
- private static final Log LOG = LogFactory.getLog(StateStoreDriver.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StateStoreDriver.class);
/** State Store configuration. */
@@ -47,13 +49,14 @@ public abstract class StateStoreDriver implements StateStoreRecordOperations {
/**
* Initialize the state store connection.
+ *
* @param config Configuration for the driver.
* @param id Identifier for the driver.
* @param records Records that are supported.
* @return If initialized and ready, false if failed to initialize driver.
*/
public boolean init(final Configuration config, final String id,
- final List<Class<? extends BaseRecord>> records) {
+ final Collection<Class<? extends BaseRecord>> records) {
this.conf = config;
this.identifier = id;
@@ -62,8 +65,20 @@ public abstract class StateStoreDriver implements StateStoreRecordOperations {
LOG.warn("The identifier for the State Store connection is not set");
}
- // TODO stub
- return false;
+ boolean success = initDriver();
+ if (!success) {
+ LOG.error("Cannot intialize driver for {}", getDriverName());
+ return false;
+ }
+
+ for (Class<? extends BaseRecord> cls : records) {
+ String recordString = StateStoreUtils.getRecordName(cls);
+ if (!initRecordStorage(recordString, cls)) {
+ LOG.error("Cannot initialize record store for {}", cls.getSimpleName());
+ return false;
+ }
+ }
+ return true;
}
/**
@@ -169,4 +184,4 @@ public abstract class StateStoreDriver implements StateStoreRecordOperations {
}
return hostname;
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57ff72c5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
index 739eeba..e76a733 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
@@ -19,11 +19,11 @@ package org.apache.hadoop.hdfs.server.federation.store.driver;
import java.io.IOException;
import java.util.List;
-import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.Query;
import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
import org.apache.hadoop.io.retry.AtMostOnce;
import org.apache.hadoop.io.retry.Idempotent;
@@ -67,14 +67,14 @@ public interface StateStoreRecordOperations {
* Get a single record from the store that matches the query.
*
* @param clazz Class of record to fetch.
- * @param query Map of field names and objects to filter results.
+ * @param query Query to filter results.
* @return A single record matching the query. Null if there are no matching
* records or more than one matching record in the store.
* @throws IOException If multiple records match or if the data store cannot
* be queried.
*/
@Idempotent
- <T extends BaseRecord> T get(Class<T> clazz, Map<String, String> query)
+ <T extends BaseRecord> T get(Class<T> clazz, Query<T> query)
throws IOException;
/**
@@ -83,14 +83,14 @@ public interface StateStoreRecordOperations {
* supports filtering it should overwrite this method.
*
* @param clazz Class of record to fetch.
- * @param query Map of field names and objects to filter results.
+ * @param query Query to filter results.
* @return Records of type clazz that match the query or empty list if none
* are found.
* @throws IOException Throws exception if unable to query the data store.
*/
@Idempotent
<T extends BaseRecord> List<T> getMultiple(
- Class<T> clazz, Map<String, String> query) throws IOException;
+ Class<T> clazz, Query<T> query) throws IOException;
/**
* Creates a single record. Optionally updates an existing record with same
@@ -152,13 +152,12 @@ public interface StateStoreRecordOperations {
* Remove multiple records of a specific class that match a query. Requires
* the getAll implementation to fetch fresh records on each call.
*
- * @param clazz Class of record to remove.
- * @param filter matching filter to remove.
+ * @param query Query to filter what to remove.
* @return The number of records removed.
* @throws IOException Throws exception if unable to query the data store.
*/
@AtMostOnce
- <T extends BaseRecord> int remove(Class<T> clazz, Map<String, String> filter)
+ <T extends BaseRecord> int remove(Class<T> clazz, Query<T> query)
throws IOException;
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57ff72c5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreBaseImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreBaseImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreBaseImpl.java
index b711fa9..1bd35f2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreBaseImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreBaseImpl.java
@@ -17,14 +17,17 @@
*/
package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
+import static org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils.filterMultiple;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
import org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils;
import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.Query;
+import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
/**
* Base implementation of a State Store driver. It contains default
@@ -41,7 +44,7 @@ public abstract class StateStoreBaseImpl extends StateStoreDriver {
@Override
public <T extends BaseRecord> T get(
- Class<T> clazz, Map<String, String> query) throws IOException {
+ Class<T> clazz, Query<T> query) throws IOException {
List<T> records = getMultiple(clazz, query);
if (records.size() > 1) {
throw new IOException("Found more than one object in collection");
@@ -53,17 +56,31 @@ public abstract class StateStoreBaseImpl extends StateStoreDriver {
}
@Override
+ public <T extends BaseRecord> List<T> getMultiple(
+ Class<T> clazz, Query<T> query) throws IOException {
+ QueryResult<T> result = get(clazz);
+ List<T> records = result.getRecords();
+ List<T> ret = filterMultiple(query, records);
+ if (ret == null) {
+ throw new IOException("Cannot fetch records from the store");
+ }
+ return ret;
+ }
+
+ @Override
public <T extends BaseRecord> boolean put(
T record, boolean allowUpdate, boolean errorIfExists) throws IOException {
- List<T> singletonList = new ArrayList<T>();
+ List<T> singletonList = new ArrayList<>();
singletonList.add(record);
return putAll(singletonList, allowUpdate, errorIfExists);
}
@Override
public <T extends BaseRecord> boolean remove(T record) throws IOException {
- Map<String, String> primaryKeys = record.getPrimaryKeys();
- Class<? extends BaseRecord> clazz = StateStoreUtils.getRecordClass(record);
- return remove(clazz, primaryKeys) == 1;
+ final Query<T> query = new Query<T>(record);
+ Class<? extends BaseRecord> clazz = record.getClass();
+ @SuppressWarnings("unchecked")
+ Class<T> recordClass = (Class<T>)StateStoreUtils.getRecordClass(clazz);
+ return remove(recordClass, query) == 1;
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57ff72c5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
new file mode 100644
index 0000000..d7c00ff
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
@@ -0,0 +1,429 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
+
+import static org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils.filterMultiple;
+import static org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils.getRecordClass;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreUnavailableException;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.Query;
+import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * {@link StateStoreDriver} implementation based on a local file.
+ */
+public abstract class StateStoreFileBaseImpl
+ extends StateStoreSerializableImpl {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StateStoreFileBaseImpl.class);
+
+ /** If it is initialized. */
+ private boolean initialized = false;
+
+ /** Name of the file containing the data. */
+ private static final String DATA_FILE_NAME = "records.data";
+
+
+ /**
+ * Lock reading records.
+ *
+ * @param clazz Class of the record.
+ */
+ protected abstract <T extends BaseRecord> void lockRecordRead(Class<T> clazz);
+
+ /**
+ * Unlock reading records.
+ *
+ * @param clazz Class of the record.
+ */
+ protected abstract <T extends BaseRecord> void unlockRecordRead(
+ Class<T> clazz);
+
+ /**
+ * Lock writing records.
+ *
+ * @param clazz Class of the record.
+ */
+ protected abstract <T extends BaseRecord> void lockRecordWrite(
+ Class<T> clazz);
+
+ /**
+ * Unlock writing records.
+ *
+ * @param clazz Class of the record.
+ */
+ protected abstract <T extends BaseRecord> void unlockRecordWrite(
+ Class<T> clazz);
+
+ /**
+ * Get the reader for the file system.
+ *
+ * @param clazz Class of the record.
+ */
+ protected abstract <T extends BaseRecord> BufferedReader getReader(
+ Class<T> clazz, String sub);
+
+ /**
+ * Get the writer for the file system.
+ *
+ * @param clazz Class of the record.
+ */
+ protected abstract <T extends BaseRecord> BufferedWriter getWriter(
+ Class<T> clazz, String sub);
+
+ /**
+ * Check if a path exists.
+ *
+ * @param path Path to check.
+ * @return If the path exists.
+ */
+ protected abstract boolean exists(String path);
+
+ /**
+ * Make a directory.
+ *
+ * @param path Path of the directory to create.
+ * @return If the directory was created.
+ */
+ protected abstract boolean mkdir(String path);
+
+ /**
+ * Get root directory.
+ *
+ * @return Root directory.
+ */
+ protected abstract String getRootDir();
+
+ /**
+ * Set the driver as initialized.
+ *
+ * @param ini If the driver is initialized.
+ */
+ public void setInitialized(boolean ini) {
+ this.initialized = ini;
+ }
+
+ @Override
+ public boolean initDriver() {
+ String rootDir = getRootDir();
+ try {
+ if (rootDir == null) {
+ LOG.error("Invalid root directory, unable to initialize driver.");
+ return false;
+ }
+
+ // Check root path
+ if (!exists(rootDir)) {
+ if (!mkdir(rootDir)) {
+ LOG.error("Cannot create State Store root directory {}", rootDir);
+ return false;
+ }
+ }
+ } catch (Exception ex) {
+ LOG.error(
+ "Cannot initialize filesystem using root directory {}", rootDir, ex);
+ return false;
+ }
+ setInitialized(true);
+ return true;
+ }
+
+ @Override
+ public <T extends BaseRecord> boolean initRecordStorage(
+ String className, Class<T> recordClass) {
+
+ String dataDirPath = getRootDir() + "/" + className;
+ try {
+ // Create data directories for files
+ if (!exists(dataDirPath)) {
+ LOG.info("{} data directory doesn't exist, creating it", dataDirPath);
+ if (!mkdir(dataDirPath)) {
+ LOG.error("Cannot create data directory {}", dataDirPath);
+ return false;
+ }
+ String dataFilePath = dataDirPath + "/" + DATA_FILE_NAME;
+ if (!exists(dataFilePath)) {
+ // Create empty file
+ List<T> emtpyList = new ArrayList<>();
+ if(!writeAll(emtpyList, recordClass)) {
+ LOG.error("Cannot create data file {}", dataFilePath);
+ return false;
+ }
+ }
+ }
+ } catch (Exception ex) {
+ LOG.error("Cannot create data directory {}", dataDirPath, ex);
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Read all lines from a file and deserialize into the desired record type.
+ *
+ * @param reader Open handle for the file.
+ * @param recordClass Record class to create.
+ * @param includeDates True if dateModified/dateCreated are serialized.
+ * @return List of records.
+ * @throws IOException
+ */
+ private <T extends BaseRecord> List<T> getAllFile(
+ BufferedReader reader, Class<T> clazz, boolean includeDates)
+ throws IOException {
+
+ List<T> ret = new ArrayList<T>();
+ String line;
+ while ((line = reader.readLine()) != null) {
+ if (!line.startsWith("#") && line.length() > 0) {
+ try {
+ T record = newRecord(line, clazz, includeDates);
+ ret.add(record);
+ } catch (Exception ex) {
+ LOG.error("Cannot parse line in data source file: {}", line, ex);
+ }
+ }
+ }
+ return ret;
+ }
+
+ @Override
+ public <T extends BaseRecord> QueryResult<T> get(Class<T> clazz)
+ throws IOException {
+ return get(clazz, (String)null);
+ }
+
+ @Override
+ public <T extends BaseRecord> QueryResult<T> get(Class<T> clazz, String sub)
+ throws IOException {
+ verifyDriverReady();
+ BufferedReader reader = null;
+ lockRecordRead(clazz);
+ try {
+ reader = getReader(clazz, sub);
+ List<T> data = getAllFile(reader, clazz, true);
+ return new QueryResult<T>(data, getTime());
+ } catch (Exception ex) {
+ LOG.error("Cannot fetch records {}", clazz.getSimpleName());
+ throw new IOException("Cannot read from data store " + ex.getMessage());
+ } finally {
+ if (reader != null) {
+ try {
+ reader.close();
+ } catch (IOException e) {
+ LOG.error("Failed closing file", e);
+ }
+ }
+ unlockRecordRead(clazz);
+ }
+ }
+
+ /**
+ * Overwrite the existing data with a new data set.
+ *
+ * @param list List of records to write.
+ * @param writer BufferedWriter stream to write to.
+ * @return If the records were succesfully written.
+ */
+ private <T extends BaseRecord> boolean writeAllFile(
+ Collection<T> records, BufferedWriter writer) {
+
+ try {
+ for (BaseRecord record : records) {
+ try {
+ String data = serializeString(record);
+ writer.write(data);
+ writer.newLine();
+ } catch (IllegalArgumentException ex) {
+ LOG.error("Cannot write record {} to file", record, ex);
+ }
+ }
+ writer.flush();
+ return true;
+ } catch (IOException e) {
+ LOG.error("Cannot commit records to file", e);
+ return false;
+ }
+ }
+
+ /**
+ * Overwrite the existing data with a new data set. Replaces all records in
+ * the data store for this record class. If all records in the data store are
+ * not successfully committed, this function must return false and leave the
+ * data store unchanged.
+ *
+ * @param records List of records to write. All records must be of type
+ * recordClass.
+ * @param recordClass Class of record to replace.
+ * @return true if all operations were successful, false otherwise.
+ * @throws StateStoreUnavailableException
+ */
+ public <T extends BaseRecord> boolean writeAll(
+ Collection<T> records, Class<T> recordClass)
+ throws StateStoreUnavailableException {
+ verifyDriverReady();
+ lockRecordWrite(recordClass);
+ BufferedWriter writer = null;
+ try {
+ writer = getWriter(recordClass, null);
+ return writeAllFile(records, writer);
+ } catch (Exception e) {
+ LOG.error(
+ "Cannot add records to file for {}", recordClass.getSimpleName(), e);
+ return false;
+ } finally {
+ if (writer != null) {
+ try {
+ writer.close();
+ } catch (IOException e) {
+ LOG.error(
+ "Cannot close writer for {}", recordClass.getSimpleName(), e);
+ }
+ }
+ unlockRecordWrite(recordClass);
+ }
+ }
+
+ /**
+ * Get the data file name.
+ *
+ * @return Data file name.
+ */
+ protected String getDataFileName() {
+ return DATA_FILE_NAME;
+ }
+
+ @Override
+ public boolean isDriverReady() {
+ return this.initialized;
+ }
+
+ @Override
+ public <T extends BaseRecord> boolean putAll(
+ List<T> records, boolean allowUpdate, boolean errorIfExists)
+ throws StateStoreUnavailableException {
+ verifyDriverReady();
+
+ if (records.isEmpty()) {
+ return true;
+ }
+
+ @SuppressWarnings("unchecked")
+ Class<T> clazz = (Class<T>) getRecordClass(records.get(0).getClass());
+ QueryResult<T> result;
+ try {
+ result = get(clazz);
+ } catch (IOException e) {
+ return false;
+ }
+ Map<Object, T> writeList = new HashMap<>();
+
+ // Write all of the existing records
+ for (T existingRecord : result.getRecords()) {
+ String key = existingRecord.getPrimaryKey();
+ writeList.put(key, existingRecord);
+ }
+
+ // Add inserts and updates, overwrite any existing values
+ for (T updatedRecord : records) {
+ try {
+ updatedRecord.validate();
+ String key = updatedRecord.getPrimaryKey();
+ if (writeList.containsKey(key) && allowUpdate) {
+ // Update
+ writeList.put(key, updatedRecord);
+ // Update the mod time stamp. Many backends will use their
+ // own timestamp for the mod time.
+ updatedRecord.setDateModified(this.getTime());
+ } else if (!writeList.containsKey(key)) {
+ // Insert
+ // Create/Mod timestamps are already initialized
+ writeList.put(key, updatedRecord);
+ } else if (errorIfExists) {
+ LOG.error("Attempt to insert record {} that already exists",
+ updatedRecord);
+ return false;
+ }
+ } catch (IllegalArgumentException ex) {
+ LOG.error("Cannot write invalid record to State Store", ex);
+ return false;
+ }
+ }
+
+ // Write all
+ boolean status = writeAll(writeList.values(), clazz);
+ return status;
+ }
+
+ @Override
+ public <T extends BaseRecord> int remove(Class<T> clazz, Query<T> query)
+ throws StateStoreUnavailableException {
+ verifyDriverReady();
+
+ if (query == null) {
+ return 0;
+ }
+
+ int removed = 0;
+ // Get the current records
+ try {
+ final QueryResult<T> result = get(clazz);
+ final List<T> existingRecords = result.getRecords();
+ // Write all of the existing records except those to be removed
+ final List<T> recordsToRemove = filterMultiple(query, existingRecords);
+ removed = recordsToRemove.size();
+ final List<T> newRecords = new LinkedList<>();
+ for (T record : existingRecords) {
+ if (!recordsToRemove.contains(record)) {
+ newRecords.add(record);
+ }
+ }
+ if (!writeAll(newRecords, clazz)) {
+ throw new IOException(
+ "Cannot remove record " + clazz + " query " + query);
+ }
+ } catch (IOException e) {
+ LOG.error("Cannot remove records {} query {}", clazz, query, e);
+ }
+
+ return removed;
+ }
+
+ @Override
+ public <T extends BaseRecord> boolean removeAll(Class<T> clazz)
+ throws StateStoreUnavailableException {
+ verifyDriverReady();
+ List<T> emptyList = new ArrayList<>();
+ boolean status = writeAll(emptyList, clazz);
+ return status;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57ff72c5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java
new file mode 100644
index 0000000..24e9660
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java
@@ -0,0 +1,161 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.io.Files;
+
+/**
+ * StateStoreDriver implementation based on a local file.
+ */
+public class StateStoreFileImpl extends StateStoreFileBaseImpl {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StateStoreFileImpl.class);
+
+ /** Configuration keys. */
+ public static final String FEDERATION_STORE_FILE_DIRECTORY =
+ DFSConfigKeys.FEDERATION_STORE_PREFIX + "driver.file.directory";
+
+ /** Synchronization. */
+ private static final ReadWriteLock READ_WRITE_LOCK =
+ new ReentrantReadWriteLock();
+
+ /** Root directory for the state store. */
+ private String rootDirectory;
+
+
+ @Override
+ protected boolean exists(String path) {
+ File test = new File(path);
+ return test.exists();
+ }
+
+ @Override
+ protected boolean mkdir(String path) {
+ File dir = new File(path);
+ return dir.mkdirs();
+ }
+
+ @Override
+ protected String getRootDir() {
+ if (this.rootDirectory == null) {
+ String dir = getConf().get(FEDERATION_STORE_FILE_DIRECTORY);
+ if (dir == null) {
+ File tempDir = Files.createTempDir();
+ dir = tempDir.getAbsolutePath();
+ }
+ this.rootDirectory = dir;
+ }
+ return this.rootDirectory;
+ }
+
+ @Override
+ protected <T extends BaseRecord> void lockRecordWrite(Class<T> recordClass) {
+ // TODO - Synchronize via FS
+ READ_WRITE_LOCK.writeLock().lock();
+ }
+
+ @Override
+ protected <T extends BaseRecord> void unlockRecordWrite(
+ Class<T> recordClass) {
+ // TODO - Synchronize via FS
+ READ_WRITE_LOCK.writeLock().unlock();
+ }
+
+ @Override
+ protected <T extends BaseRecord> void lockRecordRead(Class<T> recordClass) {
+ // TODO - Synchronize via FS
+ READ_WRITE_LOCK.readLock().lock();
+ }
+
+ @Override
+ protected <T extends BaseRecord> void unlockRecordRead(Class<T> recordClass) {
+ // TODO - Synchronize via FS
+ READ_WRITE_LOCK.readLock().unlock();
+ }
+
+ @Override
+ protected <T extends BaseRecord> BufferedReader getReader(
+ Class<T> clazz, String sub) {
+ String filename = StateStoreUtils.getRecordName(clazz);
+ if (sub != null && sub.length() > 0) {
+ filename += "/" + sub;
+ }
+ filename += "/" + getDataFileName();
+
+ try {
+ LOG.debug("Loading file: {}", filename);
+ File file = new File(getRootDir(), filename);
+ FileInputStream fis = new FileInputStream(file);
+ InputStreamReader isr =
+ new InputStreamReader(fis, StandardCharsets.UTF_8);
+ BufferedReader reader = new BufferedReader(isr);
+ return reader;
+ } catch (Exception ex) {
+ LOG.error(
+ "Cannot open read stream for record {}", clazz.getSimpleName(), ex);
+ return null;
+ }
+ }
+
+ @Override
+ protected <T extends BaseRecord> BufferedWriter getWriter(
+ Class<T> clazz, String sub) {
+ String filename = StateStoreUtils.getRecordName(clazz);
+ if (sub != null && sub.length() > 0) {
+ filename += "/" + sub;
+ }
+ filename += "/" + getDataFileName();
+
+ try {
+ File file = new File(getRootDir(), filename);
+ FileOutputStream fos = new FileOutputStream(file, false);
+ OutputStreamWriter osw =
+ new OutputStreamWriter(fos, StandardCharsets.UTF_8);
+ BufferedWriter writer = new BufferedWriter(osw);
+ return writer;
+ } catch (IOException ex) {
+ LOG.error(
+ "Cannot open read stream for record {}", clazz.getSimpleName(), ex);
+ return null;
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ setInitialized(false);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57ff72c5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileSystemImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileSystemImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileSystemImpl.java
new file mode 100644
index 0000000..5968421
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileSystemImpl.java
@@ -0,0 +1,178 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * StateStoreDriver} implementation based on a filesystem. The most common uses
+ * HDFS as a backend.
+ */
+public class StateStoreFileSystemImpl extends StateStoreFileBaseImpl {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StateStoreFileSystemImpl.class);
+
+
+ /** Configuration keys. */
+ public static final String FEDERATION_STORE_FS_PATH =
+ DFSConfigKeys.FEDERATION_STORE_PREFIX + "driver.fs.path";
+
+ /** File system to back the State Store. */
+ private FileSystem fs;
+ /** Working path in the filesystem. */
+ private String workPath;
+
+ @Override
+ protected boolean exists(String path) {
+ try {
+ return fs.exists(new Path(path));
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ @Override
+ protected boolean mkdir(String path) {
+ try {
+ return fs.mkdirs(new Path(path));
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ @Override
+ protected String getRootDir() {
+ if (this.workPath == null) {
+ String rootPath = getConf().get(FEDERATION_STORE_FS_PATH);
+ URI workUri;
+ try {
+ workUri = new URI(rootPath);
+ fs = FileSystem.get(workUri, getConf());
+ } catch (Exception ex) {
+ return null;
+ }
+ this.workPath = rootPath;
+ }
+ return this.workPath;
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (fs != null) {
+ fs.close();
+ }
+ }
+
+ /**
+ * Get the folder path for the record class' data.
+ *
+ * @param cls Data record class.
+ * @return Path of the folder containing the record class' data files.
+ */
+ private Path getPathForClass(Class<? extends BaseRecord> clazz) {
+ if (clazz == null) {
+ return null;
+ }
+ // TODO extract table name from class: entry.getTableName()
+ String className = StateStoreUtils.getRecordName(clazz);
+ return new Path(workPath, className);
+ }
+
+ @Override
+ protected <T extends BaseRecord> void lockRecordRead(Class<T> clazz) {
+ // Not required, synced with HDFS leasing
+ }
+
+ @Override
+ protected <T extends BaseRecord> void unlockRecordRead(Class<T> clazz) {
+ // Not required, synced with HDFS leasing
+ }
+
+ @Override
+ protected <T extends BaseRecord> void lockRecordWrite(Class<T> clazz) {
+ // TODO -> wait for lease to be available
+ }
+
+ @Override
+ protected <T extends BaseRecord> void unlockRecordWrite(Class<T> clazz) {
+ // TODO -> ensure lease is closed for the file
+ }
+
+ @Override
+ protected <T extends BaseRecord> BufferedReader getReader(
+ Class<T> clazz, String sub) {
+
+ Path path = getPathForClass(clazz);
+ if (sub != null && sub.length() > 0) {
+ path = Path.mergePaths(path, new Path("/" + sub));
+ }
+ path = Path.mergePaths(path, new Path("/" + getDataFileName()));
+
+ try {
+ FSDataInputStream fdis = fs.open(path);
+ InputStreamReader isr =
+ new InputStreamReader(fdis, StandardCharsets.UTF_8);
+ BufferedReader reader = new BufferedReader(isr);
+ return reader;
+ } catch (IOException ex) {
+ LOG.error("Cannot open write stream for {} to {}",
+ clazz.getSimpleName(), path);
+ return null;
+ }
+ }
+
+ @Override
+ protected <T extends BaseRecord> BufferedWriter getWriter(
+ Class<T> clazz, String sub) {
+
+ Path path = getPathForClass(clazz);
+ if (sub != null && sub.length() > 0) {
+ path = Path.mergePaths(path, new Path("/" + sub));
+ }
+ path = Path.mergePaths(path, new Path("/" + getDataFileName()));
+
+ try {
+ FSDataOutputStream fdos = fs.create(path, true);
+ OutputStreamWriter osw =
+ new OutputStreamWriter(fdos, StandardCharsets.UTF_8);
+ BufferedWriter writer = new BufferedWriter(osw);
+ return writer;
+ } catch (IOException ex) {
+ LOG.error("Cannot open write stream for {} to {}",
+ clazz.getSimpleName(), path);
+ return null;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57ff72c5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializableImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializableImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializableImpl.java
new file mode 100644
index 0000000..e9b3fdf
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializableImpl.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+
+/**
+ * State Store driver that stores a serialization of the records. The serializer
+ * is pluggable.
+ */
+public abstract class StateStoreSerializableImpl extends StateStoreBaseImpl {
+
+ /** Default serializer for this driver. */
+ private StateStoreSerializer serializer;
+
+
+ @Override
+ public boolean init(final Configuration config, final String id,
+ final Collection<Class<? extends BaseRecord>> records) {
+ boolean ret = super.init(config, id, records);
+
+ this.serializer = StateStoreSerializer.getSerializer(config);
+
+ return ret;
+ }
+
+ /**
+ * Serialize a record using the serializer.
+ * @param record Record to serialize.
+ * @return Byte array with the serialization of the record.
+ */
+ protected <T extends BaseRecord> byte[] serialize(T record) {
+ return serializer.serialize(record);
+ }
+
+ /**
+ * Serialize a record using the serializer.
+ * @param record Record to serialize.
+ * @return String with the serialization of the record.
+ */
+ protected <T extends BaseRecord> String serializeString(T record) {
+ return serializer.serializeString(record);
+ }
+
+ /**
+ * Creates a record from an input data string.
+ * @param data Serialized text of the record.
+ * @param clazz Record class.
+ * @param includeDates If dateModified and dateCreated are serialized.
+ * @return The created record.
+ * @throws IOException
+ */
+ protected <T extends BaseRecord> T newRecord(
+ String data, Class<T> clazz, boolean includeDates) throws IOException {
+ return serializer.deserialize(data, clazz);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57ff72c5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java
index 4192a3d..79f99c8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java
@@ -123,6 +123,24 @@ public abstract class BaseRecord implements Comparable<BaseRecord> {
}
/**
+ * Check if this record matches a partial record.
+ *
+ * @param other Partial record.
+ * @return If this record matches.
+ */
+ public boolean like(BaseRecord other) {
+ if (other == null) {
+ return false;
+ }
+ Map<String, String> thisKeys = this.getPrimaryKeys();
+ Map<String, String> otherKeys = other.getPrimaryKeys();
+ if (thisKeys == null) {
+ return otherKeys == null;
+ }
+ return thisKeys.equals(otherKeys);
+ }
+
+ /**
* Override equals check to use primary key(s) for comparison.
*/
@Override
@@ -186,4 +204,4 @@ public abstract class BaseRecord implements Comparable<BaseRecord> {
public String toString() {
return getPrimaryKey();
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57ff72c5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/Query.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/Query.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/Query.java
new file mode 100644
index 0000000..3c59abf
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/Query.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+/**
+ * Check if a record matches a query. The query is usually a partial record.
+ *
+ * @param <T> Type of the record to query.
+ */
+public class Query<T extends BaseRecord> {
+
+ /** Partial object to compare against. */
+ private final T partial;
+
+
+ /**
+ * Create a query to search for a partial record.
+ *
+ * @param partial It defines the attributes to search.
+ */
+ public Query(final T part) {
+ this.partial = part;
+ }
+
+ /**
+ * Get the partial record used to query.
+ *
+ * @return The partial record used for the query.
+ */
+ public T getPartial() {
+ return this.partial;
+ }
+
+ /**
+ * Check if a record matches the primary keys or the partial record.
+ *
+ * @param other Record to check.
+ * @return If the record matches. Don't match if there is no partial.
+ */
+ public boolean matches(T other) {
+ if (this.partial == null) {
+ return false;
+ }
+ return this.partial.like(other);
+ }
+
+ @Override
+ public String toString() {
+ return "Checking: " + this.partial;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57ff72c5/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 44db981..12e4f8b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4537,4 +4537,20 @@
</description>
</property>
+ <property>
+ <name>dfs.federation.router.store.driver.class</name>
+ <value>org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl</value>
+ <description>
+ Class to implement the State Store. By default it uses the local disk.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.store.connection.test</name>
+ <value>60000</value>
+ <description>
+ How often to check for the connection to the State Store in milliseconds.
+ </description>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57ff72c5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
new file mode 100644
index 0000000..fc5aebd
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
@@ -0,0 +1,232 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.FEDERATION_STORE_DRIVER_CLASS;
+import static org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl.FEDERATION_STORE_FILE_DIRECTORY;
+import static org.junit.Assert.assertNotNull;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileBaseImpl;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.util.Time;
+
+/**
+ * Utilities to test the State Store.
+ */
+public final class FederationStateStoreTestUtils {
+
+ private FederationStateStoreTestUtils() {
+ // Utility Class
+ }
+
+ /**
+ * Get the default State Store driver implementation.
+ *
+ * @return Class of the default State Store driver implementation.
+ */
+ public static Class<? extends StateStoreDriver> getDefaultDriver() {
+ return DFSConfigKeys.FEDERATION_STORE_DRIVER_CLASS_DEFAULT;
+ }
+
+ /**
+ * Create a default State Store configuration.
+ *
+ * @return State Store configuration.
+ */
+ public static Configuration getStateStoreConfiguration() {
+ Class<? extends StateStoreDriver> clazz = getDefaultDriver();
+ return getStateStoreConfiguration(clazz);
+ }
+
+ /**
+ * Create a new State Store configuration for a particular driver.
+ *
+ * @param clazz Class of the driver to create.
+ * @return State Store configuration.
+ */
+ public static Configuration getStateStoreConfiguration(
+ Class<? extends StateStoreDriver> clazz) {
+ Configuration conf = new HdfsConfiguration(false);
+
+ conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, true);
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, "hdfs://test");
+
+ conf.setClass(FEDERATION_STORE_DRIVER_CLASS, clazz, StateStoreDriver.class);
+
+ if (clazz.isAssignableFrom(StateStoreFileBaseImpl.class)) {
+ setFileConfiguration(conf);
+ }
+ return conf;
+ }
+
+ /**
+ * Create a new State Store based on a configuration.
+ *
+ * @param configuration Configuration for the State Store.
+ * @return New State Store service.
+ * @throws IOException If it cannot create the State Store.
+ * @throws InterruptedException If we cannot wait for the store to start.
+ */
+ public static StateStoreService getStateStore(
+ Configuration configuration) throws IOException, InterruptedException {
+
+ StateStoreService stateStore = new StateStoreService();
+ assertNotNull(stateStore);
+
+ // Set unique identifier, this is normally the router address
+ String identifier = UUID.randomUUID().toString();
+ stateStore.setIdentifier(identifier);
+
+ stateStore.init(configuration);
+ stateStore.start();
+
+ // Wait for state store to connect
+ waitStateStore(stateStore, TimeUnit.SECONDS.toMillis(10));
+
+ return stateStore;
+ }
+
+ /**
+ * Wait for the State Store to initialize its driver.
+ *
+ * @param stateStore State Store.
+ * @param timeoutMs Time out in milliseconds.
+ * @throws IOException If the State Store cannot be reached.
+ * @throws InterruptedException If the sleep is interrupted.
+ */
+ public static void waitStateStore(StateStoreService stateStore,
+ long timeoutMs) throws IOException, InterruptedException {
+ long startingTime = Time.monotonicNow();
+ while (!stateStore.isDriverReady()) {
+ Thread.sleep(100);
+ if (Time.monotonicNow() - startingTime > timeoutMs) {
+ throw new IOException("Timeout waiting for State Store to connect");
+ }
+ }
+ }
+
+ /**
+ * Delete the default State Store.
+ *
+ * @throws IOException
+ */
+ public static void deleteStateStore() throws IOException {
+ Class<? extends StateStoreDriver> driverClass = getDefaultDriver();
+ deleteStateStore(driverClass);
+ }
+
+ /**
+ * Delete the State Store.
+ * @param driverClass Class of the State Store driver implementation.
+ * @throws IOException If it cannot be deleted.
+ */
+ public static void deleteStateStore(
+ Class<? extends StateStoreDriver> driverClass) throws IOException {
+
+ if (StateStoreFileBaseImpl.class.isAssignableFrom(driverClass)) {
+ String workingDirectory = System.getProperty("user.dir");
+ File dir = new File(workingDirectory + "/statestore");
+ if (dir.exists()) {
+ FileUtils.cleanDirectory(dir);
+ }
+ }
+ }
+
+ /**
+ * Set the default configuration for drivers based on files.
+ *
+ * @param conf Configuration to extend.
+ */
+ public static void setFileConfiguration(Configuration conf) {
+ String workingPath = System.getProperty("user.dir");
+ String stateStorePath = workingPath + "/statestore";
+ conf.set(FEDERATION_STORE_FILE_DIRECTORY, stateStorePath);
+ }
+
+ /**
+ * Clear all the records from the State Store.
+ *
+ * @param store State Store to remove records from.
+ * @return If the State Store was cleared.
+ * @throws IOException If it cannot clear the State Store.
+ */
+ public static boolean clearAllRecords(StateStoreService store)
+ throws IOException {
+ Collection<Class<? extends BaseRecord>> allRecords =
+ store.getSupportedRecords();
+ for (Class<? extends BaseRecord> recordType : allRecords) {
+ if (!clearRecords(store, recordType)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Clear records from a certain type from the State Store.
+ *
+ * @param store State Store to remove records from.
+ * @param recordClass Class of the records to remove.
+ * @return If the State Store was cleared.
+ * @throws IOException If it cannot clear the State Store.
+ */
+ public static <T extends BaseRecord> boolean clearRecords(
+ StateStoreService store, Class<T> recordClass) throws IOException {
+ List<T> emptyList = new ArrayList<>();
+ if (!synchronizeRecords(store, emptyList, recordClass)) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Synchronize a set of records. Remove all and keep the ones specified.
+ *
+ * @param stateStore State Store service managing the driver.
+ * @param records Records to add.
+ * @param clazz Class of the record to synchronize.
+ * @return If the synchronization succeeded.
+ * @throws IOException If it cannot connect to the State Store.
+ */
+ public static <T extends BaseRecord> boolean synchronizeRecords(
+ StateStoreService stateStore, List<T> records, Class<T> clazz)
+ throws IOException {
+ StateStoreDriver driver = stateStore.getDriver();
+ driver.verifyDriverReady();
+ if (driver.removeAll(clazz)) {
+ if (driver.putAll(records, true, false)) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: YARN-6522. Make SLS JSON input
file format simple and scalable (yufeigu via rkanter)
Posted by in...@apache.org.
YARN-6522. Make SLS JSON input file format simple and scalable (yufeigu via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3082552b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3082552b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3082552b
Branch: refs/heads/HDFS-10467
Commit: 3082552b3b991df846caf572b58e44308ddf8eeb
Parents: 07761af
Author: Robert Kanter <rk...@apache.org>
Authored: Thu May 4 17:21:46 2017 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Thu May 4 17:21:46 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/sls/SLSRunner.java | 102 ++++++++++++++-----
.../hadoop/yarn/sls/appmaster/AMSimulator.java | 42 ++++----
.../sls/synthetic/SynthTraceJobProducer.java | 2 +-
.../apache/hadoop/yarn/sls/utils/SLSUtils.java | 49 ++++++---
.../src/site/markdown/SchedulerLoadSimulator.md | 28 +++--
.../hadoop/yarn/sls/utils/TestSLSUtils.java | 30 ++++++
6 files changed, 182 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3082552b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index 9d35d1b..ddd35ef 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -119,6 +119,9 @@ public class SLSRunner extends Configured implements Tool {
// logger
public final static Logger LOG = Logger.getLogger(SLSRunner.class);
+ private final static int DEFAULT_MAPPER_PRIORITY = 20;
+ private final static int DEFAULT_REDUCER_PRIORITY = 10;
+
/**
* The type of trace in input.
*/
@@ -247,8 +250,8 @@ public class SLSRunner extends Configured implements Tool {
break;
case SYNTH:
stjp = new SynthTraceJobProducer(getConf(), new Path(inputTraces[0]));
- nodeSet.addAll(SLSUtils.generateNodesFromSynth(stjp.getNumNodes(),
- stjp.getNodesPerRack()));
+ nodeSet.addAll(SLSUtils.generateNodes(stjp.getNumNodes(),
+ stjp.getNumNodes()/stjp.getNodesPerRack()));
break;
default:
throw new YarnException("Input configuration not recognized, "
@@ -259,6 +262,10 @@ public class SLSRunner extends Configured implements Tool {
nodeSet.addAll(SLSUtils.parseNodesFromNodeFile(nodeFile));
}
+ if (nodeSet.size() == 0) {
+ throw new YarnException("No node! Please configure nodes.");
+ }
+
// create NM simulators
Random random = new Random();
Set<String> rackSet = new HashSet<String>();
@@ -348,7 +355,11 @@ public class SLSRunner extends Configured implements Tool {
private void createAMForJob(Map jsonJob) throws YarnException {
long jobStartTime = Long.parseLong(jsonJob.get("job.start.ms").toString());
- long jobFinishTime = Long.parseLong(jsonJob.get("job.end.ms").toString());
+
+ long jobFinishTime = 0;
+ if (jsonJob.containsKey("job.end.ms")) {
+ jobFinishTime = Long.parseLong(jsonJob.get("job.end.ms").toString());
+ }
String user = (String) jsonJob.get("job.user");
if (user == null) {
@@ -358,25 +369,49 @@ public class SLSRunner extends Configured implements Tool {
String queue = jsonJob.get("job.queue.name").toString();
increaseQueueAppNum(queue);
- String oldAppId = jsonJob.get("job.id").toString();
+ String oldAppId = (String)jsonJob.get("job.id");
+ if (oldAppId == null) {
+ oldAppId = Integer.toString(AM_ID);
+ }
- // tasks
+ String amType = (String)jsonJob.get("am.type");
+ if (amType == null) {
+ amType = SLSUtils.DEFAULT_JOB_TYPE;
+ }
+
+ runNewAM(amType, user, queue, oldAppId, jobStartTime, jobFinishTime,
+ getTaskContainers(jsonJob), null);
+ }
+
+ private List<ContainerSimulator> getTaskContainers(Map jsonJob)
+ throws YarnException {
+ List<ContainerSimulator> containers = new ArrayList<>();
List tasks = (List) jsonJob.get("job.tasks");
if (tasks == null || tasks.size() == 0) {
throw new YarnException("No task for the job!");
}
- List<ContainerSimulator> containerList = new ArrayList<>();
for (Object o : tasks) {
Map jsonTask = (Map) o;
- String hostname = jsonTask.get("container.host").toString();
- long taskStart = Long.parseLong(jsonTask.get("container.start.ms")
- .toString());
- long taskFinish = Long.parseLong(jsonTask.get("container.end.ms")
- .toString());
- long lifeTime = taskFinish - taskStart;
-
- // Set memory and vcores from job trace file
+
+ String hostname = (String) jsonTask.get("container.host");
+
+ long duration = 0;
+ if (jsonTask.containsKey("duration.ms")) {
+ duration = Integer.parseInt(jsonTask.get("duration.ms").toString());
+ } else if (jsonTask.containsKey("container.start.ms") &&
+ jsonTask.containsKey("container.end.ms")) {
+ long taskStart = Long.parseLong(jsonTask.get("container.start.ms")
+ .toString());
+ long taskFinish = Long.parseLong(jsonTask.get("container.end.ms")
+ .toString());
+ duration = taskFinish - taskStart;
+ }
+ if (duration <= 0) {
+ throw new YarnException("Duration of a task shouldn't be less or equal"
+ + " to 0!");
+ }
+
Resource res = getDefaultContainerResource();
if (jsonTask.containsKey("container.memory")) {
int containerMemory =
@@ -390,17 +425,30 @@ public class SLSRunner extends Configured implements Tool {
res.setVirtualCores(containerVCores);
}
- int priority = Integer.parseInt(jsonTask.get("container.priority")
- .toString());
- String type = jsonTask.get("container.type").toString();
- containerList.add(
- new ContainerSimulator(res, lifeTime, hostname, priority, type));
+ int priority = DEFAULT_MAPPER_PRIORITY;
+ if (jsonTask.containsKey("container.priority")) {
+ priority = Integer.parseInt(jsonTask.get("container.priority")
+ .toString());
+ }
+
+ String type = "map";
+ if (jsonTask.containsKey("container.type")) {
+ type = jsonTask.get("container.type").toString();
+ }
+
+ int count = 1;
+ if (jsonTask.containsKey("count")) {
+ count = Integer.parseInt(jsonTask.get("count").toString());
+ }
+ count = Math.max(count, 1);
+
+ for (int i = 0; i < count; i++) {
+ containers.add(
+ new ContainerSimulator(res, duration, hostname, priority, type));
+ }
}
- // create a new AM
- String amType = jsonJob.get("am.type").toString();
- runNewAM(amType, user, queue, oldAppId, jobStartTime, jobFinishTime,
- containerList, null);
+ return containers;
}
/**
@@ -463,7 +511,7 @@ public class SLSRunner extends Configured implements Tool {
taskAttempt.getStartTime();
containerList.add(
new ContainerSimulator(getDefaultContainerResource(),
- containerLifeTime, hostname, 10, "map"));
+ containerLifeTime, hostname, DEFAULT_MAPPER_PRIORITY, "map"));
}
// reducer
@@ -479,7 +527,7 @@ public class SLSRunner extends Configured implements Tool {
taskAttempt.getStartTime();
containerList.add(
new ContainerSimulator(getDefaultContainerResource(),
- containerLifeTime, hostname, 20, "reduce"));
+ containerLifeTime, hostname, DEFAULT_REDUCER_PRIORITY, "reduce"));
}
// Only supports the default job type currently
@@ -559,7 +607,7 @@ public class SLSRunner extends Configured implements Tool {
Resource.newInstance((int) tai.getTaskInfo().getTaskMemory(),
(int) tai.getTaskInfo().getTaskVCores());
containerList.add(new ContainerSimulator(containerResource,
- containerLifeTime, hostname, 10, "map"));
+ containerLifeTime, hostname, DEFAULT_MAPPER_PRIORITY, "map"));
maxMapRes = Resources.componentwiseMax(maxMapRes, containerResource);
maxMapDur =
containerLifeTime > maxMapDur ? containerLifeTime : maxMapDur;
@@ -579,7 +627,7 @@ public class SLSRunner extends Configured implements Tool {
Resource.newInstance((int) tai.getTaskInfo().getTaskMemory(),
(int) tai.getTaskInfo().getTaskVCores());
containerList.add(new ContainerSimulator(containerResource,
- containerLifeTime, hostname, 20, "reduce"));
+ containerLifeTime, hostname, DEFAULT_REDUCER_PRIORITY, "reduce"));
maxRedRes = Resources.componentwiseMax(maxRedRes, containerResource);
maxRedDur =
containerLifeTime > maxRedDur ? containerLifeTime : maxRedDur;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3082552b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
index 45a3c07..70c5579 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
@@ -400,26 +400,28 @@ public abstract class AMSimulator extends TaskRunner.Task {
Map<String, ResourceRequest> nodeLocalRequestMap = new HashMap<String, ResourceRequest>();
ResourceRequest anyRequest = null;
for (ContainerSimulator cs : csList) {
- String rackHostNames[] = SLSUtils.getRackHostName(cs.getHostname());
- // check rack local
- String rackname = "/" + rackHostNames[0];
- if (rackLocalRequestMap.containsKey(rackname)) {
- rackLocalRequestMap.get(rackname).setNumContainers(
- rackLocalRequestMap.get(rackname).getNumContainers() + 1);
- } else {
- ResourceRequest request = createResourceRequest(
- cs.getResource(), rackname, priority, 1);
- rackLocalRequestMap.put(rackname, request);
- }
- // check node local
- String hostname = rackHostNames[1];
- if (nodeLocalRequestMap.containsKey(hostname)) {
- nodeLocalRequestMap.get(hostname).setNumContainers(
- nodeLocalRequestMap.get(hostname).getNumContainers() + 1);
- } else {
- ResourceRequest request = createResourceRequest(
- cs.getResource(), hostname, priority, 1);
- nodeLocalRequestMap.put(hostname, request);
+ if (cs.getHostname() != null) {
+ String[] rackHostNames = SLSUtils.getRackHostName(cs.getHostname());
+ // check rack local
+ String rackname = "/" + rackHostNames[0];
+ if (rackLocalRequestMap.containsKey(rackname)) {
+ rackLocalRequestMap.get(rackname).setNumContainers(
+ rackLocalRequestMap.get(rackname).getNumContainers() + 1);
+ } else {
+ ResourceRequest request =
+ createResourceRequest(cs.getResource(), rackname, priority, 1);
+ rackLocalRequestMap.put(rackname, request);
+ }
+ // check node local
+ String hostname = rackHostNames[1];
+ if (nodeLocalRequestMap.containsKey(hostname)) {
+ nodeLocalRequestMap.get(hostname).setNumContainers(
+ nodeLocalRequestMap.get(hostname).getNumContainers() + 1);
+ } else {
+ ResourceRequest request =
+ createResourceRequest(cs.getResource(), hostname, priority, 1);
+ nodeLocalRequestMap.put(hostname, request);
+ }
}
// any
if (anyRequest == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3082552b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/synthetic/SynthTraceJobProducer.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/synthetic/SynthTraceJobProducer.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/synthetic/SynthTraceJobProducer.java
index 3d2ec94..c89e4e2 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/synthetic/SynthTraceJobProducer.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/synthetic/SynthTraceJobProducer.java
@@ -131,7 +131,7 @@ public class SynthTraceJobProducer implements JobStoryProducer {
}
public int getNodesPerRack() {
- return trace.nodes_per_rack;
+ return trace.nodes_per_rack < 1 ? 1: trace.nodes_per_rack;
}
public int getNumNodes() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3082552b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java
index e27b36f..dbc2dab 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java
@@ -101,7 +101,7 @@ public class SLSUtils {
*/
public static Set<String> parseNodesFromSLSTrace(String jobTrace)
throws IOException {
- Set<String> nodeSet = new HashSet<String>();
+ Set<String> nodeSet = new HashSet<>();
JsonFactory jsonF = new JsonFactory();
ObjectMapper mapper = new ObjectMapper();
Reader input =
@@ -109,13 +109,7 @@ public class SLSUtils {
try {
Iterator<Map> i = mapper.readValues(jsonF.createParser(input), Map.class);
while (i.hasNext()) {
- Map jsonE = i.next();
- List tasks = (List) jsonE.get("job.tasks");
- for (Object o : tasks) {
- Map jsonTask = (Map) o;
- String hostname = jsonTask.get("container.host").toString();
- nodeSet.add(hostname);
- }
+ addNodes(nodeSet, i.next());
}
} finally {
input.close();
@@ -123,6 +117,29 @@ public class SLSUtils {
return nodeSet;
}
+ private static void addNodes(Set<String> nodeSet, Map jsonEntry) {
+ if (jsonEntry.containsKey("num.nodes")) {
+ int numNodes = Integer.parseInt(jsonEntry.get("num.nodes").toString());
+ int numRacks = 1;
+ if (jsonEntry.containsKey("num.racks")) {
+ numRacks = Integer.parseInt(
+ jsonEntry.get("num.racks").toString());
+ }
+ nodeSet.addAll(generateNodes(numNodes, numRacks));
+ }
+
+ if (jsonEntry.containsKey("job.tasks")) {
+ List tasks = (List) jsonEntry.get("job.tasks");
+ for (Object o : tasks) {
+ Map jsonTask = (Map) o;
+ String hostname = (String) jsonTask.get("container.host");
+ if (hostname != null) {
+ nodeSet.add(hostname);
+ }
+ }
+ }
+ }
+
/**
* parse the input node file, return each host name
*/
@@ -150,11 +167,19 @@ public class SLSUtils {
return nodeSet;
}
- public static Set<? extends String> generateNodesFromSynth(
- int numNodes, int nodesPerRack) {
- Set<String> nodeSet = new HashSet<String>();
+ public static Set<? extends String> generateNodes(int numNodes,
+ int numRacks){
+ Set<String> nodeSet = new HashSet<>();
+ if (numRacks < 1) {
+ numRacks = 1;
+ }
+
+ if (numRacks > numNodes) {
+ numRacks = numNodes;
+ }
+
for (int i = 0; i < numNodes; i++) {
- nodeSet.add("/rack" + i % nodesPerRack + "/node" + i);
+ nodeSet.add("/rack" + i % numRacks + "/node" + i);
}
return nodeSet;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3082552b/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md b/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md
index f0e3b8c..6e00e9a 100644
--- a/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md
+++ b/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md
@@ -328,18 +328,24 @@ Appendix
Here we provide an example format of the sls json file, which contains 2 jobs. The first job has 3 map tasks and the second one has 2 map tasks.
{
- "am.type" : "mapreduce",
- "job.start.ms" : 0,
- "job.end.ms" : 95375,
- "job.queue.name" : "sls_queue_1",
- "job.id" : "job_1",
- "job.user" : "default",
+ "num.nodes": 3, // total number of nodes in the cluster
+ "num.racks": 1 // total number of racks in the cluster, it divides num.nodes into the racks evenly, optional, the default value is 1
+ }
+ {
+ "am.type" : "mapreduce", // type of AM, optional, the default value is "mapreduce"
+ "job.start.ms" : 0, // job start time
+ "job.end.ms" : 95375, // job finish time, optional, the default value is 0
+ "job.queue.name" : "sls_queue_1", // the queue job will be submitted to
+ "job.id" : "job_1", // the job id used to track the job, optional, the default value is an zero-based integer increasing with number of jobs
+ "job.user" : "default", // user, optional, the default value is "default"
"job.tasks" : [ {
- "container.host" : "/default-rack/node1",
- "container.start.ms" : 6664,
- "container.end.ms" : 23707,
- "container.priority" : 20,
- "container.type" : "map"
+ "count": 1, // number of tasks, optional, the default value is 1
+ "container.host" : "/default-rack/node1", // host the container asks for
+ "container.start.ms" : 6664, // container start time, optional
+ "container.end.ms" : 23707, // container finish time, optional
+ "duration.ms": 50000, // duration of the container, optional if start and end time is specified
+ "container.priority" : 20, // priority of the container, optional, the default value is 20
+ "container.type" : "map" // type of the container, could be "map" or "reduce", optional, the default value is "map"
}, {
"container.host" : "/default-rack/node3",
"container.start.ms" : 6665,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3082552b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java
index f4eda67..30964a1 100644
--- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java
+++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java
@@ -21,6 +21,9 @@ package org.apache.hadoop.yarn.sls.utils;
import org.junit.Assert;
import org.junit.Test;
+import java.util.HashSet;
+import java.util.Set;
+
public class TestSLSUtils {
@Test
@@ -36,4 +39,31 @@ public class TestSLSUtils {
Assert.assertEquals(rackHostname[1], "node1");
}
+ @Test
+ public void testGenerateNodes() {
+ Set<? extends String> nodes = SLSUtils.generateNodes(3, 3);
+ Assert.assertEquals("Number of nodes is wrong.", 3, nodes.size());
+ Assert.assertEquals("Number of racks is wrong.", 3, getNumRack(nodes));
+
+ nodes = SLSUtils.generateNodes(3, 1);
+ Assert.assertEquals("Number of nodes is wrong.", 3, nodes.size());
+ Assert.assertEquals("Number of racks is wrong.", 1, getNumRack(nodes));
+
+ nodes = SLSUtils.generateNodes(3, 4);
+ Assert.assertEquals("Number of nodes is wrong.", 3, nodes.size());
+ Assert.assertEquals("Number of racks is wrong.", 3, getNumRack(nodes));
+
+ nodes = SLSUtils.generateNodes(3, 0);
+ Assert.assertEquals("Number of nodes is wrong.", 3, nodes.size());
+ Assert.assertEquals("Number of racks is wrong.", 1, getNumRack(nodes));
+ }
+
+ private int getNumRack(Set<? extends String> nodes) {
+ Set<String> racks = new HashSet<>();
+ for (String node : nodes) {
+ String[] rackHostname = SLSUtils.getRackHostName(node);
+ racks.add(rackHostname[0]);
+ }
+ return racks.size();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: YARN-6563
ConcurrentModificationException in TimelineCollectorManager while stopping RM
(Contributed by Haibo Chen via Vrushali C)
Posted by in...@apache.org.
YARN-6563 ConcurrentModificationException in TimelineCollectorManager while stopping RM (Contributed by Haibo Chen via Vrushali C)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7dd258d8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7dd258d8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7dd258d8
Branch: refs/heads/HDFS-10467
Commit: 7dd258d8f4aef594346e874e5ad4ba22c3171cd1
Parents: 45382bf
Author: Vrushali Channapattan <vr...@apache.org>
Authored: Tue May 9 11:18:12 2017 -0700
Committer: Vrushali Channapattan <vr...@apache.org>
Committed: Tue May 9 11:18:12 2017 -0700
----------------------------------------------------------------------
.../timelineservice/collector/TimelineCollectorManager.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7dd258d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
index 8ef9b43..07cbb2b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
@@ -219,8 +219,10 @@ public class TimelineCollectorManager extends AbstractService {
@Override
protected void serviceStop() throws Exception {
if (collectors != null && collectors.size() > 1) {
- for (TimelineCollector c : collectors.values()) {
- c.serviceStop();
+ synchronized (collectors) {
+ for (TimelineCollector c : collectors.values()) {
+ c.serviceStop();
+ }
}
}
// stop the flusher first
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: YARN-6552. Increase YARN test
timeouts from 1 second to 10 seconds. Contributed by Eric Badger
Posted by in...@apache.org.
YARN-6552. Increase YARN test timeouts from 1 second to 10 seconds. Contributed by Eric Badger
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6099deeb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6099deeb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6099deeb
Branch: refs/heads/HDFS-10467
Commit: 6099deebcb0ea23adc0f0c62db749c8f528869e9
Parents: 2ba9903
Author: Jason Lowe <jl...@yahoo-inc.com>
Authored: Wed May 10 08:47:48 2017 -0500
Committer: Jason Lowe <jl...@yahoo-inc.com>
Committed: Wed May 10 08:47:48 2017 -0500
----------------------------------------------------------------------
.../java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java | 2 +-
.../scheduler/fair/policies/TestEmptyQueues.java | 6 +++---
2 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6099deeb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
index 37c859c..c054209 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
@@ -126,7 +126,7 @@ public class TestLogsCLI {
assertTrue("Should return an error code", exitCode != 0);
}
- @Test(timeout = 1000l)
+ @Test(timeout = 10000l)
public void testInvalidOpts() throws Exception {
Configuration conf = new YarnConfiguration();
YarnClient mockYarnClient = createMockYarnClient(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6099deeb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestEmptyQueues.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestEmptyQueues.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestEmptyQueues.java
index 4636c5b..7511cec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestEmptyQueues.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestEmptyQueues.java
@@ -39,17 +39,17 @@ public class TestEmptyQueues {
policy.computeShares(schedulables, Resources.none());
}
- @Test (timeout = 1000)
+ @Test (timeout = 10000)
public void testFifoPolicy() {
testComputeShares(SchedulingPolicy.getInstance(FifoPolicy.class));
}
- @Test (timeout = 1000)
+ @Test (timeout = 10000)
public void testFairSharePolicy() {
testComputeShares(SchedulingPolicy.getInstance(FairSharePolicy.class));
}
- @Test (timeout = 1000)
+ @Test (timeout = 10000)
public void testDRFPolicy() {
testComputeShares(
SchedulingPolicy.getInstance(DominantResourceFairnessPolicy.class));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: HDFS-11546. Federation Router RPC
server. Contributed by Jason Kace and Inigo Goiri.
Posted by in...@apache.org.
HDFS-11546. Federation Router RPC server. Contributed by Jason Kace and Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6632c5a4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6632c5a4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6632c5a4
Branch: refs/heads/HDFS-10467
Commit: 6632c5a454f0d13bcb3f9d3ff2da36b5e0fc0529
Parents: 57ff72c
Author: Inigo Goiri <in...@apache.org>
Authored: Thu May 11 09:57:03 2017 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu May 11 09:57:03 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 38 +
.../resolver/FederationNamespaceInfo.java | 46 +-
.../federation/resolver/RemoteLocation.java | 46 +-
.../federation/router/ConnectionContext.java | 104 +
.../federation/router/ConnectionManager.java | 408 ++++
.../federation/router/ConnectionPool.java | 314 +++
.../federation/router/ConnectionPoolId.java | 117 ++
.../router/RemoteLocationContext.java | 38 +-
.../server/federation/router/RemoteMethod.java | 164 ++
.../server/federation/router/RemoteParam.java | 71 +
.../hdfs/server/federation/router/Router.java | 58 +-
.../federation/router/RouterRpcClient.java | 856 ++++++++
.../federation/router/RouterRpcServer.java | 1867 +++++++++++++++++-
.../src/main/resources/hdfs-default.xml | 95 +
.../server/federation/FederationTestUtils.java | 80 +-
.../hdfs/server/federation/MockResolver.java | 90 +-
.../server/federation/RouterConfigBuilder.java | 20 +-
.../server/federation/RouterDFSCluster.java | 535 +++--
.../server/federation/router/TestRouter.java | 31 +-
.../server/federation/router/TestRouterRpc.java | 869 ++++++++
.../router/TestRouterRpcMultiDestination.java | 216 ++
21 files changed, 5675 insertions(+), 388 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6632c5a4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 4045912..543a73f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -1100,6 +1100,44 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
// HDFS Router-based federation
public static final String FEDERATION_ROUTER_PREFIX =
"dfs.federation.router.";
+ public static final String DFS_ROUTER_DEFAULT_NAMESERVICE =
+ FEDERATION_ROUTER_PREFIX + "default.nameserviceId";
+ public static final String DFS_ROUTER_HANDLER_COUNT_KEY =
+ FEDERATION_ROUTER_PREFIX + "handler.count";
+ public static final int DFS_ROUTER_HANDLER_COUNT_DEFAULT = 10;
+ public static final String DFS_ROUTER_READER_QUEUE_SIZE_KEY =
+ FEDERATION_ROUTER_PREFIX + "reader.queue.size";
+ public static final int DFS_ROUTER_READER_QUEUE_SIZE_DEFAULT = 100;
+ public static final String DFS_ROUTER_READER_COUNT_KEY =
+ FEDERATION_ROUTER_PREFIX + "reader.count";
+ public static final int DFS_ROUTER_READER_COUNT_DEFAULT = 1;
+ public static final String DFS_ROUTER_HANDLER_QUEUE_SIZE_KEY =
+ FEDERATION_ROUTER_PREFIX + "handler.queue.size";
+ public static final int DFS_ROUTER_HANDLER_QUEUE_SIZE_DEFAULT = 100;
+ public static final String DFS_ROUTER_RPC_BIND_HOST_KEY =
+ FEDERATION_ROUTER_PREFIX + "rpc-bind-host";
+ public static final int DFS_ROUTER_RPC_PORT_DEFAULT = 8888;
+ public static final String DFS_ROUTER_RPC_ADDRESS_KEY =
+ FEDERATION_ROUTER_PREFIX + "rpc-address";
+ public static final String DFS_ROUTER_RPC_ADDRESS_DEFAULT =
+ "0.0.0.0:" + DFS_ROUTER_RPC_PORT_DEFAULT;
+ public static final String DFS_ROUTER_RPC_ENABLE =
+ FEDERATION_ROUTER_PREFIX + "rpc.enable";
+ public static final boolean DFS_ROUTER_RPC_ENABLE_DEFAULT = true;
+
+ // HDFS Router NN client
+ public static final String DFS_ROUTER_NAMENODE_CONNECTION_POOL_SIZE =
+ FEDERATION_ROUTER_PREFIX + "connection.pool-size";
+ public static final int DFS_ROUTER_NAMENODE_CONNECTION_POOL_SIZE_DEFAULT =
+ 64;
+ public static final String DFS_ROUTER_NAMENODE_CONNECTION_POOL_CLEAN =
+ FEDERATION_ROUTER_PREFIX + "connection.pool.clean.ms";
+ public static final long DFS_ROUTER_NAMENODE_CONNECTION_POOL_CLEAN_DEFAULT =
+ TimeUnit.MINUTES.toMillis(1);
+ public static final String DFS_ROUTER_NAMENODE_CONNECTION_CLEAN_MS =
+ FEDERATION_ROUTER_PREFIX + "connection.clean.ms";
+ public static final long DFS_ROUTER_NAMENODE_CONNECTION_CLEAN_MS_DEFAULT =
+ TimeUnit.SECONDS.toMillis(10);
// HDFS Router State Store connection
public static final String FEDERATION_FILE_RESOLVER_CLIENT_CLASS =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6632c5a4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java
index bbaeca3..edcd308 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java
@@ -23,15 +23,14 @@ import org.apache.hadoop.hdfs.server.federation.router.RemoteLocationContext;
* Represents information about a single nameservice/namespace in a federated
* HDFS cluster.
*/
-public class FederationNamespaceInfo
- implements Comparable<FederationNamespaceInfo>, RemoteLocationContext {
+public class FederationNamespaceInfo extends RemoteLocationContext {
/** Block pool identifier. */
- private String blockPoolId;
+ private final String blockPoolId;
/** Cluster identifier. */
- private String clusterId;
+ private final String clusterId;
/** Nameservice identifier. */
- private String nameserviceId;
+ private final String nameserviceId;
public FederationNamespaceInfo(String bpId, String clId, String nsId) {
this.blockPoolId = bpId;
@@ -39,15 +38,16 @@ public class FederationNamespaceInfo
this.nameserviceId = nsId;
}
- /**
- * The HDFS nameservice id for this namespace.
- *
- * @return Nameservice identifier.
- */
+ @Override
public String getNameserviceId() {
return this.nameserviceId;
}
+ @Override
+ public String getDest() {
+ return this.nameserviceId;
+ }
+
/**
* The HDFS cluster id for this namespace.
*
@@ -67,33 +67,7 @@ public class FederationNamespaceInfo
}
@Override
- public int hashCode() {
- return this.nameserviceId.hashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == null) {
- return false;
- } else if (obj instanceof FederationNamespaceInfo) {
- return this.compareTo((FederationNamespaceInfo) obj) == 0;
- } else {
- return false;
- }
- }
-
- @Override
- public int compareTo(FederationNamespaceInfo info) {
- return this.nameserviceId.compareTo(info.getNameserviceId());
- }
-
- @Override
public String toString() {
return this.nameserviceId + "->" + this.blockPoolId + ":" + this.clusterId;
}
-
- @Override
- public String getDest() {
- return this.nameserviceId;
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6632c5a4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java
index eef136d..6aa12ce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java
@@ -17,34 +17,51 @@
*/
package org.apache.hadoop.hdfs.server.federation.resolver;
-import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.hadoop.hdfs.server.federation.router.RemoteLocationContext;
/**
* A single in a remote namespace consisting of a nameservice ID
* and a HDFS path.
*/
-public class RemoteLocation implements RemoteLocationContext {
+public class RemoteLocation extends RemoteLocationContext {
/** Identifier of the remote namespace for this location. */
- private String nameserviceId;
+ private final String nameserviceId;
+ /** Identifier of the namenode in the namespace for this location. */
+ private final String namenodeId;
/** Path in the remote location. */
- private String path;
+ private final String path;
/**
* Create a new remote location.
*
+ * @param nsId
+ * @param pPath
+ */
+ public RemoteLocation(String nsId, String pPath) {
+ this(nsId, null, pPath);
+ }
+
+ /**
+ * Create a new remote location pointing to a particular namenode in the
+ * namespace.
+ *
* @param nsId Destination namespace.
* @param pPath Path in the destination namespace.
*/
- public RemoteLocation(String nsId, String pPath) {
+ public RemoteLocation(String nsId, String nnId, String pPath) {
this.nameserviceId = nsId;
+ this.namenodeId = nnId;
this.path = pPath;
}
@Override
public String getNameserviceId() {
- return this.nameserviceId;
+ String ret = this.nameserviceId;
+ if (this.namenodeId != null) {
+ ret += "-" + this.namenodeId;
+ }
+ return ret;
}
@Override
@@ -54,21 +71,6 @@ public class RemoteLocation implements RemoteLocationContext {
@Override
public String toString() {
- return this.nameserviceId + "->" + this.path;
- }
-
- @Override
- public int hashCode() {
- return new HashCodeBuilder(17, 31)
- .append(this.nameserviceId)
- .append(this.path)
- .toHashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- return (obj != null &&
- obj.getClass() == this.getClass() &&
- obj.hashCode() == this.hashCode());
+ return getNameserviceId() + "->" + this.path;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6632c5a4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java
new file mode 100644
index 0000000..1d27b51
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.hadoop.hdfs.NameNodeProxiesClient.ProxyAndInfo;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.ipc.RPC;
+
+/**
+ * Context to track a connection in a {@link ConnectionPool}. When a client uses
+ * a connection, it increments a counter to mark it as active. Once the client
+ * is done with the connection, it decreases the counter. It also takes care of
+ * closing the connection once is not active.
+ */
+public class ConnectionContext {
+
+ /** Client for the connection. */
+ private final ProxyAndInfo<ClientProtocol> client;
+ /** How many threads are using this connection. */
+ private int numThreads = 0;
+ /** If the connection is closed. */
+ private boolean closed = false;
+
+
+ public ConnectionContext(ProxyAndInfo<ClientProtocol> connection) {
+ this.client = connection;
+ }
+
+ /**
+ * Check if the connection is active.
+ *
+ * @return True if the connection is active.
+ */
+ public synchronized boolean isActive() {
+ return this.numThreads > 0;
+ }
+
+ /**
+ * Check if the connection is closed.
+ *
+ * @return If the connection is closed.
+ */
+ public synchronized boolean isClosed() {
+ return this.closed;
+ }
+
+ /**
+ * Check if the connection can be used. It checks if the connection is used by
+ * another thread or already closed.
+ *
+ * @return True if the connection can be used.
+ */
+ public synchronized boolean isUsable() {
+ return !isActive() && !isClosed();
+ }
+
+ /**
+ * Get the connection client.
+ *
+ * @return Connection client.
+ */
+ public synchronized ProxyAndInfo<ClientProtocol> getClient() {
+ this.numThreads++;
+ return this.client;
+ }
+
+ /**
+ * Release this connection. If the connection was closed, close the proxy.
+ * Otherwise, mark the connection as not used by us anymore.
+ */
+ public synchronized void release() {
+ if (--this.numThreads == 0 && this.closed) {
+ close();
+ }
+ }
+
+ /**
+ * We will not use this connection anymore. If it's not being used, we close
+ * it. Otherwise, we let release() do it once we are done with it.
+ */
+ public synchronized void close() {
+ this.closed = true;
+ if (this.numThreads == 0) {
+ ClientProtocol proxy = this.client.getProxy();
+ // Nobody should be using this anymore so it should close right away
+ RPC.stopProxy(proxy);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6632c5a4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java
new file mode 100644
index 0000000..d93d498
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java
@@ -0,0 +1,408 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Implements a pool of connections for the {@link Router} to be able to open
+ * many connections to many Namenodes.
+ */
+public class ConnectionManager {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ConnectionManager.class);
+
+ /** Number of parallel new connections to create. */
+ protected static final int MAX_NEW_CONNECTIONS = 100;
+
+ /** Minimum amount of active connections: 50%. */
+ protected static final float MIN_ACTIVE_RATIO = 0.5f;
+
+
+ /** Configuration for the connection manager, pool and sockets. */
+ private final Configuration conf;
+
+ /** Min number of connections per user + nn. */
+ private final int minSize = 1;
+ /** Max number of connections per user + nn. */
+ private final int maxSize;
+
+ /** How often we close a pool for a particular user + nn. */
+ private final long poolCleanupPeriodMs;
+ /** How often we close a connection in a pool. */
+ private final long connectionCleanupPeriodMs;
+
+ /** Map of connection pools, one pool per user + NN. */
+ private final Map<ConnectionPoolId, ConnectionPool> pools;
+ /** Lock for accessing pools. */
+ private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ private final Lock readLock = readWriteLock.readLock();
+ private final Lock writeLock = readWriteLock.writeLock();
+
+ /** Queue for creating new connections. */
+ private final BlockingQueue<ConnectionPool> creatorQueue =
+ new ArrayBlockingQueue<>(MAX_NEW_CONNECTIONS);
+ /** Create new connections asynchronously. */
+ private final ConnectionCreator creator;
+ /** Periodic executor to remove stale connection pools. */
+ private final ScheduledThreadPoolExecutor cleaner =
+ new ScheduledThreadPoolExecutor(1);
+
+ /** If the connection manager is running. */
+ private boolean running = false;
+
+
+ /**
+ * Creates a proxy client connection pool manager.
+ *
+ * @param config Configuration for the connections.
+ * @param minPoolSize Min size of the connection pool.
+ * @param maxPoolSize Max size of the connection pool.
+ */
+ public ConnectionManager(Configuration config) {
+ this.conf = config;
+
+ // Configure minimum and maximum connection pools
+ this.maxSize = this.conf.getInt(
+ DFSConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_POOL_SIZE,
+ DFSConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_POOL_SIZE_DEFAULT);
+
+ // Map with the connections indexed by UGI and Namenode
+ this.pools = new HashMap<>();
+
+ // Create connections in a thread asynchronously
+ this.creator = new ConnectionCreator(creatorQueue);
+ this.creator.setDaemon(true);
+
+ // Cleanup periods
+ this.poolCleanupPeriodMs = this.conf.getLong(
+ DFSConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_POOL_CLEAN,
+ DFSConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_POOL_CLEAN_DEFAULT);
+ LOG.info("Cleaning connection pools every {} seconds",
+ TimeUnit.MILLISECONDS.toSeconds(this.poolCleanupPeriodMs));
+ this.connectionCleanupPeriodMs = this.conf.getLong(
+ DFSConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_CLEAN_MS,
+ DFSConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_CLEAN_MS_DEFAULT);
+ LOG.info("Cleaning connections every {} seconds",
+ TimeUnit.MILLISECONDS.toSeconds(this.connectionCleanupPeriodMs));
+ }
+
+ /**
+ * Start the connection manager.
+ */
+ public void start() {
+ // Start the thread that creates connections asynchronously
+ this.creator.start();
+
+ // Schedule a task to remove stale connection pools and sockets
+ long recyleTimeMs = Math.min(
+ poolCleanupPeriodMs, connectionCleanupPeriodMs);
+ LOG.info("Cleaning every {} seconds",
+ TimeUnit.MILLISECONDS.toSeconds(recyleTimeMs));
+ this.cleaner.scheduleAtFixedRate(
+ new CleanupTask(), 0, recyleTimeMs, TimeUnit.MILLISECONDS);
+
+ // Mark the manager as running
+ this.running = true;
+ }
+
+ /**
+ * Stop the connection manager by closing all the pools.
+ */
+ public void close() {
+ this.creator.shutdown();
+ this.cleaner.shutdown();
+ this.running = false;
+
+ writeLock.lock();
+ try {
+ for (ConnectionPool pool : this.pools.values()) {
+ pool.close();
+ }
+ this.pools.clear();
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /**
+ * Fetches the next available proxy client in the pool. Each client connection
+ * is reserved for a single user and cannot be reused until free.
+ *
+ * @param ugi User group information.
+ * @param nnAddress Namenode address for the connection.
+ * @return Proxy client to connect to nnId as UGI.
+ * @throws IOException If the connection cannot be obtained.
+ */
+ public ConnectionContext getConnection(
+ UserGroupInformation ugi, String nnAddress) throws IOException {
+
+ // Check if the manager is shutdown
+ if (!this.running) {
+ LOG.error(
+ "Cannot get a connection to {} because the manager isn't running",
+ nnAddress);
+ return null;
+ }
+
+ // Try to get the pool if created
+ ConnectionPoolId connectionId = new ConnectionPoolId(ugi, nnAddress);
+ ConnectionPool pool = null;
+ readLock.lock();
+ try {
+ pool = this.pools.get(connectionId);
+ } finally {
+ readLock.unlock();
+ }
+
+ // Create the pool if not created before
+ if (pool == null) {
+ writeLock.lock();
+ try {
+ pool = this.pools.get(connectionId);
+ if (pool == null) {
+ pool = new ConnectionPool(
+ this.conf, nnAddress, ugi, this.minSize, this.maxSize);
+ this.pools.put(connectionId, pool);
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ ConnectionContext conn = pool.getConnection();
+
+ // Add a new connection to the pool if it wasn't usable
+ if (conn == null || !conn.isUsable()) {
+ if (!this.creatorQueue.offer(pool)) {
+ LOG.error("Cannot add more than {} connections at the same time",
+ MAX_NEW_CONNECTIONS);
+ }
+ }
+
+ if (conn != null && conn.isClosed()) {
+ LOG.error("We got a closed connection from {}", pool);
+ conn = null;
+ }
+
+ return conn;
+ }
+
+ /**
+ * Get the number of connection pools.
+ *
+ * @return Number of connection pools.
+ */
+ public int getNumConnectionPools() {
+ readLock.lock();
+ try {
+ return pools.size();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ /**
+ * Get number of open connections.
+ *
+ * @return Number of open connections.
+ */
+ public int getNumConnections() {
+ int total = 0;
+ readLock.lock();
+ try {
+ for (ConnectionPool pool : this.pools.values()) {
+ total += pool.getNumConnections();
+ }
+ } finally {
+ readLock.unlock();
+ }
+ return total;
+ }
+
+ /**
+ * Get number of active connections.
+ *
+ * @return Number of active connections.
+ */
+ public int getNumActiveConnections() {
+ int total = 0;
+ readLock.lock();
+ try {
+ for (ConnectionPool pool : this.pools.values()) {
+ total += pool.getNumActiveConnections();
+ }
+ } finally {
+ readLock.unlock();
+ }
+ return total;
+ }
+
+ /**
+ * Get the number of connections to be created.
+ *
+ * @return Number of connections to be created.
+ */
+ public int getNumCreatingConnections() {
+ return this.creatorQueue.size();
+ }
+
+ /**
+ * Removes stale connections not accessed recently from the pool. This is
+ * invoked periodically.
+ */
+ private class CleanupTask implements Runnable {
+
+ @Override
+ public void run() {
+ long currentTime = Time.now();
+ List<ConnectionPoolId> toRemove = new LinkedList<>();
+
+ // Look for stale pools
+ readLock.lock();
+ try {
+ for (Entry<ConnectionPoolId, ConnectionPool> entry : pools.entrySet()) {
+ ConnectionPool pool = entry.getValue();
+ long lastTimeActive = pool.getLastActiveTime();
+ boolean isStale =
+ currentTime > (lastTimeActive + poolCleanupPeriodMs);
+ if (lastTimeActive > 0 && isStale) {
+ // Remove this pool
+ LOG.debug("Closing and removing stale pool {}", pool);
+ pool.close();
+ ConnectionPoolId poolId = entry.getKey();
+ toRemove.add(poolId);
+ } else {
+ // Keep this pool but clean connections inside
+ LOG.debug("Cleaning up {}", pool);
+ cleanup(pool);
+ }
+ }
+ } finally {
+ readLock.unlock();
+ }
+
+ // Remove stale pools
+ if (!toRemove.isEmpty()) {
+ writeLock.lock();
+ try {
+ for (ConnectionPoolId poolId : toRemove) {
+ pools.remove(poolId);
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+ }
+
+ /**
+ * Clean the unused connections for this pool.
+ *
+ * @param pool Connection pool to cleanup.
+ */
+ private void cleanup(ConnectionPool pool) {
+ if (pool.getNumConnections() > pool.getMinSize()) {
+ // Check if the pool hasn't been active in a while or not 50% are used
+ long timeSinceLastActive = Time.now() - pool.getLastActiveTime();
+ int total = pool.getNumConnections();
+ int active = getNumActiveConnections();
+ if (timeSinceLastActive > connectionCleanupPeriodMs ||
+ active < MIN_ACTIVE_RATIO * total) {
+ // Remove and close 1 connection
+ List<ConnectionContext> conns = pool.removeConnections(1);
+ for (ConnectionContext conn : conns) {
+ conn.close();
+ }
+ LOG.debug("Removed connection {} used {} seconds ago. " +
+ "Pool has {}/{} connections", pool.getConnectionPoolId(),
+ TimeUnit.MILLISECONDS.toSeconds(timeSinceLastActive),
+ pool.getNumConnections(), pool.getMaxSize());
+ }
+ }
+ }
+ }
+
+ /**
+ * Thread that creates connections asynchronously.
+ */
+ private static class ConnectionCreator extends Thread {
+ /** If the creator is running. */
+ private boolean running = true;
+ /** Queue to push work to. */
+ private BlockingQueue<ConnectionPool> queue;
+
+ ConnectionCreator(BlockingQueue<ConnectionPool> blockingQueue) {
+ super("Connection creator");
+ this.queue = blockingQueue;
+ }
+
+ @Override
+ public void run() {
+ while (this.running) {
+ try {
+ ConnectionPool pool = this.queue.take();
+ try {
+ int total = pool.getNumConnections();
+ int active = pool.getNumActiveConnections();
+ if (pool.getNumConnections() < pool.getMaxSize() &&
+ active >= MIN_ACTIVE_RATIO * total) {
+ ConnectionContext conn = pool.newConnection();
+ pool.addConnection(conn);
+ } else {
+ LOG.debug("Cannot add more than {} connections to {}",
+ pool.getMaxSize(), pool);
+ }
+ } catch (IOException e) {
+ LOG.error("Cannot create a new connection", e);
+ }
+ } catch (InterruptedException e) {
+ LOG.error("The connection creator was interrupted");
+ this.running = false;
+ }
+ }
+ }
+
+ /**
+ * Stop this connection creator.
+ */
+ public void shutdown() {
+ this.running = false;
+ this.interrupt();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6632c5a4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java
new file mode 100644
index 0000000..f76f621
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java
@@ -0,0 +1,314 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import javax.net.SocketFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.NameNodeProxiesClient.ProxyAndInfo;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB;
+import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.io.retry.RetryUtils;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SaslRpcServer;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Maintains a pool of connections for each User (including tokens) + NN. The
+ * RPC client maintains a single socket, to achieve throughput similar to a NN,
+ * each request is multiplexed across multiple sockets/connections from a
+ * pool.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class ConnectionPool {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ConnectionPool.class);
+
+
+ /** Configuration settings for the connection pool. */
+ private final Configuration conf;
+
+ /** Identifier for this connection pool. */
+ private final ConnectionPoolId connectionPoolId;
+ /** Namenode this pool connects to. */
+ private final String namenodeAddress;
+ /** User for this connections. */
+ private final UserGroupInformation ugi;
+
+ /** Pool of connections. We mimic a COW array. */
+ private volatile List<ConnectionContext> connections = new ArrayList<>();
+ /** Connection index for round-robin. */
+ private final AtomicInteger clientIndex = new AtomicInteger(0);
+
+ /** Min number of connections per user. */
+ private final int minSize;
+ /** Max number of connections per user. */
+ private final int maxSize;
+
+ /** The last time a connection was active. */
+ private volatile long lastActiveTime = 0;
+
+
+ protected ConnectionPool(Configuration config, String address,
+ UserGroupInformation user, int minPoolSize, int maxPoolSize)
+ throws IOException {
+
+ this.conf = config;
+
+ // Connection pool target
+ this.ugi = user;
+ this.namenodeAddress = address;
+ this.connectionPoolId =
+ new ConnectionPoolId(this.ugi, this.namenodeAddress);
+
+ // Set configuration parameters for the pool
+ this.minSize = minPoolSize;
+ this.maxSize = maxPoolSize;
+
+ // Add minimum connections to the pool
+ for (int i=0; i<this.minSize; i++) {
+ ConnectionContext newConnection = newConnection();
+ this.connections.add(newConnection);
+ }
+ LOG.debug("Created connection pool \"{}\" with {} connections",
+ this.connectionPoolId, this.minSize);
+ }
+
+ /**
+ * Get the maximum number of connections allowed in this pool.
+ *
+ * @return Maximum number of connections.
+ */
+ protected int getMaxSize() {
+ return this.maxSize;
+ }
+
+ /**
+ * Get the minimum number of connections in this pool.
+ *
+ * @return Minimum number of connections.
+ */
+ protected int getMinSize() {
+ return this.minSize;
+ }
+
+ /**
+ * Get the connection pool identifier.
+ *
+ * @return Connection pool identifier.
+ */
+ protected ConnectionPoolId getConnectionPoolId() {
+ return this.connectionPoolId;
+ }
+
+ /**
+ * Return the next connection round-robin.
+ *
+ * @return Connection context.
+ */
+ protected ConnectionContext getConnection() {
+
+ this.lastActiveTime = Time.now();
+
+ // Get a connection from the pool following round-robin
+ ConnectionContext conn = null;
+ List<ConnectionContext> tmpConnections = this.connections;
+ int size = tmpConnections.size();
+ int threadIndex = this.clientIndex.getAndIncrement();
+ for (int i=0; i<size; i++) {
+ int index = (threadIndex + i) % size;
+ conn = tmpConnections.get(index);
+ if (conn != null && !conn.isUsable()) {
+ return conn;
+ }
+ }
+
+ // We return a connection even if it's active
+ return conn;
+ }
+
+ /**
+ * Add a connection to the current pool. It uses a Copy-On-Write approach.
+ *
+ * @param conns New connections to add to the pool.
+ */
+ public synchronized void addConnection(ConnectionContext conn) {
+ List<ConnectionContext> tmpConnections = new ArrayList<>(this.connections);
+ tmpConnections.add(conn);
+ this.connections = tmpConnections;
+
+ this.lastActiveTime = Time.now();
+ }
+
+ /**
+ * Remove connections from the current pool.
+ *
+ * @param num Number of connections to remove.
+ * @return Removed connections.
+ */
+ public synchronized List<ConnectionContext> removeConnections(int num) {
+ List<ConnectionContext> removed = new LinkedList<>();
+
+ // Remove and close the last connection
+ List<ConnectionContext> tmpConnections = new ArrayList<>();
+ for (int i=0; i<this.connections.size(); i++) {
+ ConnectionContext conn = this.connections.get(i);
+ if (i < this.minSize || i < this.connections.size() - num) {
+ tmpConnections.add(conn);
+ } else {
+ removed.add(conn);
+ }
+ }
+ this.connections = tmpConnections;
+
+ return removed;
+ }
+
+ /**
+ * Close the connection pool.
+ */
+ protected synchronized void close() {
+ long timeSinceLastActive = TimeUnit.MILLISECONDS.toSeconds(
+ Time.now() - getLastActiveTime());
+ LOG.debug("Shutting down connection pool \"{}\" used {} seconds ago",
+ this.connectionPoolId, timeSinceLastActive);
+
+ for (ConnectionContext connection : this.connections) {
+ connection.close();
+ }
+ this.connections.clear();
+ }
+
+ /**
+ * Number of connections in the pool.
+ *
+ * @return Number of connections.
+ */
+ protected int getNumConnections() {
+ return this.connections.size();
+ }
+
+ /**
+ * Number of active connections in the pool.
+ *
+ * @return Number of active connections.
+ */
+ protected int getNumActiveConnections() {
+ int ret = 0;
+
+ List<ConnectionContext> tmpConnections = this.connections;
+ for (ConnectionContext conn : tmpConnections) {
+ if (conn.isActive()) {
+ ret++;
+ }
+ }
+ return ret;
+ }
+
+ /**
+ * Get the last time the connection pool was used.
+ *
+ * @return Last time the connection pool was used.
+ */
+ protected long getLastActiveTime() {
+ return this.lastActiveTime;
+ }
+
+ @Override
+ public String toString() {
+ return this.connectionPoolId.toString();
+ }
+
+ /**
+ * Create a new proxy wrapper for a client NN connection.
+ * @return Proxy for the target ClientProtocol that contains the user's
+ * security context.
+ * @throws IOException
+ */
+ public ConnectionContext newConnection() throws IOException {
+ return newConnection(this.conf, this.namenodeAddress, this.ugi);
+ }
+
+ /**
+ * Creates a proxy wrapper for a client NN connection. Each proxy contains
+ * context for a single user/security context. To maximize throughput it is
+ * recommended to use multiple connection per user+server, allowing multiple
+ * writes and reads to be dispatched in parallel.
+ *
+ * @param conf Configuration for the connection.
+ * @param nnAddress Address of server supporting the ClientProtocol.
+ * @param ugi User context.
+ * @return Proxy for the target ClientProtocol that contains the user's
+ * security context.
+ * @throws IOException If it cannot be created.
+ */
+ protected static ConnectionContext newConnection(Configuration conf,
+ String nnAddress, UserGroupInformation ugi)
+ throws IOException {
+ RPC.setProtocolEngine(
+ conf, ClientNamenodeProtocolPB.class, ProtobufRpcEngine.class);
+
+ final RetryPolicy defaultPolicy = RetryUtils.getDefaultRetryPolicy(
+ conf,
+ HdfsClientConfigKeys.Retry.POLICY_ENABLED_KEY,
+ HdfsClientConfigKeys.Retry.POLICY_ENABLED_DEFAULT,
+ HdfsClientConfigKeys.Retry.POLICY_SPEC_KEY,
+ HdfsClientConfigKeys.Retry.POLICY_SPEC_DEFAULT,
+ HdfsConstants.SAFEMODE_EXCEPTION_CLASS_NAME);
+
+ SocketFactory factory = SocketFactory.getDefault();
+ if (UserGroupInformation.isSecurityEnabled()) {
+ SaslRpcServer.init(conf);
+ }
+ InetSocketAddress socket = NetUtils.createSocketAddr(nnAddress);
+ final long version = RPC.getProtocolVersion(ClientNamenodeProtocolPB.class);
+ ClientNamenodeProtocolPB proxy = RPC.getProtocolProxy(
+ ClientNamenodeProtocolPB.class, version, socket, ugi, conf,
+ factory, RPC.getRpcTimeout(conf), defaultPolicy, null).getProxy();
+ ClientProtocol client = new ClientNamenodeProtocolTranslatorPB(proxy);
+ Text dtService = SecurityUtil.buildTokenService(socket);
+
+ ProxyAndInfo<ClientProtocol> clientProxy =
+ new ProxyAndInfo<ClientProtocol>(client, dtService, socket);
+ ConnectionContext connection = new ConnectionContext(clientProxy);
+ return connection;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6632c5a4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java
new file mode 100644
index 0000000..a3a78de
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+
+/**
+ * Identifier for a connection for a user to a namenode.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class ConnectionPoolId implements Comparable<ConnectionPoolId> {
+
+ /** Namenode identifier. */
+ private final String nnId;
+ /** Information about the user. */
+ private final UserGroupInformation ugi;
+
+ /**
+ * New connection pool identifier.
+ *
+ * @param ugi Information of the user issuing the request.
+ * @param nnId Namenode address with port.
+ */
+ public ConnectionPoolId(final UserGroupInformation ugi, final String nnId) {
+ this.nnId = nnId;
+ this.ugi = ugi;
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = new HashCodeBuilder(17, 31)
+ .append(this.nnId)
+ .append(this.ugi.toString())
+ .append(this.getTokenIds())
+ .toHashCode();
+ return hash;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof ConnectionPoolId) {
+ ConnectionPoolId other = (ConnectionPoolId) o;
+ if (!this.nnId.equals(other.nnId)) {
+ return false;
+ }
+ if (!this.ugi.toString().equals(other.ugi.toString())) {
+ return false;
+ }
+ String thisTokens = this.getTokenIds().toString();
+ String otherTokens = other.getTokenIds().toString();
+ return thisTokens.equals(otherTokens);
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return this.ugi + " " + this.getTokenIds() + "->" + this.nnId;
+ }
+
+ @Override
+ public int compareTo(ConnectionPoolId other) {
+ int ret = this.nnId.compareTo(other.nnId);
+ if (ret == 0) {
+ ret = this.ugi.toString().compareTo(other.ugi.toString());
+ }
+ if (ret == 0) {
+ String thisTokens = this.getTokenIds().toString();
+ String otherTokens = other.getTokenIds().toString();
+ ret = thisTokens.compareTo(otherTokens);
+ }
+ return ret;
+ }
+
+ /**
+ * Get the token identifiers for this connection.
+ * @return List with the token identifiers.
+ */
+ private List<String> getTokenIds() {
+ List<String> tokenIds = new ArrayList<>();
+ Collection<Token<? extends TokenIdentifier>> tokens = this.ugi.getTokens();
+ for (Token<? extends TokenIdentifier> token : tokens) {
+ byte[] tokenIdBytes = token.getIdentifier();
+ String tokenId = Arrays.toString(tokenIdBytes);
+ tokenIds.add(tokenId);
+ }
+ Collections.sort(tokenIds);
+ return tokenIds;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6632c5a4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
index da6066b..a90c460 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
@@ -17,22 +17,52 @@
*/
package org.apache.hadoop.hdfs.server.federation.router;
+import org.apache.commons.lang.builder.HashCodeBuilder;
+
/**
- * Interface for objects that are unique to a namespace.
+ * Base class for objects that are unique to a namespace.
*/
-public interface RemoteLocationContext {
+public abstract class RemoteLocationContext
+ implements Comparable<RemoteLocationContext> {
/**
* Returns an identifier for a unique namespace.
*
* @return Namespace identifier.
*/
- String getNameserviceId();
+ public abstract String getNameserviceId();
/**
* Destination in this location. For example the path in a remote namespace.
*
* @return Destination in this location.
*/
- String getDest();
+ public abstract String getDest();
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder(17, 31)
+ .append(getNameserviceId())
+ .append(getDest())
+ .toHashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof RemoteLocationContext) {
+ RemoteLocationContext other = (RemoteLocationContext) obj;
+ return this.getNameserviceId().equals(other.getNameserviceId()) &&
+ this.getDest().equals(other.getDest());
+ }
+ return false;
+ }
+
+ @Override
+ public int compareTo(RemoteLocationContext info) {
+ int ret = this.getNameserviceId().compareTo(info.getNameserviceId());
+ if (ret == 0) {
+ ret = this.getDest().compareTo(info.getDest());
+ }
+ return ret;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6632c5a4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java
new file mode 100644
index 0000000..cd57d45
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.Arrays;
+
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Determines the remote client protocol method and the parameter list for a
+ * specific location.
+ */
+public class RemoteMethod {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RemoteMethod.class);
+
+
+ /** List of parameters: static and dynamic values, matchings types. */
+ private final Object[] params;
+ /** List of method parameters types, matches parameters. */
+ private final Class<?>[] types;
+ /** String name of the ClientProtocol method. */
+ private final String methodName;
+
+ /**
+ * Create a method with no parameters.
+ *
+ * @param method The string name of the ClientProtocol method.
+ */
+ public RemoteMethod(String method) {
+ this.params = null;
+ this.types = null;
+ this.methodName = method;
+ }
+
+ /**
+ * Creates a remote method generator.
+ *
+ * @param method The string name of the ClientProtocol method.
+ * @param pTypes A list of types to use to locate the specific method.
+ * @param pParams A list of parameters for the method. The order of the
+ * parameter list must match the order and number of the types.
+ * Parameters are grouped into 2 categories:
+ * <ul>
+ * <li>Static parameters that are immutable across locations.
+ * <li>Dynamic parameters that are determined for each location by a
+ * RemoteParam object. To specify a dynamic parameter, pass an
+ * instance of RemoteParam in place of the parameter value.
+ * </ul>
+ * @throws IOException If the types and parameter lists are not valid.
+ */
+ public RemoteMethod(String method, Class<?>[] pTypes, Object... pParams)
+ throws IOException {
+
+ if (pParams.length != pTypes.length) {
+ throw new IOException("Invalid parameters for method " + method);
+ }
+
+ this.params = pParams;
+ this.types = pTypes;
+ this.methodName = method;
+ }
+
+ /**
+ * Get the represented java method.
+ *
+ * @return Method
+ * @throws IOException If the method cannot be found.
+ */
+ public Method getMethod() throws IOException {
+ try {
+ if (types != null) {
+ return ClientProtocol.class.getDeclaredMethod(methodName, types);
+ } else {
+ return ClientProtocol.class.getDeclaredMethod(methodName);
+ }
+ } catch (NoSuchMethodException e) {
+ // Re-throw as an IOException
+ LOG.error("Cannot get method {} with types {}",
+ methodName, Arrays.toString(types), e);
+ throw new IOException(e);
+ } catch (SecurityException e) {
+ LOG.error("Cannot access method {} with types {}",
+ methodName, Arrays.toString(types), e);
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * Get the calling types for this method.
+ *
+ * @return An array of calling types.
+ */
+ public Class<?>[] getTypes() {
+ return this.types;
+ }
+
+ /**
+ * Generate a list of parameters for this specific location using no context.
+ *
+ * @return A list of parameters for the method customized for the location.
+ */
+ public Object[] getParams() {
+ return this.getParams(null);
+ }
+
+ /**
+ * Get the name of the method.
+ *
+ * @return Name of the method.
+ */
+ public String getMethodName() {
+ return this.methodName;
+ }
+
+ /**
+ * Generate a list of parameters for this specific location. Parameters are
+ * grouped into 2 categories:
+ * <ul>
+ * <li>Static parameters that are immutable across locations.
+ * <li>Dynamic parameters that are determined for each location by a
+ * RemoteParam object.
+ * </ul>
+ *
+ * @param context The context identifying the location.
+ * @return A list of parameters for the method customized for the location.
+ */
+ public Object[] getParams(RemoteLocationContext context) {
+ if (this.params == null) {
+ return new Object[] {};
+ }
+ Object[] objList = new Object[this.params.length];
+ for (int i = 0; i < this.params.length; i++) {
+ Object currentObj = this.params[i];
+ if (currentObj instanceof RemoteParam) {
+ // Map the parameter using the context
+ RemoteParam paramGetter = (RemoteParam) currentObj;
+ objList[i] = paramGetter.getParameterForContext(context);
+ } else {
+ objList[i] = currentObj;
+ }
+ }
+ return objList;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6632c5a4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteParam.java
new file mode 100644
index 0000000..8816ff6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteParam.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.util.Map;
+
+/**
+ * A dynamically assignable parameter that is location-specific.
+ * <p>
+ * There are 2 ways this mapping is determined:
+ * <ul>
+ * <li>Default: Uses the RemoteLocationContext's destination
+ * <li>Map: Uses the value of the RemoteLocationContext key provided in the
+ * parameter map.
+ * </ul>
+ */
+public class RemoteParam {
+
+ private final Map<? extends Object, ? extends Object> paramMap;
+
+ /**
+ * Constructs a default remote parameter. Always maps the value to the
+ * destination of the provided RemoveLocationContext.
+ */
+ public RemoteParam() {
+ this.paramMap = null;
+ }
+
+ /**
+ * Constructs a map based remote parameter. Determines the value using the
+ * provided RemoteLocationContext as a key into the map.
+ *
+ * @param map Map with RemoteLocationContext keys.
+ */
+ public RemoteParam(
+ Map<? extends RemoteLocationContext, ? extends Object> map) {
+ this.paramMap = map;
+ }
+
+ /**
+ * Determine the appropriate value for this parameter based on the location.
+ *
+ * @param context Context identifying the location.
+ * @return A parameter specific to this location.
+ */
+ public Object getParameterForContext(RemoteLocationContext context) {
+ if (context == null) {
+ return null;
+ } else if (this.paramMap != null) {
+ return this.paramMap.get(context);
+ } else {
+ // Default case
+ return context.getDest();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6632c5a4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
index fe0d02a..019a5cd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
@@ -22,12 +22,14 @@ import static org.apache.hadoop.hdfs.server.federation.router.FederationUtil.new
import static org.apache.hadoop.util.ExitUtil.terminate;
import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
@@ -36,6 +38,8 @@ import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Router that provides a unified view of multiple federated HDFS clusters. It
@@ -60,7 +64,7 @@ import org.apache.hadoop.util.StringUtils;
@InterfaceStability.Evolving
public class Router extends CompositeService {
- private static final Log LOG = LogFactory.getLog(Router.class);
+ private static final Logger LOG = LoggerFactory.getLogger(Router.class);
/** Configuration for the Router. */
@@ -71,6 +75,7 @@ public class Router extends CompositeService {
/** RPC interface to the client. */
private RouterRpcServer rpcServer;
+ private InetSocketAddress rpcAddress;
/** Interface with the State Store. */
private StateStoreService stateStore;
@@ -105,9 +110,6 @@ public class Router extends CompositeService {
protected void serviceInit(Configuration configuration) throws Exception {
this.conf = configuration;
- // TODO Interface to the State Store
- this.stateStore = null;
-
// Resolver to track active NNs
this.namenodeResolver = newActiveNamenodeResolver(
this.conf, this.stateStore);
@@ -122,6 +124,15 @@ public class Router extends CompositeService {
throw new IOException("Cannot find subcluster resolver");
}
+ if (conf.getBoolean(
+ DFSConfigKeys.DFS_ROUTER_RPC_ENABLE,
+ DFSConfigKeys.DFS_ROUTER_RPC_ENABLE_DEFAULT)) {
+ // Create RPC server
+ this.rpcServer = createRpcServer();
+ addService(this.rpcServer);
+ this.setRpcServerAddress(rpcServer.getRpcAddress());
+ }
+
super.serviceInit(conf);
}
@@ -171,11 +182,13 @@ public class Router extends CompositeService {
router.init(conf);
router.start();
} catch (Throwable e) {
- LOG.error("Failed to start router.", e);
+ LOG.error("Failed to start router", e);
terminate(1, e);
}
}
+
+
/////////////////////////////////////////////////////////
// RPC Server
/////////////////////////////////////////////////////////
@@ -183,7 +196,7 @@ public class Router extends CompositeService {
/**
* Create a new Router RPC server to proxy ClientProtocol requests.
*
- * @return RouterRpcServer
+ * @return New Router RPC Server.
* @throws IOException If the router RPC server was not started.
*/
protected RouterRpcServer createRpcServer() throws IOException {
@@ -200,6 +213,35 @@ public class Router extends CompositeService {
return this.rpcServer;
}
+ /**
+ * Set the current RPC socket for the router.
+ *
+ * @param rpcAddress RPC address.
+ */
+ protected void setRpcServerAddress(InetSocketAddress address) {
+ this.rpcAddress = address;
+
+ // Use the RPC address as our unique router Id
+ if (this.rpcAddress != null) {
+ try {
+ String hostname = InetAddress.getLocalHost().getHostName();
+ setRouterId(hostname + ":" + this.rpcAddress.getPort());
+ } catch (UnknownHostException ex) {
+ LOG.error("Cannot set unique router ID, address not resolvable {}",
+ this.rpcAddress);
+ }
+ }
+ }
+
+ /**
+ * Get the current RPC socket address for the router.
+ *
+ * @return InetSocketAddress
+ */
+ public InetSocketAddress getRpcServerAddress() {
+ return this.rpcAddress;
+ }
+
/////////////////////////////////////////////////////////
// Submodule getters
/////////////////////////////////////////////////////////
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: HDFS-10629. Federation Roter.
Contributed by Jason Kace and Inigo Goiri.
Posted by in...@apache.org.
HDFS-10629. Federation Roter. Contributed by Jason Kace and Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d710513a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d710513a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d710513a
Branch: refs/heads/HDFS-10467
Commit: d710513acaee4fb5bc2588f04998d7701b063850
Parents: 51b671e
Author: Inigo <in...@apache.org>
Authored: Tue Mar 28 14:30:59 2017 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu May 11 09:54:07 2017 -0700
----------------------------------------------------------------------
.../hadoop-hdfs/src/main/bin/hdfs | 5 +
.../hadoop-hdfs/src/main/bin/hdfs.cmd | 8 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 17 +
.../resolver/ActiveNamenodeResolver.java | 117 +++
.../resolver/FederationNamenodeContext.java | 87 +++
.../FederationNamenodeServiceState.java | 46 ++
.../resolver/FederationNamespaceInfo.java | 99 +++
.../resolver/FileSubclusterResolver.java | 75 ++
.../resolver/NamenodePriorityComparator.java | 63 ++
.../resolver/NamenodeStatusReport.java | 195 +++++
.../federation/resolver/PathLocation.java | 122 +++
.../federation/resolver/RemoteLocation.java | 74 ++
.../federation/resolver/package-info.java | 41 +
.../federation/router/FederationUtil.java | 117 +++
.../router/RemoteLocationContext.java | 38 +
.../hdfs/server/federation/router/Router.java | 263 +++++++
.../federation/router/RouterRpcServer.java | 102 +++
.../server/federation/router/package-info.java | 31 +
.../federation/store/StateStoreService.java | 77 ++
.../server/federation/store/package-info.java | 62 ++
.../src/main/resources/hdfs-default.xml | 16 +
.../server/federation/FederationTestUtils.java | 233 ++++++
.../hdfs/server/federation/MockResolver.java | 290 +++++++
.../server/federation/RouterConfigBuilder.java | 40 +
.../server/federation/RouterDFSCluster.java | 767 +++++++++++++++++++
.../server/federation/router/TestRouter.java | 96 +++
26 files changed, 3080 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d710513a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index 594a468..6606979 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -57,6 +57,7 @@ function hadoop_usage
hadoop_add_subcommand "oiv" "apply the offline fsimage viewer to an fsimage"
hadoop_add_subcommand "oiv_legacy" "apply the offline fsimage viewer to a legacy fsimage"
hadoop_add_subcommand "portmap" "run a portmap service"
+ hadoop_add_subcommand "router" "run the DFS router"
hadoop_add_subcommand "secondarynamenode" "run the DFS secondary namenode"
hadoop_add_subcommand "snapshotDiff" "diff two snapshots of a directory or diff the current directory contents with a snapshot"
hadoop_add_subcommand "storagepolicies" "list/get/set block storage policies"
@@ -176,6 +177,10 @@ function hdfscmd_case
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME=org.apache.hadoop.portmap.Portmap
;;
+ router)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.federation.router.Router'
+ ;;
secondarynamenode)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode'
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d710513a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
index 2181e47..b9853d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
@@ -59,7 +59,7 @@ if "%1" == "--loglevel" (
)
)
- set hdfscommands=dfs namenode secondarynamenode journalnode zkfc datanode dfsadmin haadmin fsck balancer jmxget oiv oev fetchdt getconf groups snapshotDiff lsSnapshottableDir cacheadmin mover storagepolicies classpath crypto debug
+ set hdfscommands=dfs namenode secondarynamenode journalnode zkfc datanode dfsadmin haadmin fsck balancer jmxget oiv oev fetchdt getconf groups snapshotDiff lsSnapshottableDir cacheadmin mover storagepolicies classpath crypto router debug
for %%i in ( %hdfscommands% ) do (
if %hdfs-command% == %%i set hdfscommand=true
)
@@ -179,6 +179,11 @@ goto :eof
set CLASS=org.apache.hadoop.hdfs.tools.CryptoAdmin
goto :eof
+:router
+ set CLASS=org.apache.hadoop.hdfs.server.federation.router.Router
+ set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_ROUTER_OPTS%
+ goto :eof
+
:debug
set CLASS=org.apache.hadoop.hdfs.tools.DebugAdmin
goto :eof
@@ -219,6 +224,7 @@ goto :eof
@echo secondarynamenode run the DFS secondary namenode
@echo namenode run the DFS namenode
@echo journalnode run the DFS journalnode
+ @echo router run the DFS router
@echo zkfc run the ZK Failover Controller daemon
@echo datanode run a DFS datanode
@echo dfsadmin run a DFS admin client
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d710513a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index b95c7e6..0e04c63 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -1089,6 +1089,23 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
"dfs.use.dfs.network.topology";
public static final boolean DFS_USE_DFS_NETWORK_TOPOLOGY_DEFAULT = false;
+ // HDFS federation
+ public static final String FEDERATION_PREFIX = "dfs.federation.";
+
+ // HDFS Router-based federation
+ public static final String FEDERATION_ROUTER_PREFIX =
+ "dfs.federation.router.";
+
+ // HDFS Router State Store connection
+ public static final String FEDERATION_FILE_RESOLVER_CLIENT_CLASS =
+ FEDERATION_ROUTER_PREFIX + "file.resolver.client.class";
+ public static final String FEDERATION_FILE_RESOLVER_CLIENT_CLASS_DEFAULT =
+ "org.apache.hadoop.hdfs.server.federation.MockResolver";
+ public static final String FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS =
+ FEDERATION_ROUTER_PREFIX + "namenode.resolver.client.class";
+ public static final String FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS_DEFAULT =
+ "org.apache.hadoop.hdfs.server.federation.MockResolver";
+
// dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry
@Deprecated
public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d710513a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java
new file mode 100644
index 0000000..477053d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Locates the most active NN for a given nameservice ID or blockpool ID. This
+ * interface is used by the {@link org.apache.hadoop.hdfs.server.federation.
+ * router.RouterRpcServer RouterRpcServer} to:
+ * <ul>
+ * <li>Determine the target NN for a given subcluster.
+ * <li>List of all namespaces discovered/active in the federation.
+ * <li>Update the currently active NN empirically.
+ * </ul>
+ * The interface is also used by the {@link org.apache.hadoop.hdfs.server.
+ * federation.router.NamenodeHeartbeatService NamenodeHeartbeatService} to
+ * register a discovered NN.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface ActiveNamenodeResolver {
+
+ /**
+ * Report a successful, active NN address for a nameservice or blockPool.
+ *
+ * @param ns Nameservice identifier.
+ * @param successfulAddress The address the successful responded to the
+ * command.
+ * @throws IOException If the state store cannot be accessed.
+ */
+ void updateActiveNamenode(
+ String ns, InetSocketAddress successfulAddress) throws IOException;
+
+ /**
+ * Returns a prioritized list of the most recent cached registration entries
+ * for a single nameservice ID.
+ * Returns an empty list if none are found. Returns entries in preference of:
+ * <ul>
+ * <li>The most recent ACTIVE NN
+ * <li>The most recent STANDBY NN
+ * <li>The most recent UNAVAILABLE NN
+ * </ul>
+ *
+ * @param nameserviceId Nameservice identifier.
+ * @return Prioritized list of namenode contexts.
+ * @throws IOException If the state store cannot be accessed.
+ */
+ List<? extends FederationNamenodeContext>
+ getNamenodesForNameserviceId(String nameserviceId) throws IOException;
+
+ /**
+ * Returns a prioritized list of the most recent cached registration entries
+ * for a single block pool ID.
+ * Returns an empty list if none are found. Returns entries in preference of:
+ * <ul>
+ * <li>The most recent ACTIVE NN
+ * <li>The most recent STANDBY NN
+ * <li>The most recent UNAVAILABLE NN
+ * </ul>
+ *
+ * @param blockPoolId Block pool identifier for the nameservice.
+ * @return Prioritized list of namenode contexts.
+ * @throws IOException If the state store cannot be accessed.
+ */
+ List<? extends FederationNamenodeContext>
+ getNamenodesForBlockPoolId(String blockPoolId) throws IOException;
+
+ /**
+ * Register a namenode in the State Store.
+ *
+ * @param report Namenode status report.
+ * @return True if the node was registered and successfully committed to the
+ * data store.
+ * @throws IOException Throws exception if the namenode could not be
+ * registered.
+ */
+ boolean registerNamenode(NamenodeStatusReport report) throws IOException;
+
+ /**
+ * Get a list of all namespaces that are registered and active in the
+ * federation.
+ *
+ * @return List of name spaces in the federation
+ * @throws Throws exception if the namespace list is not available.
+ */
+ Set<FederationNamespaceInfo> getNamespaces() throws IOException;
+
+ /**
+ * Assign a unique identifier for the parent router service.
+ * Required to report the status to the namenode resolver.
+ *
+ * @param router Unique string identifier for the router.
+ */
+ void setRouterId(String routerId);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d710513a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeContext.java
new file mode 100644
index 0000000..68ef02a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeContext.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+/**
+ * Interface for a discovered NN and its current server endpoints.
+ */
+public interface FederationNamenodeContext {
+
+ /**
+ * Get the RPC server address of the namenode.
+ *
+ * @return RPC server address in the form of host:port.
+ */
+ String getRpcAddress();
+
+ /**
+ * Get the Service RPC server address of the namenode.
+ *
+ * @return Service RPC server address in the form of host:port.
+ */
+ String getServiceAddress();
+
+ /**
+ * Get the Lifeline RPC server address of the namenode.
+ *
+ * @return Lifeline RPC server address in the form of host:port.
+ */
+ String getLifelineAddress();
+
+ /**
+ * Get the HTTP server address of the namenode.
+ *
+ * @return HTTP address in the form of host:port.
+ */
+ String getWebAddress();
+
+ /**
+ * Get the unique key representing the namenode.
+ *
+ * @return Combination of the nameservice and the namenode IDs.
+ */
+ String getNamenodeKey();
+
+ /**
+ * Identifier for the nameservice/namespace.
+ *
+ * @return Namenode nameservice identifier.
+ */
+ String getNameserviceId();
+
+ /**
+ * Identifier for the namenode.
+ *
+ * @return String
+ */
+ String getNamenodeId();
+
+ /**
+ * The current state of the namenode (active, standby, etc).
+ *
+ * @return FederationNamenodeServiceState State of the namenode.
+ */
+ FederationNamenodeServiceState getState();
+
+ /**
+ * The update date.
+ *
+ * @return Long with the update date.
+ */
+ long getDateModified();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d710513a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java
new file mode 100644
index 0000000..c773f82
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+
+/**
+ * Namenode state in the federation. The order of this enum is used to evaluate
+ * NN priority for RPC calls.
+ */
+public enum FederationNamenodeServiceState {
+ ACTIVE, // HAServiceState.ACTIVE or operational.
+ STANDBY, // HAServiceState.STANDBY.
+ UNAVAILABLE, // When the namenode cannot be reached.
+ EXPIRED; // When the last update is too old.
+
+ public static FederationNamenodeServiceState getState(HAServiceState state) {
+ switch(state) {
+ case ACTIVE:
+ return FederationNamenodeServiceState.ACTIVE;
+ case STANDBY:
+ return FederationNamenodeServiceState.STANDBY;
+ case INITIALIZING:
+ return FederationNamenodeServiceState.UNAVAILABLE;
+ case STOPPING:
+ return FederationNamenodeServiceState.UNAVAILABLE;
+ default:
+ return FederationNamenodeServiceState.UNAVAILABLE;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d710513a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java
new file mode 100644
index 0000000..bbaeca3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import org.apache.hadoop.hdfs.server.federation.router.RemoteLocationContext;
+
+/**
+ * Represents information about a single nameservice/namespace in a federated
+ * HDFS cluster.
+ */
+public class FederationNamespaceInfo
+ implements Comparable<FederationNamespaceInfo>, RemoteLocationContext {
+
+ /** Block pool identifier. */
+ private String blockPoolId;
+ /** Cluster identifier. */
+ private String clusterId;
+ /** Nameservice identifier. */
+ private String nameserviceId;
+
+ public FederationNamespaceInfo(String bpId, String clId, String nsId) {
+ this.blockPoolId = bpId;
+ this.clusterId = clId;
+ this.nameserviceId = nsId;
+ }
+
+ /**
+ * The HDFS nameservice id for this namespace.
+ *
+ * @return Nameservice identifier.
+ */
+ public String getNameserviceId() {
+ return this.nameserviceId;
+ }
+
+ /**
+ * The HDFS cluster id for this namespace.
+ *
+ * @return Cluster identifier.
+ */
+ public String getClusterId() {
+ return this.clusterId;
+ }
+
+ /**
+ * The HDFS block pool id for this namespace.
+ *
+ * @return Block pool identifier.
+ */
+ public String getBlockPoolId() {
+ return this.blockPoolId;
+ }
+
+ @Override
+ public int hashCode() {
+ return this.nameserviceId.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ } else if (obj instanceof FederationNamespaceInfo) {
+ return this.compareTo((FederationNamespaceInfo) obj) == 0;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public int compareTo(FederationNamespaceInfo info) {
+ return this.nameserviceId.compareTo(info.getNameserviceId());
+ }
+
+ @Override
+ public String toString() {
+ return this.nameserviceId + "->" + this.blockPoolId + ":" + this.clusterId;
+ }
+
+ @Override
+ public String getDest() {
+ return this.nameserviceId;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d710513a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FileSubclusterResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FileSubclusterResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FileSubclusterResolver.java
new file mode 100644
index 0000000..af9f493
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FileSubclusterResolver.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Interface to map a file path in the global name space to a specific
+ * subcluster and path in an HDFS name space.
+ * <p>
+ * Each path in the global/federated namespace may map to 1-N different HDFS
+ * locations. Each location specifies a single nameservice and a single HDFS
+ * path. The behavior is similar to MergeFS and Nfly and allows the merger
+ * of multiple HDFS locations into a single path. See HADOOP-8298 and
+ * HADOOP-12077
+ * <p>
+ * For example, a directory listing will fetch listings for each destination
+ * path and combine them into a single set of results.
+ * <p>
+ * When multiple destinations are available for a path, the destinations are
+ * prioritized in a consistent manner. This allows the proxy server to
+ * guess the best/most likely destination and attempt it first.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface FileSubclusterResolver {
+
+ /**
+ * Get the destinations for a global path. Results are from the mount table
+ * cache. If multiple destinations are available, the first result is the
+ * highest priority destination.
+ *
+ * @param path Global path.
+ * @return Location in a destination namespace or null if it does not exist.
+ * @throws IOException Throws exception if the data is not available.
+ */
+ PathLocation getDestinationForPath(String path) throws IOException;
+
+ /**
+ * Get a list of mount points for a path. Results are from the mount table
+ * cache.
+ *
+ * @return List of mount points present at this path or zero-length list if
+ * none are found.
+ * @throws IOException Throws exception if the data is not available.
+ */
+ List<String> getMountPoints(String path) throws IOException;
+
+ /**
+ * Get the default namespace for the cluster.
+ *
+ * @return Default namespace identifier.
+ */
+ String getDefaultNamespace();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d710513a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodePriorityComparator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodePriorityComparator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodePriorityComparator.java
new file mode 100644
index 0000000..fe82f29
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodePriorityComparator.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import java.util.Comparator;
+
+/**
+ * Compares NNs in the same namespace and prioritizes by their status. The
+ * priorities are:
+ * <ul>
+ * <li>ACTIVE
+ * <li>STANDBY
+ * <li>UNAVAILABLE
+ * </ul>
+ * When two NNs have the same state, the last modification date is the tie
+ * breaker, newest has priority. Expired NNs are excluded.
+ */
+public class NamenodePriorityComparator
+ implements Comparator<FederationNamenodeContext> {
+
+ @Override
+ public int compare(FederationNamenodeContext o1,
+ FederationNamenodeContext o2) {
+ FederationNamenodeServiceState state1 = o1.getState();
+ FederationNamenodeServiceState state2 = o2.getState();
+
+ if (state1 == state2) {
+ // Both have the same state, use mode dates
+ return compareModDates(o1, o2);
+ } else {
+ // Enum is ordered by priority
+ return state1.compareTo(state2);
+ }
+ }
+
+ /**
+ * Compare the modification dates.
+ *
+ * @param o1 Context 1.
+ * @param o2 Context 2.
+ * @return Comparison between dates.
+ */
+ private int compareModDates(FederationNamenodeContext o1,
+ FederationNamenodeContext o2) {
+ // Reverse sort, lowest position is highest priority.
+ return (int) (o2.getDateModified() - o1.getDateModified());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d710513a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java
new file mode 100644
index 0000000..9259048
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java
@@ -0,0 +1,195 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
+
+/**
+ * Status of the namenode.
+ */
+public class NamenodeStatusReport {
+
+ /** Namenode information. */
+ private String nameserviceId = "";
+ private String namenodeId = "";
+ private String clusterId = "";
+ private String blockPoolId = "";
+ private String rpcAddress = "";
+ private String serviceAddress = "";
+ private String lifelineAddress = "";
+ private String webAddress = "";
+
+ /** Namenode state. */
+ private HAServiceState status = HAServiceState.STANDBY;
+ private boolean safeMode = false;
+
+ /** If the fields are valid. */
+ private boolean registrationValid = false;
+ private boolean haStateValid = false;
+
+ public NamenodeStatusReport(String ns, String nn, String rpc, String service,
+ String lifeline, String web) {
+ this.nameserviceId = ns;
+ this.namenodeId = nn;
+ this.rpcAddress = rpc;
+ this.serviceAddress = service;
+ this.lifelineAddress = lifeline;
+ this.webAddress = web;
+ }
+
+ /**
+ * If the registration is valid.
+ *
+ * @return If the registration is valid.
+ */
+ public boolean registrationValid() {
+ return this.registrationValid;
+ }
+
+ /**
+ * If the HA state is valid.
+ *
+ * @return If the HA state is valid.
+ */
+ public boolean haStateValid() {
+ return this.haStateValid;
+ }
+
+ /**
+ * Get the state of the Namenode being monitored.
+ *
+ * @return State of the Namenode.
+ */
+ public FederationNamenodeServiceState getState() {
+ if (!registrationValid) {
+ return FederationNamenodeServiceState.UNAVAILABLE;
+ } else if (haStateValid) {
+ return FederationNamenodeServiceState.getState(status);
+ } else {
+ return FederationNamenodeServiceState.ACTIVE;
+ }
+ }
+
+ /**
+ * Get the name service identifier.
+ *
+ * @return The name service identifier.
+ */
+ public String getNameserviceId() {
+ return this.nameserviceId;
+ }
+
+ /**
+ * Get the namenode identifier.
+ *
+ * @return The namenode identifier.
+ */
+ public String getNamenodeId() {
+ return this.namenodeId;
+ }
+
+ /**
+ * Get the cluster identifier.
+ *
+ * @return The cluster identifier.
+ */
+ public String getClusterId() {
+ return this.clusterId;
+ }
+
+ /**
+ * Get the block pool identifier.
+ *
+ * @return The block pool identifier.
+ */
+ public String getBlockPoolId() {
+ return this.blockPoolId;
+ }
+
+ /**
+ * Get the RPC address.
+ *
+ * @return The RPC address.
+ */
+ public String getRpcAddress() {
+ return this.rpcAddress;
+ }
+
+ /**
+ * Get the Service RPC address.
+ *
+ * @return The Service RPC address.
+ */
+ public String getServiceAddress() {
+ return this.serviceAddress;
+ }
+
+ /**
+ * Get the Lifeline RPC address.
+ *
+ * @return The Lifeline RPC address.
+ */
+ public String getLifelineAddress() {
+ return this.lifelineAddress;
+ }
+
+ /**
+ * Get the web address.
+ *
+ * @return The web address.
+ */
+ public String getWebAddress() {
+ return this.webAddress;
+ }
+
+ /**
+ * Get the HA service state.
+ *
+ * @return The HA service state.
+ */
+ public void setHAServiceState(HAServiceState state) {
+ this.status = state;
+ this.haStateValid = true;
+ }
+
+ /**
+ * Set the namespace information.
+ *
+ * @param info Namespace information.
+ */
+ public void setNamespaceInfo(NamespaceInfo info) {
+ this.clusterId = info.getClusterID();
+ this.blockPoolId = info.getBlockPoolID();
+ this.registrationValid = true;
+ }
+
+ public void setSafeMode(boolean safemode) {
+ this.safeMode = safemode;
+ }
+
+ public boolean getSafemode() {
+ return this.safeMode;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("%s-%s:%s",
+ nameserviceId, namenodeId, serviceAddress);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d710513a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/PathLocation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/PathLocation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/PathLocation.java
new file mode 100644
index 0000000..d90565c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/PathLocation.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * A map of the properties and target destinations (name space + path) for
+ * a path in the global/federated namespace.
+ * This data is generated from the @see MountTable records.
+ */
+public class PathLocation {
+
+ /** Source path in global namespace. */
+ private final String sourcePath;
+
+ /** Remote paths in the target namespaces. */
+ private final List<RemoteLocation> destinations;
+
+ /** List of name spaces present. */
+ private final Set<String> namespaces;
+
+
+ /**
+ * Create a new PathLocation.
+ *
+ * @param source Source path in the global name space.
+ * @param dest Destinations of the mount table entry.
+ * @param namespaces Unique identifier representing the combination of
+ * name spaces present in the destination list.
+ */
+ public PathLocation(
+ String source, List<RemoteLocation> dest, Set<String> nss) {
+ this.sourcePath = source;
+ this.destinations = dest;
+ this.namespaces = nss;
+ }
+
+ /**
+ * Create a path location from another path.
+ *
+ * @param other Other path location to copy from.
+ */
+ public PathLocation(PathLocation other) {
+ this.sourcePath = other.sourcePath;
+ this.destinations = new LinkedList<RemoteLocation>(other.destinations);
+ this.namespaces = new HashSet<String>(other.namespaces);
+ }
+
+ /**
+ * Get the source path in the global namespace for this path location.
+ *
+ * @return The path in the global namespace.
+ */
+ public String getSourcePath() {
+ return this.sourcePath;
+ }
+
+ /**
+ * Get the list of subclusters defined for the destinations.
+ */
+ public Set<String> getNamespaces() {
+ return Collections.unmodifiableSet(this.namespaces);
+ }
+
+ @Override
+ public String toString() {
+ RemoteLocation loc = getDefaultLocation();
+ return loc.getNameserviceId() + "->" + loc.getDest();
+ }
+
+ /**
+ * Check if this location supports multiple clusters/paths.
+ *
+ * @return If it has multiple destinations.
+ */
+ public boolean hasMultipleDestinations() {
+ return this.destinations.size() > 1;
+ }
+
+ /**
+ * Get the list of locations found in the mount table.
+ * The first result is the highest priority path.
+ *
+ * @return List of remote locations.
+ */
+ public List<RemoteLocation> getDestinations() {
+ return Collections.unmodifiableList(this.destinations);
+ }
+
+ /**
+ * Get the default or highest priority location.
+ *
+ * @return The default location.
+ */
+ public RemoteLocation getDefaultLocation() {
+ if (destinations.isEmpty() || destinations.get(0).getDest() == null) {
+ throw new UnsupportedOperationException(
+ "Unsupported path " + sourcePath + " please check mount table");
+ }
+ return destinations.get(0);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d710513a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java
new file mode 100644
index 0000000..eef136d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.hadoop.hdfs.server.federation.router.RemoteLocationContext;
+
+/**
+ * A single in a remote namespace consisting of a nameservice ID
+ * and a HDFS path.
+ */
+public class RemoteLocation implements RemoteLocationContext {
+
+ /** Identifier of the remote namespace for this location. */
+ private String nameserviceId;
+ /** Path in the remote location. */
+ private String path;
+
+ /**
+ * Create a new remote location.
+ *
+ * @param nsId Destination namespace.
+ * @param pPath Path in the destination namespace.
+ */
+ public RemoteLocation(String nsId, String pPath) {
+ this.nameserviceId = nsId;
+ this.path = pPath;
+ }
+
+ @Override
+ public String getNameserviceId() {
+ return this.nameserviceId;
+ }
+
+ @Override
+ public String getDest() {
+ return this.path;
+ }
+
+ @Override
+ public String toString() {
+ return this.nameserviceId + "->" + this.path;
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder(17, 31)
+ .append(this.nameserviceId)
+ .append(this.path)
+ .toHashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return (obj != null &&
+ obj.getClass() == this.getClass() &&
+ obj.hashCode() == this.hashCode());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d710513a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/package-info.java
new file mode 100644
index 0000000..d8be9e3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/package-info.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The resolver package contains indepedent data resolvers used in HDFS
+ * federation. The data resolvers collect data from the cluster, including from
+ * the state store. The resolvers expose APIs used by HDFS federation to collect
+ * aggregated, cached data for use in Real-time request processing. The
+ * resolvers are perf-sensitive and are used in the flow of the
+ * {@link RouterRpcServer} request path.
+ * <p>
+ * The principal resolvers are:
+ * <ul>
+ * <li>{@link ActiveNamenodeResolver} Real-time interface for locating the most
+ * recently active NN for a nameservice.
+ * <li>{@link FileSubclusterResolver} Real-time interface for determining the NN
+ * and local file path for a given file/folder based on the global namespace
+ * path.
+ * </ul>
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d710513a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
new file mode 100644
index 0000000..6e7e865
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.lang.reflect.Constructor;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+
+/**
+ * Utilities for managing HDFS federation.
+ */
+public final class FederationUtil {
+
+ private static final Log LOG = LogFactory.getLog(FederationUtil.class);
+
+ private FederationUtil() {
+ // Utility Class
+ }
+
+ /**
+ * Create an instance of an interface with a constructor using a state store
+ * constructor.
+ *
+ * @param conf Configuration
+ * @param context Context object to pass to the instance.
+ * @param contextType Type of the context passed to the constructor.
+ * @param configurationKeyName Configuration key to retrieve the class to load
+ * @param defaultClassName Default class to load if the configuration key is
+ * not set
+ * @param clazz Class/interface that must be implemented by the instance.
+ * @return New instance of the specified class that implements the desired
+ * interface and a single parameter constructor containing a
+ * StateStore reference.
+ */
+ private static <T, R> T newInstance(final Configuration conf,
+ final R context, final Class<R> contextClass,
+ final String configKeyName, final String defaultClassName,
+ final Class<T> clazz) {
+
+ String className = conf.get(configKeyName, defaultClassName);
+ try {
+ Class<?> instance = conf.getClassByName(className);
+ if (clazz.isAssignableFrom(instance)) {
+ if (contextClass == null) {
+ // Default constructor if no context
+ @SuppressWarnings("unchecked")
+ Constructor<T> constructor =
+ (Constructor<T>) instance.getConstructor();
+ return constructor.newInstance();
+ } else {
+ // Constructor with context
+ @SuppressWarnings("unchecked")
+ Constructor<T> constructor = (Constructor<T>) instance.getConstructor(
+ Configuration.class, contextClass);
+ return constructor.newInstance(conf, context);
+ }
+ } else {
+ throw new RuntimeException("Class " + className + " not instance of "
+ + clazz.getCanonicalName());
+ }
+ } catch (ReflectiveOperationException e) {
+ LOG.error("Could not instantiate: " + className, e);
+ return null;
+ }
+ }
+
+ /**
+ * Creates an instance of a FileSubclusterResolver from the configuration.
+ *
+ * @param conf Configuration that defines the file resolver class.
+ * @param obj Context object passed to class constructor.
+ * @return FileSubclusterResolver
+ */
+ public static FileSubclusterResolver newFileSubclusterResolver(
+ Configuration conf, StateStoreService stateStore) {
+ return newInstance(conf, stateStore, StateStoreService.class,
+ DFSConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
+ DFSConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS_DEFAULT,
+ FileSubclusterResolver.class);
+ }
+
+ /**
+ * Creates an instance of an ActiveNamenodeResolver from the configuration.
+ *
+ * @param conf Configuration that defines the namenode resolver class.
+ * @param obj Context object passed to class constructor.
+ * @return ActiveNamenodeResolver
+ */
+ public static ActiveNamenodeResolver newActiveNamenodeResolver(
+ Configuration conf, StateStoreService stateStore) {
+ return newInstance(conf, stateStore, StateStoreService.class,
+ DFSConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS,
+ DFSConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS_DEFAULT,
+ ActiveNamenodeResolver.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d710513a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
new file mode 100644
index 0000000..da6066b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+/**
+ * Interface for objects that are unique to a namespace.
+ */
+public interface RemoteLocationContext {
+
+ /**
+ * Returns an identifier for a unique namespace.
+ *
+ * @return Namespace identifier.
+ */
+ String getNameserviceId();
+
+ /**
+ * Destination in this location. For example the path in a remote namespace.
+ *
+ * @return Destination in this location.
+ */
+ String getDest();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d710513a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
new file mode 100644
index 0000000..fe0d02a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
@@ -0,0 +1,263 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.apache.hadoop.hdfs.server.federation.router.FederationUtil.newActiveNamenodeResolver;
+import static org.apache.hadoop.hdfs.server.federation.router.FederationUtil.newFileSubclusterResolver;
+import static org.apache.hadoop.util.ExitUtil.terminate;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.util.ShutdownHookManager;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * Router that provides a unified view of multiple federated HDFS clusters. It
+ * has two main roles: (1) federated interface and (2) NameNode heartbeat.
+ * <p>
+ * For the federated interface, the Router receives a client request, checks the
+ * State Store for the correct subcluster, and forwards the request to the
+ * active Namenode of that subcluster. The reply from the Namenode then flows in
+ * the opposite direction. The Routers are stateless and can be behind a load
+ * balancer. HDFS clients connect to the router using the same interfaces as are
+ * used to communicate with a namenode, namely the ClientProtocol RPC interface
+ * and the WebHdfs HTTP interface exposed by the router. {@link RouterRpcServer}
+ * {@link RouterHttpServer}
+ * <p>
+ * For NameNode heartbeat, the Router periodically checks the state of a
+ * NameNode (usually on the same server) and reports their high availability
+ * (HA) state and load/space status to the State Store. Note that this is an
+ * optional role as a Router can be independent of any subcluster.
+ * {@link StateStoreService} {@link NamenodeHeartbeatService}
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class Router extends CompositeService {
+
+ private static final Log LOG = LogFactory.getLog(Router.class);
+
+
+ /** Configuration for the Router. */
+ private Configuration conf;
+
+ /** Router address/identifier. */
+ private String routerId;
+
+ /** RPC interface to the client. */
+ private RouterRpcServer rpcServer;
+
+ /** Interface with the State Store. */
+ private StateStoreService stateStore;
+
+ /** Interface to map global name space to HDFS subcluster name spaces. */
+ private FileSubclusterResolver subclusterResolver;
+
+ /** Interface to identify the active NN for a nameservice or blockpool ID. */
+ private ActiveNamenodeResolver namenodeResolver;
+
+
+ /** Usage string for help message. */
+ private static final String USAGE = "Usage: java Router";
+
+ /** Priority of the Router shutdown hook. */
+ public static final int SHUTDOWN_HOOK_PRIORITY = 30;
+
+
+ /////////////////////////////////////////////////////////
+ // Constructor
+ /////////////////////////////////////////////////////////
+
+ public Router() {
+ super(Router.class.getName());
+ }
+
+ /////////////////////////////////////////////////////////
+ // Service management
+ /////////////////////////////////////////////////////////
+
+ @Override
+ protected void serviceInit(Configuration configuration) throws Exception {
+ this.conf = configuration;
+
+ // TODO Interface to the State Store
+ this.stateStore = null;
+
+ // Resolver to track active NNs
+ this.namenodeResolver = newActiveNamenodeResolver(
+ this.conf, this.stateStore);
+ if (this.namenodeResolver == null) {
+ throw new IOException("Cannot find namenode resolver.");
+ }
+
+ // Lookup interface to map between the global and subcluster name spaces
+ this.subclusterResolver = newFileSubclusterResolver(
+ this.conf, this.stateStore);
+ if (this.subclusterResolver == null) {
+ throw new IOException("Cannot find subcluster resolver");
+ }
+
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+
+ super.serviceStop();
+ }
+
+ /**
+ * Shutdown the router.
+ */
+ public void shutDown() {
+ new Thread() {
+ @Override
+ public void run() {
+ Router.this.stop();
+ }
+ }.start();
+ }
+
+ /**
+ * Main run loop for the router.
+ *
+ * @param argv parameters.
+ */
+ public static void main(String[] argv) {
+ if (DFSUtil.parseHelpArgument(argv, Router.USAGE, System.out, true)) {
+ System.exit(0);
+ }
+
+ try {
+ StringUtils.startupShutdownMessage(Router.class, argv, LOG);
+
+ Router router = new Router();
+
+ ShutdownHookManager.get().addShutdownHook(
+ new CompositeServiceShutdownHook(router), SHUTDOWN_HOOK_PRIORITY);
+
+ Configuration conf = new HdfsConfiguration();
+ router.init(conf);
+ router.start();
+ } catch (Throwable e) {
+ LOG.error("Failed to start router.", e);
+ terminate(1, e);
+ }
+ }
+
+ /////////////////////////////////////////////////////////
+ // RPC Server
+ /////////////////////////////////////////////////////////
+
+ /**
+ * Create a new Router RPC server to proxy ClientProtocol requests.
+ *
+ * @return RouterRpcServer
+ * @throws IOException If the router RPC server was not started.
+ */
+ protected RouterRpcServer createRpcServer() throws IOException {
+ return new RouterRpcServer(this.conf, this, this.getNamenodeResolver(),
+ this.getSubclusterResolver());
+ }
+
+ /**
+ * Get the Router RPC server.
+ *
+ * @return Router RPC server.
+ */
+ public RouterRpcServer getRpcServer() {
+ return this.rpcServer;
+ }
+
+ /////////////////////////////////////////////////////////
+ // Submodule getters
+ /////////////////////////////////////////////////////////
+
+ /**
+ * Get the State Store service.
+ *
+ * @return State Store service.
+ */
+ public StateStoreService getStateStore() {
+ return this.stateStore;
+ }
+
+ /**
+ * Get the subcluster resolver for files.
+ *
+ * @return Subcluster resolver for files.
+ */
+ public FileSubclusterResolver getSubclusterResolver() {
+ return this.subclusterResolver;
+ }
+
+ /**
+ * Get the namenode resolver for a subcluster.
+ *
+ * @return The namenode resolver for a subcluster.
+ */
+ public ActiveNamenodeResolver getNamenodeResolver() {
+ return this.namenodeResolver;
+ }
+
+ /////////////////////////////////////////////////////////
+ // Router info
+ /////////////////////////////////////////////////////////
+
+ /**
+ * Unique ID for the router, typically the hostname:port string for the
+ * router's RPC server. This ID may be null on router startup before the RPC
+ * server has bound to a port.
+ *
+ * @return Router identifier.
+ */
+ public String getRouterId() {
+ return this.routerId;
+ }
+
+ /**
+ * Sets a unique ID for this router.
+ *
+ * @param router Identifier of the Router.
+ */
+ public void setRouterId(String id) {
+ this.routerId = id;
+ if (this.stateStore != null) {
+ this.stateStore.setIdentifier(this.routerId);
+ }
+ if (this.namenodeResolver != null) {
+ this.namenodeResolver.setRouterId(this.routerId);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d710513a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
new file mode 100644
index 0000000..24792bb
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.ipc.RPC.Server;
+import org.apache.hadoop.service.AbstractService;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * This class is responsible for handling all of the RPC calls to the It is
+ * created, started, and stopped by {@link Router}. It implements the
+ * {@link ClientProtocol} to mimic a
+ * {@link org.apache.hadoop.hdfs.server.namenode.NameNode NameNode} and proxies
+ * the requests to the active
+ * {@link org.apache.hadoop.hdfs.server.namenode.NameNode NameNode}.
+ */
+public class RouterRpcServer extends AbstractService {
+
+ /** The RPC server that listens to requests from clients. */
+ private final Server rpcServer;
+
+ /**
+ * Construct a router RPC server.
+ *
+ * @param configuration HDFS Configuration.
+ * @param nnResolver The NN resolver instance to determine active NNs in HA.
+ * @param fileResolver File resolver to resolve file paths to subclusters.
+ * @throws IOException If the RPC server could not be created.
+ */
+ public RouterRpcServer(Configuration configuration, Router router,
+ ActiveNamenodeResolver nnResolver, FileSubclusterResolver fileResolver)
+ throws IOException {
+ super(RouterRpcServer.class.getName());
+
+ this.rpcServer = null;
+ }
+
+ /**
+ * Allow access to the client RPC server for testing.
+ *
+ * @return The RPC server.
+ */
+ @VisibleForTesting
+ public Server getServer() {
+ return this.rpcServer;
+ }
+
+ @Override
+ protected void serviceInit(Configuration configuration) throws Exception {
+ super.serviceInit(configuration);
+ }
+
+ /**
+ * Start client and service RPC servers.
+ */
+ @Override
+ protected void serviceStart() throws Exception {
+ if (this.rpcServer != null) {
+ this.rpcServer.start();
+ }
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (this.rpcServer != null) {
+ this.rpcServer.stop();
+ }
+ super.serviceStop();
+ }
+
+ /**
+ * Wait until the RPC servers have shutdown.
+ */
+ void join() throws InterruptedException {
+ if (this.rpcServer != null) {
+ this.rpcServer.join();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d710513a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/package-info.java
new file mode 100644
index 0000000..327f39b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/package-info.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The router package includes the core services for a HDFS federation router.
+ * The {@link Router} acts as a transparent proxy in front of a cluster of
+ * multiple NameNodes and nameservices. The {@link RouterRpcServer} exposes the
+ * NameNode clientProtocol and is the primary contact point for DFS clients in a
+ * federated cluster.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d710513a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
new file mode 100644
index 0000000..866daa3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.service.CompositeService;
+
+/**
+ * A service to initialize a
+ * {@link org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver
+ * StateStoreDriver} and maintain the connection to the data store. There
+ * are multiple state store driver connections supported:
+ * <ul>
+ * <li>File {@link org.apache.hadoop.hdfs.server.federation.store.driver.impl.
+ * StateStoreFileImpl StateStoreFileImpl}
+ * <li>ZooKeeper {@link org.apache.hadoop.hdfs.server.federation.store.driver.
+ * impl.StateStoreZooKeeperImpl StateStoreZooKeeperImpl}
+ * </ul>
+ * <p>
+ * The service also supports the dynamic registration of data interfaces such as
+ * the following:
+ * <ul>
+ * <li>{@link MembershipStateStore}: state of the Namenodes in the
+ * federation.
+ * <li>{@link MountTableStore}: Mount table between to subclusters.
+ * See {@link org.apache.hadoop.fs.viewfs.ViewFs ViewFs}.
+ * <li>{@link RouterStateStore}: State of the routers in the federation.
+ * </ul>
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class StateStoreService extends CompositeService {
+
+ /** Identifier for the service. */
+ private String identifier;
+
+ // Stub class
+ public StateStoreService(String name) {
+ super(name);
+ }
+
+ /**
+ * Fetch a unique identifier for this state store instance. Typically it is
+ * the address of the router.
+ *
+ * @return Unique identifier for this store.
+ */
+ public String getIdentifier() {
+ return this.identifier;
+ }
+
+ /**
+ * Set a unique synchronization identifier for this store.
+ *
+ * @param id Unique identifier, typically the router's RPC address.
+ */
+ public void setIdentifier(String id) {
+ this.identifier = id;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d710513a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/package-info.java
new file mode 100644
index 0000000..949ec7c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/package-info.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The federation state store tracks persistent values that are shared between
+ * multiple routers.
+ * <p>
+ * Data is stored in data records that inherit from a common class. Data records
+ * are serialized when written to the data store using a modular serialization
+ * implementation. The default is profobuf serialization. Data is stored as rows
+ * of records of the same type with each data member in a record representing a
+ * column.
+ * <p>
+ * The state store uses a modular data storage
+ * {@link org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver
+ * StateStoreDriver} to handle querying, updating and deleting data records. The
+ * data storage driver is initialized and maintained by the
+ * {@link org.apache.hadoop.hdfs.server.federation.store.
+ * StateStoreService FederationStateStoreService}. The state store
+ * supports fetching all records of a type, filtering by column values or
+ * fetching a single record by its primary key.
+ * <p>
+ * The state store contains several API interfaces, one for each data records
+ * type.
+ * <p>
+ * <ul>
+ * <li>FederationMembershipStateStore: state of all Namenodes in the federation.
+ * Uses the MembershipState record.
+ * <li>FederationMountTableStore: Mount table mapping paths in the global
+ * namespace to individual subcluster paths. Uses the MountTable record.
+ * <li>RouterStateStore: State of all routers in the federation. Uses the
+ * RouterState record.
+ * </ul>
+ * <p>
+ * Each API is defined in a separate interface. The implementations of these
+ * interfaces are responsible for accessing the
+ * {@link org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver
+ * StateStoreDriver} to query, update and delete data records.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d710513a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index f0f2220..82f8516 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4513,4 +4513,20 @@
</description>
</property>
+ <property>
+ <name>dfs.federation.router.file.resolver.client.class</name>
+ <value>org.apache.hadoop.hdfs.server.federation.MockResolver</value>
+ <description>
+ Class to resolve files to subclusters.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.namenode.resolver.client.class</name>
+ <value>org.apache.hadoop.hdfs.server.federation.MockResolver</value>
+ <description>
+ Class to resolve the namenode for a subcluster.
+ </description>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d710513a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
new file mode 100644
index 0000000..8674682
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
@@ -0,0 +1,233 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.BufferedReader;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.Date;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
+import org.apache.hadoop.security.AccessControlException;
+
+/**
+ * Helper utilities for testing HDFS Federation.
+ */
+public final class FederationTestUtils {
+
+ public final static String NAMESERVICE1 = "ns0";
+ public final static String NAMESERVICE2 = "ns1";
+ public final static String NAMENODE1 = "nn0";
+ public final static String NAMENODE2 = "nn1";
+ public final static String NAMENODE3 = "nn2";
+ public final static String NAMENODE4 = "nn3";
+ public final static String ROUTER1 = "router0";
+ public final static String ROUTER2 = "router1";
+ public final static String ROUTER3 = "router2";
+ public final static String ROUTER4 = "router3";
+ public final static long BLOCK_SIZE_BYTES = 134217728;
+
+ private FederationTestUtils() {
+ // Utility class
+ }
+
+ public static void verifyException(Object obj, String methodName,
+ Class<? extends Exception> exceptionClass, Class<?>[] parameterTypes,
+ Object[] arguments) {
+
+ Throwable triggeredException = null;
+ try {
+ Method m = obj.getClass().getMethod(methodName, parameterTypes);
+ m.invoke(obj, arguments);
+ } catch (InvocationTargetException ex) {
+ triggeredException = ex.getTargetException();
+ } catch (Exception e) {
+ triggeredException = e;
+ }
+ if (exceptionClass != null) {
+ assertNotNull("No exception was triggered, expected exception - "
+ + exceptionClass.getName(), triggeredException);
+ assertEquals(exceptionClass, triggeredException.getClass());
+ } else {
+ assertNull("Exception was triggered but no exception was expected",
+ triggeredException);
+ }
+ }
+
+ public static NamenodeStatusReport createNamenodeReport(String ns, String nn,
+ HAServiceState state) {
+ Random rand = new Random();
+ NamenodeStatusReport report = new NamenodeStatusReport(ns, nn,
+ "localhost:" + rand.nextInt(10000), "localhost:" + rand.nextInt(10000),
+ "localhost:" + rand.nextInt(10000), "testwebaddress-" + ns + nn);
+ if (state == null) {
+ // Unavailable, no additional info
+ return report;
+ }
+ report.setHAServiceState(state);
+ report.setNamespaceInfo(new NamespaceInfo(1, "tesclusterid", ns, 0,
+ "testbuildvesion", "testsoftwareversion"));
+ return report;
+ }
+
+ public static void waitNamenodeRegistered(ActiveNamenodeResolver resolver,
+ String nameserviceId, String namenodeId,
+ FederationNamenodeServiceState finalState)
+ throws InterruptedException, IllegalStateException, IOException {
+
+ for (int loopCount = 0; loopCount < 20; loopCount++) {
+
+ if (loopCount > 0) {
+ Thread.sleep(1000);
+ }
+
+ List<? extends FederationNamenodeContext> namenodes;
+ namenodes =
+ resolver.getNamenodesForNameserviceId(nameserviceId);
+ for (FederationNamenodeContext namenode : namenodes) {
+ if (namenodeId != null
+ && !namenode.getNamenodeId().equals(namenodeId)) {
+ // Keep looking
+ continue;
+ }
+ if (finalState != null && !namenode.getState().equals(finalState)) {
+ // Wrong state, wait a bit more
+ break;
+ }
+ // Found
+ return;
+ }
+ }
+ assertTrue("Failed to verify state store registration for state - "
+ + finalState + " - " + " - " + nameserviceId + " - ", false);
+ }
+
+ public static boolean verifyDate(Date d1, Date d2, long precision) {
+ if (Math.abs(d1.getTime() - d2.getTime()) < precision) {
+ return true;
+ }
+ return false;
+ }
+
+ public static boolean addDirectory(FileSystem context, String path)
+ throws IOException {
+ context.mkdirs(new Path(path), new FsPermission("777"));
+ return verifyFileExists(context, path);
+ }
+
+ public static FileStatus getFileStatus(FileSystem context, String path)
+ throws IOException {
+ return context.getFileStatus(new Path(path));
+ }
+
+ public static boolean verifyFileExists(FileSystem context, String path) {
+ try {
+ FileStatus status = getFileStatus(context, path);
+ if (status != null) {
+ return true;
+ }
+ } catch (Exception e) {
+ return false;
+ }
+
+ return false;
+ }
+
+ public static boolean checkForFileInDirectory(FileSystem context,
+ String testPath, String targetFile) throws AccessControlException,
+ FileNotFoundException,
+ UnsupportedFileSystemException, IllegalArgumentException,
+ IOException {
+ FileStatus[] fileStatus = context.listStatus(new Path(testPath));
+ String file = null;
+ String verifyPath = testPath + "/" + targetFile;
+ if (testPath.equals("/")) {
+ verifyPath = testPath + targetFile;
+ }
+
+ Boolean found = false;
+ for (int i = 0; i < fileStatus.length; i++) {
+ FileStatus f = fileStatus[i];
+ file = Path.getPathWithoutSchemeAndAuthority(f.getPath()).toString();
+ if (file.equals(verifyPath)) {
+ found = true;
+ }
+ }
+ return found;
+ }
+
+ public static int countContents(FileSystem context, String testPath)
+ throws IOException {
+ FileStatus[] fileStatus = context.listStatus(new Path(testPath));
+ return fileStatus.length;
+ }
+
+ public static void createFile(FileSystem fs, String path, long length)
+ throws IOException {
+ FsPermission permissions = new FsPermission("700");
+ FSDataOutputStream writeStream = fs.create(new Path(path), permissions,
+ true, 1000, (short) 1, BLOCK_SIZE_BYTES, null);
+ for (int i = 0; i < length; i++) {
+ writeStream.write(i);
+ }
+ writeStream.close();
+ }
+
+ public static String readFile(FileSystem fs, String path) throws IOException {
+ // Read the file from the filesystem via the active namenode
+ Path fileName = new Path(path);
+ InputStreamReader reader = new InputStreamReader(fs.open(fileName));
+ BufferedReader bufferedReader = new BufferedReader(reader);
+ StringBuilder data = new StringBuilder();
+ String line;
+
+ while ((line = bufferedReader.readLine()) != null) {
+ data.append(line);
+ }
+
+ bufferedReader.close();
+ reader.close();
+ return data.toString();
+ }
+
+ public static boolean deleteFile(FileSystem fs, String path)
+ throws IOException {
+ return fs.delete(new Path(path), true);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: HDFS-10882. Federation State Store
Interface API. Contributed by Jason Kace and Inigo Goiri.
Posted by in...@apache.org.
HDFS-10882. Federation State Store Interface API. Contributed by Jason Kace and Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/515ab554
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/515ab554
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/515ab554
Branch: refs/heads/HDFS-10467
Commit: 515ab554df2c971bd1522f86af7fe1ce80e4e9b3
Parents: 15976e0
Author: Inigo <in...@apache.org>
Authored: Thu Apr 6 19:18:52 2017 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu May 11 09:54:08 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 11 ++
.../server/federation/store/RecordStore.java | 100 ++++++++++++++++
.../store/driver/StateStoreSerializer.java | 119 +++++++++++++++++++
.../driver/impl/StateStoreSerializerPBImpl.java | 115 ++++++++++++++++++
.../store/records/impl/pb/PBRecord.java | 47 ++++++++
.../store/records/impl/pb/package-info.java | 29 +++++
.../src/main/resources/hdfs-default.xml | 8 ++
7 files changed, 429 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/515ab554/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 0e04c63..f55be20 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRackFaultTolerant;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.RamDiskReplicaLruTracker;
+import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreSerializerPBImpl;
import org.apache.hadoop.http.HttpConfig;
/**
@@ -1106,6 +1107,16 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS_DEFAULT =
"org.apache.hadoop.hdfs.server.federation.MockResolver";
+ // HDFS Router-based federation State Store
+ public static final String FEDERATION_STORE_PREFIX =
+ FEDERATION_ROUTER_PREFIX + "store.";
+
+ public static final String FEDERATION_STORE_SERIALIZER_CLASS =
+ DFSConfigKeys.FEDERATION_STORE_PREFIX + "serializer";
+ public static final Class<StateStoreSerializerPBImpl>
+ FEDERATION_STORE_SERIALIZER_CLASS_DEFAULT =
+ StateStoreSerializerPBImpl.class;
+
// dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry
@Deprecated
public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY
http://git-wip-us.apache.org/repos/asf/hadoop/blob/515ab554/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RecordStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RecordStore.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RecordStore.java
new file mode 100644
index 0000000..524f432
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RecordStore.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import java.lang.reflect.Constructor;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+
+/**
+ * Store records in the State Store. Subclasses provide interfaces to operate on
+ * those records.
+ *
+ * @param <R> Record to store by this interface.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public abstract class RecordStore<R extends BaseRecord> {
+
+ private static final Log LOG = LogFactory.getLog(RecordStore.class);
+
+
+ /** Class of the record stored in this State Store. */
+ private final Class<R> recordClass;
+
+ /** State store driver backed by persistent storage. */
+ private final StateStoreDriver driver;
+
+
+ /**
+ * Create a new store for records.
+ *
+ * @param clazz Class of the record to store.
+ * @param stateStoreDriver Driver for the State Store.
+ */
+ protected RecordStore(Class<R> clazz, StateStoreDriver stateStoreDriver) {
+ this.recordClass = clazz;
+ this.driver = stateStoreDriver;
+ }
+
+ /**
+ * Report a required record to the data store. The data store uses this to
+ * create/maintain storage for the record.
+ *
+ * @return The class of the required record or null if no record is required
+ * for this interface.
+ */
+ public Class<R> getRecordClass() {
+ return this.recordClass;
+ }
+
+ /**
+ * Get the State Store driver.
+ *
+ * @return State Store driver.
+ */
+ protected StateStoreDriver getDriver() {
+ return this.driver;
+ }
+
+ /**
+ * Build a state store API implementation interface.
+ *
+ * @param interfaceClass The specific interface implementation to create
+ * @param driver The {@link StateStoreDriver} implementation in use.
+ * @return An initialized instance of the specified state store API
+ * implementation.
+ */
+ public static <T extends RecordStore<?>> T newInstance(
+ final Class<T> clazz, final StateStoreDriver driver) {
+
+ try {
+ Constructor<T> constructor = clazz.getConstructor(StateStoreDriver.class);
+ T recordStore = constructor.newInstance(driver);
+ return recordStore;
+ } catch (Exception e) {
+ LOG.error("Cannot create new instance for " + clazz, e);
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/515ab554/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreSerializer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreSerializer.java
new file mode 100644
index 0000000..8540405
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreSerializer.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.util.ReflectionUtils;
+
+/**
+ * Serializer to store and retrieve data in the State Store.
+ */
+public abstract class StateStoreSerializer {
+
+ /** Singleton for the serializer instance. */
+ private static StateStoreSerializer defaultSerializer;
+
+ /**
+ * Get the default serializer based.
+ * @return Singleton serializer.
+ */
+ public static StateStoreSerializer getSerializer() {
+ return getSerializer(null);
+ }
+
+ /**
+ * Get a serializer based on the provided configuration.
+ * @param conf Configuration. Default if null.
+ * @return Singleton serializer.
+ */
+ public static StateStoreSerializer getSerializer(Configuration conf) {
+ if (conf == null) {
+ synchronized (StateStoreSerializer.class) {
+ if (defaultSerializer == null) {
+ conf = new Configuration();
+ defaultSerializer = newSerializer(conf);
+ }
+ }
+ return defaultSerializer;
+ } else {
+ return newSerializer(conf);
+ }
+ }
+
+ private static StateStoreSerializer newSerializer(final Configuration conf) {
+ Class<? extends StateStoreSerializer> serializerName = conf.getClass(
+ DFSConfigKeys.FEDERATION_STORE_SERIALIZER_CLASS,
+ DFSConfigKeys.FEDERATION_STORE_SERIALIZER_CLASS_DEFAULT,
+ StateStoreSerializer.class);
+ return ReflectionUtils.newInstance(serializerName, conf);
+ }
+
+ /**
+ * Create a new record.
+ * @param clazz Class of the new record.
+ * @return New record.
+ */
+ public static <T> T newRecord(Class<T> clazz) {
+ return getSerializer(null).newRecordInstance(clazz);
+ }
+
+ /**
+ * Create a new record.
+ * @param clazz Class of the new record.
+ * @return New record.
+ */
+ public abstract <T> T newRecordInstance(Class<T> clazz);
+
+ /**
+ * Serialize a record into a byte array.
+ * @param record Record to serialize.
+ * @return Byte array with the serialized record.
+ */
+ public abstract byte[] serialize(BaseRecord record);
+
+ /**
+ * Serialize a record into a string.
+ * @param record Record to serialize.
+ * @return String with the serialized record.
+ */
+ public abstract String serializeString(BaseRecord record);
+
+ /**
+ * Deserialize a bytes array into a record.
+ * @param byteArray Byte array to deserialize.
+ * @param clazz Class of the record.
+ * @return New record.
+ * @throws IOException If it cannot deserialize the record.
+ */
+ public abstract <T extends BaseRecord> T deserialize(
+ byte[] byteArray, Class<T> clazz) throws IOException;
+
+ /**
+ * Deserialize a string into a record.
+ * @param data String with the data to deserialize.
+ * @param clazz Class of the record.
+ * @return New record.
+ * @throws IOException If it cannot deserialize the record.
+ */
+ public abstract <T extends BaseRecord> T deserialize(
+ String data, Class<T> clazz) throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/515ab554/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializerPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializerPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializerPBImpl.java
new file mode 100644
index 0000000..45c5dd6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializerPBImpl.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
+
+import java.io.IOException;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.codec.binary.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+import org.apache.hadoop.util.ReflectionUtils;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the State Store serializer.
+ */
+public final class StateStoreSerializerPBImpl extends StateStoreSerializer {
+
+ private static final String PB_IMPL_PACKAGE_SUFFIX = "impl.pb";
+ private static final String PB_IMPL_CLASS_SUFFIX = "PBImpl";
+
+ private Configuration localConf = new Configuration();
+
+
+ private StateStoreSerializerPBImpl() {
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <T> T newRecordInstance(Class<T> clazz) {
+ try {
+ String clazzPBImpl = getPBImplClassName(clazz);
+ Class<?> pbClazz = localConf.getClassByName(clazzPBImpl);
+ Object retObject = ReflectionUtils.newInstance(pbClazz, localConf);
+ return (T)retObject;
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private String getPBImplClassName(Class<?> clazz) {
+ String srcPackagePart = getPackageName(clazz);
+ String srcClassName = getClassName(clazz);
+ String destPackagePart = srcPackagePart + "." + PB_IMPL_PACKAGE_SUFFIX;
+ String destClassPart = srcClassName + PB_IMPL_CLASS_SUFFIX;
+ return destPackagePart + "." + destClassPart;
+ }
+
+ private String getClassName(Class<?> clazz) {
+ String fqName = clazz.getName();
+ return (fqName.substring(fqName.lastIndexOf(".") + 1, fqName.length()));
+ }
+
+ private String getPackageName(Class<?> clazz) {
+ return clazz.getPackage().getName();
+ }
+
+ @Override
+ public byte[] serialize(BaseRecord record) {
+ byte[] byteArray64 = null;
+ if (record instanceof PBRecord) {
+ PBRecord recordPB = (PBRecord) record;
+ Message msg = recordPB.getProto();
+ byte[] byteArray = msg.toByteArray();
+ byteArray64 = Base64.encodeBase64(byteArray, false);
+ }
+ return byteArray64;
+ }
+
+ @Override
+ public String serializeString(BaseRecord record) {
+ byte[] byteArray64 = serialize(record);
+ String base64Encoded = StringUtils.newStringUtf8(byteArray64);
+ return base64Encoded;
+ }
+
+ @Override
+ public <T extends BaseRecord> T deserialize(
+ byte[] byteArray, Class<T> clazz) throws IOException {
+
+ T record = newRecord(clazz);
+ if (record instanceof PBRecord) {
+ PBRecord pbRecord = (PBRecord) record;
+ byte[] byteArray64 = Base64.encodeBase64(byteArray, false);
+ String base64Encoded = StringUtils.newStringUtf8(byteArray64);
+ pbRecord.readInstance(base64Encoded);
+ }
+ return record;
+ }
+
+ @Override
+ public <T extends BaseRecord> T deserialize(String data, Class<T> clazz)
+ throws IOException {
+ byte[] byteArray64 = Base64.decodeBase64(data);
+ return deserialize(byteArray64, clazz);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/515ab554/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/PBRecord.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/PBRecord.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/PBRecord.java
new file mode 100644
index 0000000..c369275
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/PBRecord.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records.impl.pb;
+
+import java.io.IOException;
+
+import com.google.protobuf.Message;
+
+/**
+ * A record implementation using Protobuf.
+ */
+public interface PBRecord {
+
+ /**
+ * Get the protocol for the record.
+ * @return The protocol for this record.
+ */
+ Message getProto();
+
+ /**
+ * Set the protocol for the record.
+ * @param proto Protocol for this record.
+ */
+ void setProto(Message proto);
+
+ /**
+ * Populate this record with serialized data.
+ * @param base64String Serialized data in base64.
+ * @throws IOException If it cannot read the data.
+ */
+ void readInstance(String base64String) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/515ab554/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/package-info.java
new file mode 100644
index 0000000..b329732
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/package-info.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The protobuf implementations of state store data records defined in the
+ * org.apache.hadoop.hdfs.server.federation.store.records package. Each
+ * implementation wraps an associated protobuf proto definition.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+package org.apache.hadoop.hdfs.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/515ab554/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 82f8516..44db981 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4529,4 +4529,12 @@
</description>
</property>
+ <property>
+ <name>dfs.federation.router.store.serializer</name>
+ <value>org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreSerializerPBImpl</value>
+ <description>
+ Class to serialize State Store records.
+ </description>
+ </property>
+
</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: HADOOP-14386. Rewind trunk from
Guava 21.0 back to Guava 11.0.2.
Posted by in...@apache.org.
HADOOP-14386. Rewind trunk from Guava 21.0 back to Guava 11.0.2.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/543aac9f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/543aac9f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/543aac9f
Branch: refs/heads/HDFS-10467
Commit: 543aac9f281871a40473e83061f6deadc0bbdab7
Parents: c60164f
Author: Andrew Wang <wa...@apache.org>
Authored: Tue May 9 09:22:53 2017 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Tue May 9 09:22:53 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/fs/shell/XAttrCommands.java | 7 +-
.../hdfs/server/datanode/DataStorage.java | 10 +-
.../hdfs/server/namenode/AclTransformation.java | 7 +-
.../hadoop/hdfs/server/namenode/JournalSet.java | 33 ++--
.../qjournal/client/DirectExecutorService.java | 154 +++++++++++++++++++
.../hdfs/qjournal/client/TestQJMWithFaults.java | 3 +-
.../client/TestQuorumJournalManager.java | 3 +-
hadoop-project/pom.xml | 2 +-
.../pb/ApplicationSubmissionContextPBImpl.java | 3 +-
.../org/apache/hadoop/yarn/webapp/WebApp.java | 6 +-
.../server/resourcemanager/RMAppManager.java | 20 ++-
.../fair/AllocationFileLoaderService.java | 3 +-
.../scheduler/fair/FairSchedulerUtilities.java | 69 +++++++++
.../scheduler/fair/QueueManager.java | 6 +-
.../fair/TestFairSchedulerUtilities.java | 67 ++++++++
15 files changed, 341 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/543aac9f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/XAttrCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/XAttrCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/XAttrCommands.java
index 4505aa9..6301776 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/XAttrCommands.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/XAttrCommands.java
@@ -20,12 +20,9 @@ package org.apache.hadoop.fs.shell;
import java.io.IOException;
import java.util.Iterator;
import java.util.LinkedList;
-import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
-import com.google.common.base.Enums;
-import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -66,8 +63,6 @@ class XAttrCommands extends FsCommand {
" and values encoded as hexadecimal and base64 are prefixed with " +
"0x and 0s, respectively.\n" +
"<path>: The file or directory.\n";
- private final static Function<String, XAttrCodec> enValueOfFunc =
- Enums.stringConverter(XAttrCodec.class);
private String name = null;
private boolean dump = false;
@@ -79,7 +74,7 @@ class XAttrCommands extends FsCommand {
String en = StringUtils.popOptionWithArgument("-e", args);
if (en != null) {
try {
- encoding = enValueOfFunc.apply(StringUtils.toUpperCase(en));
+ encoding = XAttrCodec.valueOf(StringUtils.toUpperCase(en));
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException(
"Invalid/unsupported encoding option specified: " + en);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/543aac9f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
index 835643b..9a71081 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
@@ -65,7 +65,6 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ComparisonChain;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import com.google.common.util.concurrent.Futures;
/**
* Data storage information file.
@@ -1109,7 +1108,14 @@ public class DataStorage extends Storage {
}
linkWorkers.shutdown();
for (Future<Void> f : futures) {
- Futures.getChecked(f, IOException.class);
+ try {
+ f.get();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException(e);
+ } catch (ExecutionException e) {
+ throw new IOException(e);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/543aac9f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java
index 3e4a319..4402e26 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java
@@ -28,7 +28,6 @@ import java.util.EnumSet;
import java.util.Iterator;
import java.util.List;
-import com.google.common.base.MoreObjects;
import com.google.common.collect.ComparisonChain;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -366,8 +365,10 @@ final class AclTransformation {
for (AclEntry entry: aclBuilder) {
scopeFound.add(entry.getScope());
if (entry.getType() == GROUP || entry.getName() != null) {
- FsAction scopeUnionPerms = MoreObjects.firstNonNull(
- unionPerms.get(entry.getScope()), FsAction.NONE);
+ FsAction scopeUnionPerms = unionPerms.get(entry.getScope());
+ if (scopeUnionPerms == null) {
+ scopeUnionPerms = FsAction.NONE;
+ }
unionPerms.put(entry.getScope(),
scopeUnionPerms.or(entry.getPermission()));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/543aac9f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java
index db77d31..e7f2adb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java
@@ -40,7 +40,6 @@ import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
import com.google.common.base.Preconditions;
-import com.google.common.collect.ComparisonChain;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableListMultimap;
import com.google.common.collect.Lists;
@@ -57,28 +56,18 @@ public class JournalSet implements JournalManager {
static final Log LOG = LogFactory.getLog(FSEditLog.class);
+ // we want local logs to be ordered earlier in the collection, and true
+ // is considered larger than false, so reverse the comparator
private static final Comparator<EditLogInputStream>
- LOCAL_LOG_PREFERENCE_COMPARATOR = new Comparator<EditLogInputStream>() {
- @Override
- public int compare(EditLogInputStream elis1, EditLogInputStream elis2) {
- // we want local logs to be ordered earlier in the collection, and true
- // is considered larger than false, so we want to invert the booleans here
- return ComparisonChain.start().compareFalseFirst(!elis1.isLocalLog(),
- !elis2.isLocalLog()).result();
- }
- };
-
- static final public Comparator<EditLogInputStream>
- EDIT_LOG_INPUT_STREAM_COMPARATOR = new Comparator<EditLogInputStream>() {
- @Override
- public int compare(EditLogInputStream a, EditLogInputStream b) {
- return ComparisonChain.start().
- compare(a.getFirstTxId(), b.getFirstTxId()).
- compare(b.getLastTxId(), a.getLastTxId()).
- result();
- }
- };
-
+ LOCAL_LOG_PREFERENCE_COMPARATOR = Comparator
+ .comparing(EditLogInputStream::isLocalLog)
+ .reversed();
+
+ public static final Comparator<EditLogInputStream>
+ EDIT_LOG_INPUT_STREAM_COMPARATOR = Comparator
+ .comparing(EditLogInputStream::getFirstTxId)
+ .thenComparing(EditLogInputStream::getLastTxId);
+
/**
* Container for a JournalManager paired with its currently
* active stream.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/543aac9f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/DirectExecutorService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/DirectExecutorService.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/DirectExecutorService.java
new file mode 100644
index 0000000..15d2a13
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/DirectExecutorService.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.qjournal.client;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * A very basic ExecutorService for running submitted Callables serially.
+ * Many bits of functionality are not implemented.
+ */
+public class DirectExecutorService implements ExecutorService {
+
+ private static class DirectFuture<V> implements Future<V> {
+ private V result = null;
+ private Exception ex = null;
+
+ DirectFuture(Callable<V> c) {
+ try {
+ result = c.call();
+ } catch (Exception e) {
+ ex = e;
+ }
+ }
+
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning) {
+ return false;
+ }
+
+ @Override
+ public boolean isCancelled() {
+ return false;
+ }
+
+ @Override
+ public boolean isDone() {
+ return true;
+ }
+
+ @Override
+ public V get() throws InterruptedException, ExecutionException {
+ if (ex != null) {
+ throw new ExecutionException(ex);
+ }
+ return result;
+ }
+
+ @Override
+ public V get(long timeout, TimeUnit unit)
+ throws InterruptedException, ExecutionException, TimeoutException {
+ return get();
+ }
+ }
+
+ private boolean isShutdown = false;
+
+ @Override
+ synchronized public void shutdown() {
+ isShutdown = true;
+ }
+
+ @Override
+ public List<Runnable> shutdownNow() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isShutdown() {
+ return isShutdown;
+ }
+
+ @Override
+ synchronized public boolean isTerminated() {
+ return isShutdown;
+ }
+
+ @Override
+ public boolean awaitTermination(long timeout, TimeUnit unit)
+ throws InterruptedException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ synchronized public <T> Future<T> submit(Callable<T> task) {
+ if (isShutdown) {
+ throw new RejectedExecutionException("ExecutorService was shutdown");
+ }
+ return new DirectFuture<>(task);
+ }
+
+ @Override
+ public <T> Future<T> submit(Runnable task, T result) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Future<?> submit(Runnable task) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public <T> List<Future<T>> invokeAll(Collection<? extends Callable<T>> tasks)
+ throws InterruptedException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public <T> List<Future<T>> invokeAll(Collection<? extends Callable<T>> tasks,
+ long timeout, TimeUnit unit) throws InterruptedException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public <T> T invokeAny(Collection<? extends Callable<T>> tasks)
+ throws InterruptedException, ExecutionException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public <T> T invokeAny(Collection<? extends Callable<T>> tasks, long timeout,
+ TimeUnit unit)
+ throws InterruptedException, ExecutionException, TimeoutException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ synchronized public void execute(Runnable command) {
+ command.run();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/543aac9f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java
index c752f23..9ada40f6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java
@@ -65,7 +65,6 @@ import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-import com.google.common.util.concurrent.MoreExecutors;
public class TestQJMWithFaults {
@@ -402,7 +401,7 @@ public class TestQJMWithFaults {
@Override
protected ExecutorService createSingleThreadExecutor() {
- return MoreExecutors.newDirectExecutorService();
+ return new DirectExecutorService();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/543aac9f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
index 9aada1d..8d92666 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
@@ -66,7 +66,6 @@ import org.mockito.Mockito;
import org.mockito.stubbing.Stubber;
import com.google.common.collect.Lists;
-import com.google.common.util.concurrent.MoreExecutors;
/**
* Functional tests for QuorumJournalManager.
@@ -946,7 +945,7 @@ public class TestQuorumJournalManager {
protected ExecutorService createSingleThreadExecutor() {
// Don't parallelize calls to the quorum in the tests.
// This makes the tests more deterministic.
- return MoreExecutors.newDirectExecutorService();
+ return new DirectExecutorService();
}
};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/543aac9f/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index c9b6522..4097a0a 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -90,7 +90,7 @@
<findbugs.version>3.0.0</findbugs.version>
<spotbugs.version>3.1.0-RC1</spotbugs.version>
- <guava.version>21.0</guava.version>
+ <guava.version>11.0.2</guava.version>
<guice.version>4.0</guice.version>
<joda-time.version>2.9.4</joda-time.version>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/543aac9f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
index 0148d0e..6e46eb6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
@@ -51,7 +51,6 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ReservationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceRequestProto;
-import com.google.common.base.CharMatcher;
import com.google.protobuf.TextFormat;
@Private
@@ -286,7 +285,7 @@ extends ApplicationSubmissionContext {
"maximum allowed length of a tag is " +
YarnConfiguration.APPLICATION_MAX_TAG_LENGTH);
}
- if (!CharMatcher.ascii().matchesAllOf(tag)) {
+ if (!org.apache.commons.lang3.StringUtils.isAsciiPrintable(tag)) {
throw new IllegalArgumentException("A tag can only have ASCII " +
"characters! Invalid tag - " + tag);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/543aac9f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java
index 300bf3e..fad6fe2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java
@@ -26,6 +26,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpServer2;
@@ -33,7 +34,6 @@ import org.apache.hadoop.yarn.webapp.view.RobotsTextPage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.CharMatcher;
import com.google.common.base.Splitter;
import com.google.common.collect.Lists;
import com.google.inject.Provides;
@@ -275,7 +275,7 @@ public abstract class WebApp extends ServletModule {
static String getPrefix(String pathSpec) {
int start = 0;
- while (CharMatcher.whitespace().matches(pathSpec.charAt(start))) {
+ while (StringUtils.isAnyBlank(Character.toString(pathSpec.charAt(start)))) {
++start;
}
if (pathSpec.charAt(start) != '/') {
@@ -291,7 +291,7 @@ public abstract class WebApp extends ServletModule {
char c;
do {
c = pathSpec.charAt(--ci);
- } while (c == '/' || CharMatcher.whitespace().matches(c));
+ } while (c == '/' || StringUtils.isAnyBlank(Character.toString(c)));
return pathSpec.substring(start, ci + 1);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/543aac9f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index e49c3ed..368832a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -21,6 +21,8 @@ import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -68,7 +70,6 @@ import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.SettableFuture;
/**
@@ -210,6 +211,17 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
ApplicationSummary.logAppSummary(rmContext.getRMApps().get(appId));
}
+ private static <V> V getChecked(Future<V> future) throws YarnException {
+ try {
+ return future.get();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new YarnException(e);
+ } catch (ExecutionException e) {
+ throw new YarnException(e);
+ }
+ }
+
protected synchronized int getCompletedAppsListSize() {
return this.completedApps.size();
}
@@ -641,7 +653,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
this.rmContext.getStateStore()
.updateApplicationStateSynchronously(appState, false, future);
- Futures.getChecked(future, YarnException.class);
+ getChecked(future);
// update in-memory
((RMAppImpl) app).updateApplicationTimeout(newExpireTime);
@@ -678,7 +690,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
return;
}
- Futures.getChecked(future, YarnException.class);
+ getChecked(future);
// update in-memory
((RMAppImpl) app).setApplicationPriority(appPriority);
@@ -761,7 +773,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
false, future);
try {
- Futures.getChecked(future, YarnException.class);
+ getChecked(future);
} catch (YarnException ex) {
if (!toSuppressException) {
throw ex;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/543aac9f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
index d29d34e..bc204cb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
@@ -58,7 +58,6 @@ import org.w3c.dom.NodeList;
import org.w3c.dom.Text;
import org.xml.sax.SAXException;
-import com.google.common.base.CharMatcher;
import com.google.common.annotations.VisibleForTesting;
@Public
@@ -465,7 +464,7 @@ public class AllocationFileLoaderService extends AbstractService {
Set<String> reservableQueues,
Set<String> nonPreemptableQueues)
throws AllocationConfigurationException {
- String queueName = CharMatcher.whitespace().trimFrom(
+ String queueName = FairSchedulerUtilities.trimQueueName(
element.getAttribute("name"));
if (queueName.contains(".")) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/543aac9f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerUtilities.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerUtilities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerUtilities.java
new file mode 100644
index 0000000..f394a93
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerUtilities.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
+
+/**
+ * Utility class for the Fair Scheduler.
+ */
+public final class FairSchedulerUtilities {
+
+ /**
+ * Table copied from Google Guava v19:
+ * com/google/common/base/CharMatcher.java
+ * <p>
+ * Licensed under the Apache License Version 2.0.
+ */
+ static final String WHITESPACE_TABLE =
+ "\u2002\u3000\r\u0085\u200A\u2005\u2000\u3000"
+ + "\u2029\u000B\u3000\u2008\u2003\u205F\u3000\u1680"
+ + "\u0009\u0020\u2006\u2001\u202F\u00A0\u000C\u2009"
+ + "\u3000\u2004\u3000\u3000\u2028\n\u2007\u3000";
+
+ private FairSchedulerUtilities() {
+ // private constructor because this is a utility class.
+ }
+
+ private static boolean isWhitespace(char c) {
+ for (int i = 0; i < WHITESPACE_TABLE.length(); i++) {
+ if (WHITESPACE_TABLE.charAt(i) == c) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public static String trimQueueName(String name) {
+ if (name == null) {
+ return null;
+ }
+ int start = 0;
+ while (start < name.length()
+ && isWhitespace(name.charAt(start))
+ && start < name.length()) {
+ start++;
+ }
+ int end = name.length() - 1;
+ while (end >= 0
+ && isWhitespace(name.charAt(end))
+ && end > start) {
+ end--;
+ }
+ return name.substring(start, end+1);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/543aac9f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
index 5b006df..c08d13e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
@@ -38,7 +38,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy;
import org.xml.sax.SAXException;
-import com.google.common.base.CharMatcher;
import com.google.common.annotations.VisibleForTesting;
import java.util.Iterator;
import java.util.Set;
@@ -533,8 +532,9 @@ public class QueueManager {
@VisibleForTesting
boolean isQueueNameValid(String node) {
// use the same white space trim as in QueueMetrics() otherwise things fail
- // guava uses a different definition for whitespace than java.
+ // This needs to trim additional Unicode whitespace characters beyond what
+ // the built-in JDK methods consider whitespace. See YARN-5272.
return !node.isEmpty() &&
- node.equals(CharMatcher.whitespace().trimFrom(node));
+ node.equals(FairSchedulerUtilities.trimQueueName(node));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/543aac9f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerUtilities.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerUtilities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerUtilities.java
new file mode 100644
index 0000000..37f686e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerUtilities.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
+
+import org.junit.Test;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerUtilities.trimQueueName;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests for {@link FairSchedulerUtilities}.
+ */
+public class TestFairSchedulerUtilities {
+
+ @Test
+ public void testTrimQueueNameEquals() throws Exception {
+ final String[] equalsStrings = {
+ // no spaces
+ "a",
+ // leading spaces
+ " a",
+ " \u3000a",
+ "\u2002\u3000\r\u0085\u200A\u2005\u2000\u3000a",
+ "\u2029\u000B\u3000\u2008\u2003\u205F\u3000\u1680a",
+ "\u0009\u0020\u2006\u2001\u202F\u00A0\u000C\u2009a",
+ "\u3000\u2004\u3000\u3000\u2028\n\u2007\u3000a",
+ // trailing spaces
+ "a\u200A",
+ "a \u0085 ",
+ // spaces on both sides
+ " a ",
+ " a\u00A0",
+ "\u0009\u0020\u2006\u2001\u202F\u00A0\u000C\u2009a" +
+ "\u3000\u2004\u3000\u3000\u2028\n\u2007\u3000",
+ };
+ for (String s : equalsStrings) {
+ assertEquals("a", trimQueueName(s));
+ }
+ }
+
+ @Test
+ public void testTrimQueueNamesEmpty() throws Exception {
+ assertNull(trimQueueName(null));
+ final String spaces = "\u2002\u3000\r\u0085\u200A\u2005\u2000\u3000"
+ + "\u2029\u000B\u3000\u2008\u2003\u205F\u3000\u1680"
+ + "\u0009\u0020\u2006\u2001\u202F\u00A0\u000C\u2009"
+ + "\u3000\u2004\u3000\u3000\u2028\n\u2007\u3000";
+ assertTrue(trimQueueName(spaces).isEmpty());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: AuditLogger and TestAuditLogger
are dead code. Contributed by Vrushali C
Posted by in...@apache.org.
AuditLogger and TestAuditLogger are dead code. Contributed by Vrushali C
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/45382bf0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/45382bf0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/45382bf0
Branch: refs/heads/HDFS-10467
Commit: 45382bf047d1ffa2ce1fa88f1f472daf524e2d22
Parents: 149b903
Author: Chris Douglas <cd...@apache.org>
Authored: Tue May 9 10:25:10 2017 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Tue May 9 10:31:52 2017 -0700
----------------------------------------------------------------------
.../src/main/conf/log4j.properties | 17 +-
.../org/apache/hadoop/mapred/AuditLogger.java | 154 ------------------
.../apache/hadoop/mapred/TestAuditLogger.java | 161 -------------------
3 files changed, 1 insertion(+), 331 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/45382bf0/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
index 22a2117..3752ad1 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
+++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
@@ -176,21 +176,6 @@ log4j.appender.DNMETRICSRFA.layout.ConversionPattern=%d{ISO8601} %m%n
log4j.appender.DNMETRICSRFA.MaxBackupIndex=1
log4j.appender.DNMETRICSRFA.MaxFileSize=64MB
-#
-# mapred audit logging
-#
-mapred.audit.logger=INFO,NullAppender
-mapred.audit.log.maxfilesize=256MB
-mapred.audit.log.maxbackupindex=20
-log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}
-log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false
-log4j.appender.MRAUDIT=org.apache.log4j.RollingFileAppender
-log4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log
-log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout
-log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
-log4j.appender.MRAUDIT.MaxFileSize=${mapred.audit.log.maxfilesize}
-log4j.appender.MRAUDIT.MaxBackupIndex=${mapred.audit.log.maxbackupindex}
-
# Custom Logging levels
#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG
@@ -334,4 +319,4 @@ log4j.appender.EWMA.maxUniqueMessages=${yarn.ewma.maxUniqueMessages}
#log4j.appender.FSSTATEDUMP.layout=org.apache.log4j.PatternLayout
#log4j.appender.FSSTATEDUMP.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
#log4j.appender.FSSTATEDUMP.MaxFileSize=${hadoop.log.maxfilesize}
-#log4j.appender.FSSTATEDUMP.MaxBackupIndex=${hadoop.log.maxbackupindex}
\ No newline at end of file
+#log4j.appender.FSSTATEDUMP.MaxBackupIndex=${hadoop.log.maxbackupindex}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/45382bf0/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/AuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/AuditLogger.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/AuditLogger.java
deleted file mode 100644
index d1ce94c..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/AuditLogger.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-import java.net.InetAddress;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.ipc.Server;
-
-/** Manages MapReduce audit logs. Audit logs provides information about
- * authorization/authentication events (success/failure).
- *
- * Audit log format is written as key=value pairs.
- */
-class AuditLogger {
- private static final Log LOG = LogFactory.getLog(AuditLogger.class);
-
- static enum Keys {USER, OPERATION, TARGET, RESULT, IP, PERMISSIONS,
- DESCRIPTION}
-
- static class Constants {
- static final String SUCCESS = "SUCCESS";
- static final String FAILURE = "FAILURE";
- static final String KEY_VAL_SEPARATOR = "=";
- static final char PAIR_SEPARATOR = '\t';
-
- // Some constants used by others using AuditLogger.
-
- // Some commonly used targets
- static final String JOBTRACKER = "JobTracker";
-
- // Some commonly used operations
- static final String REFRESH_QUEUE = "REFRESH_QUEUE";
- static final String REFRESH_NODES = "REFRESH_NODES";
-
- // Some commonly used descriptions
- static final String UNAUTHORIZED_USER = "Unauthorized user";
- }
-
- /**
- * A helper api for creating an audit log for a successful event.
- * This is factored out for testing purpose.
- */
- static String createSuccessLog(String user, String operation, String target) {
- StringBuilder b = new StringBuilder();
- start(Keys.USER, user, b);
- addRemoteIP(b);
- add(Keys.OPERATION, operation, b);
- add(Keys.TARGET, target ,b);
- add(Keys.RESULT, Constants.SUCCESS, b);
- return b.toString();
- }
-
- /**
- * Create a readable and parseable audit log string for a successful event.
- *
- * @param user User who made the service request to the JobTracker.
- * @param operation Operation requested by the user
- * @param target The target on which the operation is being performed. Most
- * commonly operated targets are jobs, JobTracker, queues etc
- *
- * <br><br>
- * Note that the {@link AuditLogger} uses tabs ('\t') as a key-val delimiter
- * and hence the value fields should not contains tabs ('\t').
- */
- static void logSuccess(String user, String operation, String target) {
- if (LOG.isInfoEnabled()) {
- LOG.info(createSuccessLog(user, operation, target));
- }
- }
-
- /**
- * A helper api for creating an audit log for a failure event.
- * This is factored out for testing purpose.
- */
- static String createFailureLog(String user, String operation, String perm,
- String target, String description) {
- StringBuilder b = new StringBuilder();
- start(Keys.USER, user, b);
- addRemoteIP(b);
- add(Keys.OPERATION, operation, b);
- add(Keys.TARGET, target ,b);
- add(Keys.RESULT, Constants.FAILURE, b);
- add(Keys.DESCRIPTION, description, b);
- add(Keys.PERMISSIONS, perm, b);
- return b.toString();
- }
-
- /**
- * Create a readable and parseable audit log string for a failed event.
- *
- * @param user User who made the service request to the JobTracker.
- * @param operation Operation requested by the user
- * @param perm Target permissions like JobACLs for jobs, QueueACLs for queues.
- * @param target The target on which the operation is being performed. Most
- * commonly operated targets are jobs, JobTracker, queues etc
- * @param description Some additional information as to why the operation
- * failed.
- *
- * <br><br>
- * Note that the {@link AuditLogger} uses tabs ('\t') as a key-val delimiter
- * and hence the value fields should not contains tabs ('\t').
- */
- static void logFailure(String user, String operation, String perm,
- String target, String description) {
- if (LOG.isWarnEnabled()) {
- LOG.warn(createFailureLog(user, operation, perm, target, description));
- }
- }
-
- /**
- * A helper api to add remote IP address
- */
- static void addRemoteIP(StringBuilder b) {
- InetAddress ip = Server.getRemoteIp();
- // ip address can be null for testcases
- if (ip != null) {
- add(Keys.IP, ip.getHostAddress(), b);
- }
- }
-
- /**
- * Adds the first key-val pair to the passed builder in the following format
- * key=value
- */
- static void start(Keys key, String value, StringBuilder b) {
- b.append(key.name()).append(Constants.KEY_VAL_SEPARATOR).append(value);
- }
-
- /**
- * Appends the key-val pair to the passed builder in the following format
- * <pair-delim>key=value
- */
- static void add(Keys key, String value, StringBuilder b) {
- b.append(Constants.PAIR_SEPARATOR).append(key.name())
- .append(Constants.KEY_VAL_SEPARATOR).append(value);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/45382bf0/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestAuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestAuditLogger.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestAuditLogger.java
deleted file mode 100644
index bc85703..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestAuditLogger.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.ProtocolInfo;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.ipc.TestRPC.TestImpl;
-import org.apache.hadoop.ipc.TestRPC.TestProtocol;
-import org.apache.hadoop.mapred.AuditLogger.Keys;
-import org.apache.hadoop.net.NetUtils;
-import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-
-/**
- * Tests {@link AuditLogger}.
- */
-public class TestAuditLogger {
- private static final String USER = "test";
- private static final String OPERATION = "oper";
- private static final String TARGET = "tgt";
- private static final String PERM = "admin group";
- private static final String DESC = "description of an audit log";
-
- /**
- * Test the AuditLog format with key-val pair.
- */
- @Test
- public void testKeyValLogFormat() {
- StringBuilder actLog = new StringBuilder();
- StringBuilder expLog = new StringBuilder();
- // add the first k=v pair and check
- AuditLogger.start(Keys.USER, USER, actLog);
- expLog.append("USER=test");
- assertEquals(expLog.toString(), actLog.toString());
-
- // append another k1=v1 pair to already added k=v and test
- AuditLogger.add(Keys.OPERATION, OPERATION, actLog);
- expLog.append("\tOPERATION=oper");
- assertEquals(expLog.toString(), actLog.toString());
-
- // append another k1=null pair and test
- AuditLogger.add(Keys.PERMISSIONS, (String)null, actLog);
- expLog.append("\tPERMISSIONS=null");
- assertEquals(expLog.toString(), actLog.toString());
-
- // now add the target and check of the final string
- AuditLogger.add(Keys.TARGET, TARGET, actLog);
- expLog.append("\tTARGET=tgt");
- assertEquals(expLog.toString(), actLog.toString());
- }
-
- /**
- * Test the AuditLog format for successful events.
- */
- private void testSuccessLogFormat(boolean checkIP) {
- // check without the IP
- String sLog = AuditLogger.createSuccessLog(USER, OPERATION, TARGET);
- StringBuilder expLog = new StringBuilder();
- expLog.append("USER=test\t");
- if (checkIP) {
- InetAddress ip = Server.getRemoteIp();
- expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
- }
- expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=SUCCESS");
- assertEquals(expLog.toString(), sLog);
-
- }
-
- /**
- * Test the AuditLog format for failure events.
- */
- private void testFailureLogFormat(boolean checkIP, String perm) {
- String fLog =
- AuditLogger.createFailureLog(USER, OPERATION, perm, TARGET, DESC);
- StringBuilder expLog = new StringBuilder();
- expLog.append("USER=test\t");
- if (checkIP) {
- InetAddress ip = Server.getRemoteIp();
- expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
- }
- expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=FAILURE\t");
- expLog.append("DESCRIPTION=description of an audit log\t");
- expLog.append("PERMISSIONS=" + perm);
- assertEquals(expLog.toString(), fLog);
- }
-
- /**
- * Test the AuditLog format for failure events.
- */
- private void testFailureLogFormat(boolean checkIP) {
- testFailureLogFormat(checkIP, PERM);
- testFailureLogFormat(checkIP, null);
- }
-
- /**
- * Test {@link AuditLogger} without IP set.
- */
- @Test
- public void testAuditLoggerWithoutIP() throws Exception {
- // test without ip
- testSuccessLogFormat(false);
- testFailureLogFormat(false);
- }
-
- /**
- * A special extension of {@link TestImpl} RPC server with
- * {@link TestImpl#ping()} testing the audit logs.
- */
- @ProtocolInfo(protocolName = "org.apache.hadoop.ipc.TestRPC$TestProtocol")
- private class MyTestRPCServer extends TestImpl {
- @Override
- public void ping() {
- // test with ip set
- testSuccessLogFormat(true);
- testFailureLogFormat(true);
- }
- }
-
- /**
- * Test {@link AuditLogger} with IP set.
- */
- @Test
- public void testAuditLoggerWithIP() throws Exception {
- Configuration conf = new Configuration();
- // start the IPC server
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new MyTestRPCServer()).setBindAddress("0.0.0.0")
- .setPort(0).build();
- server.start();
-
- InetSocketAddress addr = NetUtils.getConnectAddress(server);
-
- // Make a client connection and test the audit log
- TestProtocol proxy = (TestProtocol)RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, addr, conf);
- // Start the testcase
- proxy.ping();
-
- server.stop();
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: YARN-6565. Fix memory leak and
finish app trigger in AMRMProxy. (Botong Huang via Subru).
Posted by in...@apache.org.
YARN-6565. Fix memory leak and finish app trigger in AMRMProxy. (Botong Huang via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d6eed5ac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d6eed5ac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d6eed5ac
Branch: refs/heads/HDFS-10467
Commit: d6eed5acca222a281401c1d524824f0460ea7826
Parents: e4f34ec
Author: Subru Krishnan <su...@apache.org>
Authored: Fri May 5 16:27:49 2017 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri May 5 16:27:49 2017 -0700
----------------------------------------------------------------------
.../nodemanager/amrmproxy/AMRMProxyService.java | 6 +-
.../TestAMRMProxyTokenSecretManager.java | 81 ++++++++++++++++++++
2 files changed, 86 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6eed5ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
index 5e91a20..9f2d9a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
@@ -320,6 +320,10 @@ public class AMRMProxyService extends AbstractService implements
LOG.info("Request to stop an application that does not exist. Id:"
+ applicationId);
} else {
+ // Remove the appAttempt in AMRMTokenSecretManager
+ this.secretManager
+ .applicationMasterFinished(pipeline.getApplicationAttemptId());
+
LOG.info("Stopping the request processing pipeline for application: "
+ applicationId);
try {
@@ -548,7 +552,7 @@ public class AMRMProxyService extends AbstractService implements
event.getApplicationID());
if (app != null) {
switch (event.getType()) {
- case FINISH_APPLICATION:
+ case APPLICATION_RESOURCES_CLEANEDUP:
LOG.info("Application stop event received for stopping AppId:"
+ event.getApplicationID().toString());
AMRMProxyService.this.stopApplication(event.getApplicationID());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6eed5ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyTokenSecretManager.java
new file mode 100644
index 0000000..927563e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyTokenSecretManager.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.amrmproxy;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Unit test for AMRMProxyTokenSecretManager.
+ */
+public class TestAMRMProxyTokenSecretManager {
+
+ private YarnConfiguration conf;
+ private AMRMProxyTokenSecretManager secretManager;
+
+ @Before
+ public void setup() {
+ conf = new YarnConfiguration();
+ secretManager = new AMRMProxyTokenSecretManager(conf);
+ secretManager.start();
+ }
+
+ @After
+ public void breakdown() {
+ if (secretManager != null) {
+ secretManager.stop();
+ }
+ }
+
+ @Test
+ public void testNormalCase() throws IOException {
+ ApplicationId appId = ApplicationId.newInstance(1, 1);
+ ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
+
+ Token<AMRMTokenIdentifier> localToken =
+ secretManager.createAndGetAMRMToken(attemptId);
+
+ AMRMTokenIdentifier identifier = secretManager.createIdentifier();
+ identifier.readFields(new DataInputStream(
+ new ByteArrayInputStream(localToken.getIdentifier())));
+
+ secretManager.retrievePassword(identifier);
+
+ secretManager.applicationMasterFinished(attemptId);
+
+ try {
+ secretManager.retrievePassword(identifier);
+ Assert.fail("Expect InvalidToken exception");
+ } catch (InvalidToken e) {
+ }
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: HDFS-10630. Federation State Store
FS Implementation. Contributed by Jason Kace and Inigo Goiri.
Posted by in...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57ff72c5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
new file mode 100644
index 0000000..7f0b36a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
@@ -0,0 +1,483 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.Query;
+import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
+import org.junit.AfterClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Base tests for the driver. The particular implementations will use this to
+ * test their functionality.
+ */
+public class TestStateStoreDriverBase {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestStateStoreDriverBase.class);
+
+ private static StateStoreService stateStore;
+ private static Configuration conf;
+
+
+ /**
+ * Get the State Store driver.
+ * @return State Store driver.
+ */
+ protected StateStoreDriver getStateStoreDriver() {
+ return stateStore.getDriver();
+ }
+
+ @AfterClass
+ public static void tearDownCluster() {
+ if (stateStore != null) {
+ stateStore.stop();
+ }
+ }
+
+ /**
+ * Get a new State Store using this configuration.
+ *
+ * @param config Configuration for the State Store.
+ * @throws Exception If we cannot get the State Store.
+ */
+ public static void getStateStore(Configuration config) throws Exception {
+ conf = config;
+ stateStore = FederationStateStoreTestUtils.getStateStore(conf);
+ }
+
+ private <T extends BaseRecord> T generateFakeRecord(Class<T> recordClass)
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+
+ // TODO add record
+ return null;
+ }
+
+ /**
+ * Validate if a record is the same.
+ *
+ * @param original
+ * @param committed
+ * @param assertEquals Assert if the records are equal or just return.
+ * @return
+ * @throws IllegalArgumentException
+ * @throws IllegalAccessException
+ */
+ private boolean validateRecord(
+ BaseRecord original, BaseRecord committed, boolean assertEquals)
+ throws IllegalArgumentException, IllegalAccessException {
+
+ boolean ret = true;
+
+ Map<String, Class<?>> fields = getFields(original);
+ for (String key : fields.keySet()) {
+ if (key.equals("dateModified") ||
+ key.equals("dateCreated") ||
+ key.equals("proto")) {
+ // Fields are updated/set on commit and fetch and may not match
+ // the fields that are initialized in a non-committed object.
+ continue;
+ }
+ Object data1 = getField(original, key);
+ Object data2 = getField(committed, key);
+ if (assertEquals) {
+ assertEquals("Field " + key + " does not match", data1, data2);
+ } else if (!data1.equals(data2)) {
+ ret = false;
+ }
+ }
+
+ long now = stateStore.getDriver().getTime();
+ assertTrue(
+ committed.getDateCreated() <= now && committed.getDateCreated() > 0);
+ assertTrue(committed.getDateModified() >= committed.getDateCreated());
+
+ return ret;
+ }
+
+ public static void removeAll(StateStoreDriver driver) throws IOException {
+ // TODO add records to remove
+ }
+
+ public <T extends BaseRecord> void testInsert(
+ StateStoreDriver driver, Class<T> recordClass)
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+
+ assertTrue(driver.removeAll(recordClass));
+ QueryResult<T> records = driver.get(recordClass);
+ assertTrue(records.getRecords().isEmpty());
+
+ // Insert single
+ BaseRecord record = generateFakeRecord(recordClass);
+ driver.put(record, true, false);
+
+ // Verify
+ records = driver.get(recordClass);
+ assertEquals(1, records.getRecords().size());
+ validateRecord(record, records.getRecords().get(0), true);
+
+ // Insert multiple
+ List<T> insertList = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ T newRecord = generateFakeRecord(recordClass);
+ insertList.add(newRecord);
+ }
+ driver.putAll(insertList, true, false);
+
+ // Verify
+ records = driver.get(recordClass);
+ assertEquals(11, records.getRecords().size());
+ }
+
+ public <T extends BaseRecord> void testFetchErrors(StateStoreDriver driver,
+ Class<T> clazz) throws IllegalAccessException, IOException {
+
+ // Fetch empty list
+ driver.removeAll(clazz);
+ QueryResult<T> result0 = driver.get(clazz);
+ assertNotNull(result0);
+ List<T> records0 = result0.getRecords();
+ assertEquals(records0.size(), 0);
+
+ // Insert single
+ BaseRecord record = generateFakeRecord(clazz);
+ assertTrue(driver.put(record, true, false));
+
+ // Verify
+ QueryResult<T> result1 = driver.get(clazz);
+ List<T> records1 = result1.getRecords();
+ assertEquals(1, records1.size());
+ validateRecord(record, records1.get(0), true);
+
+ // Test fetch single object with a bad query
+ final T fakeRecord = generateFakeRecord(clazz);
+ final Query<T> query = new Query<T>(fakeRecord);
+ T getRecord = driver.get(clazz, query);
+ assertNull(getRecord);
+
+ // Test fetch multiple objects does not exist returns empty list
+ assertEquals(driver.getMultiple(clazz, query).size(), 0);
+ }
+
+ public <T extends BaseRecord> void testPut(
+ StateStoreDriver driver, Class<T> clazz)
+ throws IllegalArgumentException, ReflectiveOperationException,
+ IOException, SecurityException {
+
+ driver.removeAll(clazz);
+ QueryResult<T> records = driver.get(clazz);
+ assertTrue(records.getRecords().isEmpty());
+
+ // Insert multiple
+ List<T> insertList = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ T newRecord = generateFakeRecord(clazz);
+ insertList.add(newRecord);
+ }
+
+ // Verify
+ assertTrue(driver.putAll(insertList, false, true));
+ records = driver.get(clazz);
+ assertEquals(records.getRecords().size(), 10);
+
+ // Generate a new record with the same PK fields as an existing record
+ BaseRecord updatedRecord = generateFakeRecord(clazz);
+ BaseRecord existingRecord = records.getRecords().get(0);
+ Map<String, String> primaryKeys = existingRecord.getPrimaryKeys();
+ for (Entry<String, String> entry : primaryKeys.entrySet()) {
+ String key = entry.getKey();
+ String value = entry.getValue();
+ Class<?> fieldType = getFieldType(existingRecord, key);
+ Object field = fromString(value, fieldType);
+ assertTrue(setField(updatedRecord, key, field));
+ }
+
+ // Attempt an update of an existing entry, but it is not allowed.
+ assertFalse(driver.put(updatedRecord, false, true));
+
+ // Verify no update occurred, all original records are unchanged
+ QueryResult<T> newRecords = driver.get(clazz);
+ assertTrue(newRecords.getRecords().size() == 10);
+ assertEquals("A single entry was improperly updated in the store", 10,
+ countMatchingEntries(records.getRecords(), newRecords.getRecords()));
+
+ // Update the entry (allowing updates)
+ assertTrue(driver.put(updatedRecord, true, false));
+
+ // Verify that one entry no longer matches the original set
+ newRecords = driver.get(clazz);
+ assertEquals(10, newRecords.getRecords().size());
+ assertEquals(
+ "Record of type " + clazz + " not updated in the store", 9,
+ countMatchingEntries(records.getRecords(), newRecords.getRecords()));
+ }
+
+ private int countMatchingEntries(
+ Collection<? extends BaseRecord> committedList,
+ Collection<? extends BaseRecord> matchList) {
+
+ int matchingCount = 0;
+ for (BaseRecord committed : committedList) {
+ for (BaseRecord match : matchList) {
+ try {
+ if (match.getPrimaryKey().equals(committed.getPrimaryKey())) {
+ if (validateRecord(match, committed, false)) {
+ matchingCount++;
+ }
+ break;
+ }
+ } catch (Exception ex) {
+ }
+ }
+ }
+ return matchingCount;
+ }
+
+ public <T extends BaseRecord> void testRemove(
+ StateStoreDriver driver, Class<T> clazz)
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+
+ // Remove all
+ assertTrue(driver.removeAll(clazz));
+ QueryResult<T> records = driver.get(clazz);
+ assertTrue(records.getRecords().isEmpty());
+
+ // Insert multiple
+ List<T> insertList = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ T newRecord = generateFakeRecord(clazz);
+ insertList.add(newRecord);
+ }
+
+ // Verify
+ assertTrue(driver.putAll(insertList, false, true));
+ records = driver.get(clazz);
+ assertEquals(records.getRecords().size(), 10);
+
+ // Remove Single
+ assertTrue(driver.remove(records.getRecords().get(0)));
+
+ // Verify
+ records = driver.get(clazz);
+ assertEquals(records.getRecords().size(), 9);
+
+ // Remove with filter
+ final T firstRecord = records.getRecords().get(0);
+ final Query<T> query0 = new Query<T>(firstRecord);
+ assertTrue(driver.remove(clazz, query0) > 0);
+
+ final T secondRecord = records.getRecords().get(1);
+ final Query<T> query1 = new Query<T>(secondRecord);
+ assertTrue(driver.remove(clazz, query1) > 0);
+
+ // Verify
+ records = driver.get(clazz);
+ assertEquals(records.getRecords().size(), 7);
+
+ // Remove all
+ assertTrue(driver.removeAll(clazz));
+
+ // Verify
+ records = driver.get(clazz);
+ assertTrue(records.getRecords().isEmpty());
+ }
+
+ public void testInsert(StateStoreDriver driver)
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ // TODO add records
+ }
+
+ public void testPut(StateStoreDriver driver)
+ throws IllegalArgumentException, ReflectiveOperationException,
+ IOException, SecurityException {
+ // TODO add records
+ }
+
+ public void testRemove(StateStoreDriver driver)
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ // TODO add records
+ }
+
+ public void testFetchErrors(StateStoreDriver driver)
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ // TODO add records
+ }
+
+ /**
+ * Sets the value of a field on the object.
+ *
+ * @param fieldName The string name of the field.
+ * @param data The data to pass to the field's setter.
+ *
+ * @return True if successful, fails if failed.
+ */
+ private static boolean setField(
+ BaseRecord record, String fieldName, Object data) {
+
+ Method m = locateSetter(record, fieldName);
+ if (m != null) {
+ try {
+ m.invoke(record, data);
+ } catch (Exception e) {
+ LOG.error("Cannot set field " + fieldName + " on object "
+ + record.getClass().getName() + " to data " + data + " of type "
+ + data.getClass(), e);
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Finds the appropriate setter for a field name.
+ *
+ * @param fieldName The legacy name of the field.
+ * @return The matching setter or null if not found.
+ */
+ private static Method locateSetter(BaseRecord record, String fieldName) {
+ for (Method m : record.getClass().getMethods()) {
+ if (m.getName().equalsIgnoreCase("set" + fieldName)) {
+ return m;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Returns all serializable fields in the object.
+ *
+ * @return Map with the fields.
+ */
+ private static Map<String, Class<?>> getFields(BaseRecord record) {
+ Map<String, Class<?>> getters = new HashMap<>();
+ for (Method m : record.getClass().getDeclaredMethods()) {
+ if (m.getName().startsWith("get")) {
+ try {
+ Class<?> type = m.getReturnType();
+ char[] c = m.getName().substring(3).toCharArray();
+ c[0] = Character.toLowerCase(c[0]);
+ String key = new String(c);
+ getters.put(key, type);
+ } catch (Exception e) {
+ LOG.error("Cannot execute getter " + m.getName()
+ + " on object " + record);
+ }
+ }
+ }
+ return getters;
+ }
+
+ /**
+ * Get the type of a field.
+ *
+ * @param fieldName
+ * @return Field type
+ */
+ private static Class<?> getFieldType(BaseRecord record, String fieldName) {
+ Method m = locateGetter(record, fieldName);
+ return m.getReturnType();
+ }
+
+ /**
+ * Fetches the value for a field name.
+ *
+ * @param fieldName the legacy name of the field.
+ * @return The field data or null if not found.
+ */
+ private static Object getField(BaseRecord record, String fieldName) {
+ Object result = null;
+ Method m = locateGetter(record, fieldName);
+ if (m != null) {
+ try {
+ result = m.invoke(record);
+ } catch (Exception e) {
+ LOG.error("Cannot get field " + fieldName + " on object " + record);
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Finds the appropriate getter for a field name.
+ *
+ * @param fieldName The legacy name of the field.
+ * @return The matching getter or null if not found.
+ */
+ private static Method locateGetter(BaseRecord record, String fieldName) {
+ for (Method m : record.getClass().getMethods()) {
+ if (m.getName().equalsIgnoreCase("get" + fieldName)) {
+ return m;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Expands a data object from the store into an record object. Default store
+ * data type is a String. Override if additional serialization is required.
+ *
+ * @param data Object containing the serialized data. Only string is
+ * supported.
+ * @param clazz Target object class to hold the deserialized data.
+ * @return An instance of the target data object initialized with the
+ * deserialized data.
+ */
+ @Deprecated
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ private static <T> T fromString(String data, Class<T> clazz) {
+
+ if (data.equals("null")) {
+ return null;
+ } else if (clazz == String.class) {
+ return (T) data;
+ } else if (clazz == Long.class || clazz == long.class) {
+ return (T) Long.valueOf(data);
+ } else if (clazz == Integer.class || clazz == int.class) {
+ return (T) Integer.valueOf(data);
+ } else if (clazz == Double.class || clazz == double.class) {
+ return (T) Double.valueOf(data);
+ } else if (clazz == Float.class || clazz == float.class) {
+ return (T) Float.valueOf(data);
+ } else if (clazz == Boolean.class || clazz == boolean.class) {
+ return (T) Boolean.valueOf(data);
+ } else if (clazz.isEnum()) {
+ return (T) Enum.valueOf((Class<Enum>) clazz, data);
+ }
+ return null;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57ff72c5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFile.java
new file mode 100644
index 0000000..920e280
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFile.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver;
+
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.getStateStoreConfiguration;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test the FileSystem (e.g., HDFS) implementation of the State Store driver.
+ */
+public class TestStateStoreFile extends TestStateStoreDriverBase {
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ Configuration conf = getStateStoreConfiguration(StateStoreFileImpl.class);
+ getStateStore(conf);
+ }
+
+ @Before
+ public void startup() throws IOException {
+ removeAll(getStateStoreDriver());
+ }
+
+ @Test
+ public void testInsert()
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testInsert(getStateStoreDriver());
+ }
+
+ @Test
+ public void testUpdate()
+ throws IllegalArgumentException, ReflectiveOperationException,
+ IOException, SecurityException {
+ testPut(getStateStoreDriver());
+ }
+
+ @Test
+ public void testDelete()
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testRemove(getStateStoreDriver());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57ff72c5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileSystem.java
new file mode 100644
index 0000000..da2e51d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileSystem.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils;
+import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileSystemImpl;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test the FileSystem (e.g., HDFS) implementation of the State Store driver.
+ */
+public class TestStateStoreFileSystem extends TestStateStoreDriverBase {
+
+ private static MiniDFSCluster dfsCluster;
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ Configuration conf = FederationStateStoreTestUtils
+ .getStateStoreConfiguration(StateStoreFileSystemImpl.class);
+ conf.set(StateStoreFileSystemImpl.FEDERATION_STORE_FS_PATH,
+ "/hdfs-federation/");
+
+ // Create HDFS cluster to back the state tore
+ MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
+ builder.numDataNodes(1);
+ dfsCluster = builder.build();
+ dfsCluster.waitClusterUp();
+ getStateStore(conf);
+ }
+
+ @AfterClass
+ public static void tearDownCluster() {
+ if (dfsCluster != null) {
+ dfsCluster.shutdown();
+ }
+ }
+
+ @Before
+ public void startup() throws IOException {
+ removeAll(getStateStoreDriver());
+ }
+
+ @Test
+ public void testInsert()
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testInsert(getStateStoreDriver());
+ }
+
+ @Test
+ public void testUpdate()
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testInsert(getStateStoreDriver());
+ }
+
+ @Test
+ public void testDelete()
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testInsert(getStateStoreDriver());
+ }
+
+ @Test
+ public void testFetchErrors()
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testFetchErrors(getStateStoreDriver());
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: HADOOP-14400. Fix warnings from
spotbugs in hadoop-tools. Contributed by Weiwei Yang.
Posted by in...@apache.org.
HADOOP-14400. Fix warnings from spotbugs in hadoop-tools. Contributed by Weiwei Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2ba99039
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2ba99039
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2ba99039
Branch: refs/heads/HDFS-10467
Commit: 2ba9903932e3c99afb0e6abb7cd3c5d29a554bd9
Parents: a5fa954
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed May 10 05:57:12 2017 -0500
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed May 10 05:57:12 2017 -0500
----------------------------------------------------------------------
.../hadoop/fs/azure/NativeAzureFileSystem.java | 10 -------
.../hadoop/mapred/gridmix/InputStriper.java | 6 ++---
.../TotalHeapUsageEmulatorPlugin.java | 16 +++++++++--
.../gridmix/TestGridmixMemoryEmulation.java | 2 +-
.../util/MapReduceJobPropertiesParser.java | 10 +++----
.../org/apache/hadoop/yarn/sls/SLSRunner.java | 11 +++++++-
.../sls/scheduler/FairSchedulerMetrics.java | 20 +++++++-------
.../apache/hadoop/yarn/sls/web/SLSWebApp.java | 4 +--
.../hadoop/yarn/sls/web/TestSLSWebApp.java | 28 ++++++++++----------
9 files changed, 58 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba99039/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 8f6dd4b..45c45b1 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -2451,8 +2451,6 @@ public class NativeAzureFileSystem extends FileSystem {
ArrayList<String> keysToCreateAsFolder = new ArrayList<String>();
- ArrayList<String> keysToUpdateAsFolder = new ArrayList<String>();
- boolean childCreated = false;
// Check that there is no file in the parent chain of the given path.
for (Path current = absolutePath, parent = current.getParent();
parent != null; // Stop when you get to the root
@@ -2464,14 +2462,6 @@ public class NativeAzureFileSystem extends FileSystem {
+ current + " is an existing file.");
} else if (currentMetadata == null) {
keysToCreateAsFolder.add(currentKey);
- childCreated = true;
- } else {
- // The directory already exists. Its last modified time need to be
- // updated if there is a child directory created under it.
- if (childCreated) {
- keysToUpdateAsFolder.add(currentKey);
- }
- childCreated = false;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba99039/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/InputStriper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/InputStriper.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/InputStriper.java
index a9d404d..6cdcc4e 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/InputStriper.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/InputStriper.java
@@ -131,9 +131,7 @@ class InputStriper {
static final Comparator<Entry<String,Double>> hostRank =
new Comparator<Entry<String,Double>>() {
public int compare(Entry<String,Double> a, Entry<String,Double> b) {
- final double va = a.getValue();
- final double vb = b.getValue();
- return va > vb ? -1 : va < vb ? 1 : 0;
- }
+ return Double.compare(a.getValue(), b.getValue());
+ }
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba99039/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java
index 4cfe1b5..bfbf516 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.mapred.gridmix.emulators.resourceusage;
import java.io.IOException;
import java.util.ArrayList;
+
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.gridmix.Progressive;
import org.apache.hadoop.tools.rumen.ResourceUsageMetrics;
@@ -129,7 +131,7 @@ implements ResourceUsageEmulatorPlugin {
public static class DefaultHeapUsageEmulator
implements HeapUsageEmulatorCore {
// store the unit loads in a list
- protected static final ArrayList<Object> heapSpace =
+ private static final ArrayList<Object> heapSpace =
new ArrayList<Object>();
/**
@@ -142,7 +144,17 @@ implements ResourceUsageEmulatorPlugin {
heapSpace.add((Object)new byte[ONE_MB]);
}
}
-
+
+ /**
+ * Gets the total number of 1mb objects stored in the emulator.
+ *
+ * @return total number of 1mb objects.
+ */
+ @VisibleForTesting
+ public int getHeapSpaceSize() {
+ return heapSpace.size();
+ }
+
/**
* This will initialize the core and check if the core can emulate the
* desired target on the underlying hardware.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba99039/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java
index 7ec3c93..d79c010 100644
--- a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java
+++ b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java
@@ -58,7 +58,7 @@ public class TestGridmixMemoryEmulation {
// Get the total number of 1mb objects stored within
long getHeapUsageInMB() {
- return heapSpace.size();
+ return getHeapSpaceSize();
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba99039/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/util/MapReduceJobPropertiesParser.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/util/MapReduceJobPropertiesParser.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/util/MapReduceJobPropertiesParser.java
index 7547eca..1c92caf 100644
--- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/util/MapReduceJobPropertiesParser.java
+++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/util/MapReduceJobPropertiesParser.java
@@ -130,7 +130,7 @@ public class MapReduceJobPropertiesParser implements JobPropertyParser {
/**
* Extracts the -Xmx heap option from the specified string.
*/
- public static void extractMaxHeapOpts(String javaOptions,
+ public static void extractMaxHeapOpts(final String javaOptions,
List<String> heapOpts,
List<String> others) {
for (String opt : javaOptions.split(" ")) {
@@ -160,6 +160,7 @@ public class MapReduceJobPropertiesParser implements JobPropertyParser {
// Maps the value of the specified key.
private DataType<?> fromString(String key, String value) {
+ DefaultDataType defaultValue = new DefaultDataType(value);
if (value != null) {
// check known configs
// job-name
@@ -190,14 +191,13 @@ public class MapReduceJobPropertiesParser implements JobPropertyParser {
// check if the config parameter represents a number
try {
format.parse(value);
- return new DefaultDataType(value);
+ return defaultValue;
} catch (ParseException pe) {}
// check if the config parameters represents a boolean
// avoiding exceptions
if ("true".equals(value) || "false".equals(value)) {
- Boolean.parseBoolean(value);
- return new DefaultDataType(value);
+ return defaultValue;
}
// check if the config parameter represents a class
@@ -208,7 +208,7 @@ public class MapReduceJobPropertiesParser implements JobPropertyParser {
// handle distributed cache sizes and timestamps
if (latestKey.endsWith("sizes")
|| latestKey.endsWith(".timestamps")) {
- new DefaultDataType(value);
+ return defaultValue;
}
// check if the config parameter represents a file-system path
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba99039/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index ddd35ef..03281a5 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -32,6 +32,7 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
+import java.util.Collections;
import java.util.concurrent.ConcurrentHashMap;
import com.fasterxml.jackson.core.JsonFactory;
@@ -113,7 +114,8 @@ public class SLSRunner extends Configured implements Tool {
// other simulation information
private int numNMs, numRacks, numAMs, numTasks;
private long maxRuntime;
- public final static Map<String, Object> simulateInfoMap =
+
+ private final static Map<String, Object> simulateInfoMap =
new HashMap<String, Object>();
// logger
@@ -165,6 +167,13 @@ public class SLSRunner extends Configured implements Tool {
}
}
+ /**
+ * @return an unmodifiable view of the simulated info map.
+ */
+ public static Map<String, Object> getSimulateInfoMap() {
+ return Collections.unmodifiableMap(simulateInfoMap);
+ }
+
public void setSimulationParams(TraceType inType, String[] inTraces,
String nodes, String outDir, Set<String> trackApps,
boolean printsimulation) throws IOException, ClassNotFoundException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba99039/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
index 08362b1..7b306f0 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
@@ -180,16 +180,16 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
new Gauge<Long>() {
@Override
public Long getValue() {
- if (! maxReset &&
- SLSRunner.simulateInfoMap.containsKey("Number of nodes") &&
- SLSRunner.simulateInfoMap.containsKey("Node memory (MB)") &&
- SLSRunner.simulateInfoMap.containsKey("Node VCores")) {
- int numNMs = Integer.parseInt(
- SLSRunner.simulateInfoMap.get("Number of nodes").toString());
- int numMemoryMB = Integer.parseInt(
- SLSRunner.simulateInfoMap.get("Node memory (MB)").toString());
- int numVCores = Integer.parseInt(
- SLSRunner.simulateInfoMap.get("Node VCores").toString());
+ if (! maxReset
+ && SLSRunner.getSimulateInfoMap().containsKey("Number of nodes")
+ && SLSRunner.getSimulateInfoMap().containsKey("Node memory (MB)")
+ && SLSRunner.getSimulateInfoMap().containsKey("Node VCores")) {
+ int numNMs = Integer.parseInt(SLSRunner.getSimulateInfoMap()
+ .get("Number of nodes").toString());
+ int numMemoryMB = Integer.parseInt(SLSRunner.getSimulateInfoMap()
+ .get("Node memory (MB)").toString());
+ int numVCores = Integer.parseInt(SLSRunner.getSimulateInfoMap()
+ .get("Node VCores").toString());
totalMemoryMB = numNMs * numMemoryMB;
totalVCores = numNMs * numVCores;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba99039/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java
index 29bbe1a..3daadca 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java
@@ -187,14 +187,14 @@ public class SLSWebApp extends HttpServlet {
response.setStatus(HttpServletResponse.SC_OK);
String simulateInfo;
- if (SLSRunner.simulateInfoMap.isEmpty()) {
+ if (SLSRunner.getSimulateInfoMap().isEmpty()) {
String empty = "<tr><td colspan='2' align='center'>" +
"No information available</td></tr>";
simulateInfo = MessageFormat.format(simulateInfoTemplate, empty);
} else {
StringBuilder info = new StringBuilder();
for (Map.Entry<String, Object> entry :
- SLSRunner.simulateInfoMap.entrySet()) {
+ SLSRunner.getSimulateInfoMap().entrySet()) {
info.append("<tr>");
info.append("<td class='td1'>").append(entry.getKey()).append("</td>");
info.append("<td class='td2'>").append(entry.getValue())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba99039/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/web/TestSLSWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/web/TestSLSWebApp.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/web/TestSLSWebApp.java
index 1c1e63c..c9be450 100644
--- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/web/TestSLSWebApp.java
+++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/web/TestSLSWebApp.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.sls.web;
import org.junit.Assert;
import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.yarn.sls.SLSRunner;
import org.junit.Test;
import java.io.File;
@@ -28,6 +27,7 @@ import java.text.MessageFormat;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import java.util.HashMap;
public class TestSLSWebApp {
@@ -36,20 +36,21 @@ public class TestSLSWebApp {
String simulateInfoTemplate = FileUtils.readFileToString(
new File("src/main/html/simulate.info.html.template"));
- SLSRunner.simulateInfoMap.put("Number of racks", 10);
- SLSRunner.simulateInfoMap.put("Number of nodes", 100);
- SLSRunner.simulateInfoMap.put("Node memory (MB)", 1024);
- SLSRunner.simulateInfoMap.put("Node VCores", 1);
- SLSRunner.simulateInfoMap.put("Number of applications", 100);
- SLSRunner.simulateInfoMap.put("Number of tasks", 1000);
- SLSRunner.simulateInfoMap.put("Average tasks per applicaion", 10);
- SLSRunner.simulateInfoMap.put("Number of queues", 4);
- SLSRunner.simulateInfoMap.put("Average applications per queue", 25);
- SLSRunner.simulateInfoMap.put("Estimated simulate time (s)", 10000);
+ Map<String, Object> simulateInfoMap = new HashMap<>();
+ simulateInfoMap.put("Number of racks", 10);
+ simulateInfoMap.put("Number of nodes", 100);
+ simulateInfoMap.put("Node memory (MB)", 1024);
+ simulateInfoMap.put("Node VCores", 1);
+ simulateInfoMap.put("Number of applications", 100);
+ simulateInfoMap.put("Number of tasks", 1000);
+ simulateInfoMap.put("Average tasks per applicaion", 10);
+ simulateInfoMap.put("Number of queues", 4);
+ simulateInfoMap.put("Average applications per queue", 25);
+ simulateInfoMap.put("Estimated simulate time (s)", 10000);
StringBuilder info = new StringBuilder();
for (Map.Entry<String, Object> entry :
- SLSRunner.simulateInfoMap.entrySet()) {
+ simulateInfoMap.entrySet()) {
info.append("<tr>");
info.append("<td class='td1'>" + entry.getKey() + "</td>");
info.append("<td class='td2'>" + entry.getValue() + "</td>");
@@ -60,8 +61,7 @@ public class TestSLSWebApp {
MessageFormat.format(simulateInfoTemplate, info.toString());
Assert.assertTrue("The simulate info html page should not be empty",
simulateInfo.length() > 0);
- for (Map.Entry<String, Object> entry :
- SLSRunner.simulateInfoMap.entrySet()) {
+ for (Map.Entry<String, Object> entry : simulateInfoMap.entrySet()) {
Assert.assertTrue("The simulate info html page should have information "
+ "of " + entry.getKey(), simulateInfo.contains("<td class='td1'>"
+ entry.getKey() + "</td><td class='td2'>"
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: HDFS-9807. Add an optional
StorageID to writes. Contributed by Ewan Higgs
Posted by in...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
index e7f0228..75baf84 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
@@ -81,10 +81,11 @@ class FsVolumeList {
return Collections.unmodifiableList(volumes);
}
- private FsVolumeReference chooseVolume(List<FsVolumeImpl> list, long blockSize)
- throws IOException {
+ private FsVolumeReference chooseVolume(List<FsVolumeImpl> list,
+ long blockSize, String storageId) throws IOException {
while (true) {
- FsVolumeImpl volume = blockChooser.chooseVolume(list, blockSize);
+ FsVolumeImpl volume = blockChooser.chooseVolume(list, blockSize,
+ storageId);
try {
return volume.obtainReference();
} catch (ClosedChannelException e) {
@@ -100,18 +101,20 @@ class FsVolumeList {
* Get next volume.
*
* @param blockSize free space needed on the volume
- * @param storageType the desired {@link StorageType}
+ * @param storageType the desired {@link StorageType}
+ * @param storageId the storage id which may or may not be used by
+ * the VolumeChoosingPolicy.
* @return next volume to store the block in.
*/
- FsVolumeReference getNextVolume(StorageType storageType, long blockSize)
- throws IOException {
+ FsVolumeReference getNextVolume(StorageType storageType, String storageId,
+ long blockSize) throws IOException {
final List<FsVolumeImpl> list = new ArrayList<>(volumes.size());
for(FsVolumeImpl v : volumes) {
if (v.getStorageType() == storageType) {
list.add(v);
}
}
- return chooseVolume(list, blockSize);
+ return chooseVolume(list, blockSize, storageId);
}
/**
@@ -129,7 +132,7 @@ class FsVolumeList {
list.add(v);
}
}
- return chooseVolume(list, blockSize);
+ return chooseVolume(list, blockSize, null);
}
long getDfsUsed() throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index 74cdeae..c98a336 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -1018,7 +1018,8 @@ public class DFSTestUtil {
// send the request
new Sender(out).transferBlock(b, new Token<BlockTokenIdentifier>(),
dfsClient.clientName, new DatanodeInfo[]{datanodes[1]},
- new StorageType[]{StorageType.DEFAULT});
+ new StorageType[]{StorageType.DEFAULT},
+ new String[0]);
out.flush();
return BlockOpResponseProto.parseDelimitedFrom(in);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
index b6884da..3a8fb59 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
@@ -1448,12 +1448,33 @@ public class TestBlockStoragePolicy {
testStorageTypeCheckAccessResult(new StorageType[]{StorageType.RAM_DISK,
StorageType.SSD, StorageType.ARCHIVE},
new StorageType[]{StorageType.DISK}, false);
+
+ testStorageTypeCheckAccessResult(
+ new StorageType[]{StorageType.DISK, StorageType.SSD},
+ new StorageType[]{StorageType.SSD},
+ true);
+
+ testStorageTypeCheckAccessResult(new StorageType[]{StorageType.RAM_DISK},
+ new StorageType[]{StorageType.DISK}, false);
+
+ testStorageTypeCheckAccessResult(
+ new StorageType[]{StorageType.RAM_DISK, StorageType.SSD,
+ StorageType.ARCHIVE},
+ new StorageType[]{StorageType.DISK},
+ false);
+
+ testStorageTypeCheckAccessResult(
+ new StorageType[]{StorageType.RAM_DISK, StorageType.SSD,
+ StorageType.ARCHIVE},
+ new StorageType[]{StorageType.DISK},
+ false);
+
}
private void testStorageTypeCheckAccessResult(StorageType[] requested,
StorageType[] allowed, boolean expAccess) {
try {
- BlockTokenSecretManager.checkAccess(requested, allowed);
+ BlockTokenSecretManager.checkAccess(requested, allowed, "StorageTypes");
if (!expAccess) {
fail("No expected access with allowed StorageTypes "
+ Arrays.toString(allowed) + " and requested StorageTypes "
@@ -1467,4 +1488,56 @@ public class TestBlockStoragePolicy {
}
}
}
+
+ @Test
+ public void testStorageIDCheckAccess() {
+ testStorageIDCheckAccessResult(
+ new String[]{"DN1-Storage1"},
+ new String[]{"DN1-Storage1"}, true);
+
+ testStorageIDCheckAccessResult(new String[]{"DN1-Storage1", "DN2-Storage1"},
+ new String[]{"DN1-Storage1"},
+ true);
+
+ testStorageIDCheckAccessResult(new String[]{"DN1-Storage1", "DN2-Storage1"},
+ new String[]{"DN1-Storage1", "DN1-Storage2"}, false);
+
+ testStorageIDCheckAccessResult(
+ new String[]{"DN1-Storage1", "DN1-Storage2"},
+ new String[]{"DN1-Storage1"}, true);
+
+ testStorageIDCheckAccessResult(
+ new String[]{"DN1-Storage1", "DN1-Storage2"},
+ new String[]{"DN2-Storage1"}, false);
+
+ testStorageIDCheckAccessResult(
+ new String[]{"DN1-Storage2", "DN2-Storage2"},
+ new String[]{"DN1-Storage1", "DN2-Storage1"}, false);
+
+ testStorageIDCheckAccessResult(new String[0], new String[0], false);
+
+ testStorageIDCheckAccessResult(new String[0], new String[]{"DN1-Storage1"},
+ true);
+
+ testStorageIDCheckAccessResult(new String[]{"DN1-Storage1"}, new String[0],
+ false);
+ }
+
+ private void testStorageIDCheckAccessResult(String[] requested,
+ String[] allowed, boolean expAccess) {
+ try {
+ BlockTokenSecretManager.checkAccess(requested, allowed, "StorageIDs");
+ if (!expAccess) {
+ fail("No expected access with allowed StorageIDs"
+ + Arrays.toString(allowed) + " and requested StorageIDs"
+ + Arrays.toString(requested));
+ }
+ } catch (SecretManager.InvalidToken e) {
+ if (expAccess) {
+ fail("Expected access with allowed StorageIDs "
+ + Arrays.toString(allowed) + " and requested StorageIDs"
+ + Arrays.toString(requested));
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
index 3f4fe28..7a2ac1b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
@@ -559,6 +559,7 @@ public class TestDataTransferProtocol {
BlockTokenSecretManager.DUMMY_TOKEN, "cl",
new DatanodeInfo[1], new StorageType[1], null, stage,
0, block.getNumBytes(), block.getNumBytes(), newGS,
- checksum, CachingStrategy.newDefaultStrategy(), false, false, null);
+ checksum, CachingStrategy.newDefaultStrategy(), false, false,
+ null, null, new String[0]);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteBlockGetsBlockLengthHint.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteBlockGetsBlockLengthHint.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteBlockGetsBlockLengthHint.java
index 5c1b38f..e159914 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteBlockGetsBlockLengthHint.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteBlockGetsBlockLengthHint.java
@@ -98,11 +98,11 @@ public class TestWriteBlockGetsBlockLengthHint {
* correctly propagate the hint to FsDatasetSpi.
*/
@Override
- public synchronized ReplicaHandler createRbw(
- StorageType storageType, ExtendedBlock b, boolean allowLazyPersist)
+ public synchronized ReplicaHandler createRbw(StorageType storageType,
+ String storageId, ExtendedBlock b, boolean allowLazyPersist)
throws IOException {
assertThat(b.getLocalBlock().getNumBytes(), is(EXPECTED_BLOCK_LENGTH));
- return super.createRbw(storageType, b, allowLazyPersist);
+ return super.createRbw(storageType, storageId, b, allowLazyPersist);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
index e98207f..747f295 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
@@ -151,7 +151,7 @@ public class TestBlockToken {
assertTrue("Received BlockTokenIdentifier is wrong", ident.equals(id));
sm.checkAccess(id, null, PBHelperClient.convert(req.getBlock()),
BlockTokenIdentifier.AccessMode.WRITE,
- new StorageType[]{StorageType.DEFAULT});
+ new StorageType[]{StorageType.DEFAULT}, null);
result = id.getBlockId();
}
return GetReplicaVisibleLengthResponseProto.newBuilder()
@@ -160,11 +160,11 @@ public class TestBlockToken {
}
private BlockTokenIdentifier generateTokenId(BlockTokenSecretManager sm,
- ExtendedBlock block,
- EnumSet<BlockTokenIdentifier.AccessMode> accessModes,
- StorageType... storageTypes) throws IOException {
+ ExtendedBlock block, EnumSet<BlockTokenIdentifier.AccessMode> accessModes,
+ StorageType[] storageTypes, String[] storageIds)
+ throws IOException {
Token<BlockTokenIdentifier> token = sm.generateToken(block, accessModes,
- storageTypes);
+ storageTypes, storageIds);
BlockTokenIdentifier id = sm.createIdentifier();
id.readFields(new DataInputStream(new ByteArrayInputStream(token
.getIdentifier())));
@@ -178,29 +178,28 @@ public class TestBlockToken {
enableProtobuf);
TestWritable.testWritable(generateTokenId(sm, block3,
EnumSet.noneOf(BlockTokenIdentifier.AccessMode.class),
- StorageType.DEFAULT));
+ new StorageType[]{StorageType.DEFAULT}, null));
TestWritable.testWritable(generateTokenId(sm, block3,
EnumSet.of(BlockTokenIdentifier.AccessMode.WRITE),
- StorageType.DEFAULT));
+ new StorageType[]{StorageType.DEFAULT}, null));
TestWritable.testWritable(generateTokenId(sm, block3,
EnumSet.allOf(BlockTokenIdentifier.AccessMode.class),
- StorageType.DEFAULT));
+ new StorageType[]{StorageType.DEFAULT}, null));
TestWritable.testWritable(generateTokenId(sm, block1,
EnumSet.allOf(BlockTokenIdentifier.AccessMode.class),
- StorageType.DEFAULT));
+ new StorageType[]{StorageType.DEFAULT}, null));
TestWritable.testWritable(generateTokenId(sm, block2,
EnumSet.of(BlockTokenIdentifier.AccessMode.WRITE),
- StorageType.DEFAULT));
+ new StorageType[]{StorageType.DEFAULT}, null));
TestWritable.testWritable(generateTokenId(sm, block3,
EnumSet.noneOf(BlockTokenIdentifier.AccessMode.class),
- StorageType.DEFAULT));
+ new StorageType[]{StorageType.DEFAULT}, null));
// We must be backwards compatible when adding storageType
TestWritable.testWritable(generateTokenId(sm, block3,
- EnumSet.noneOf(BlockTokenIdentifier.AccessMode.class),
- (StorageType[]) null));
+ EnumSet.noneOf(BlockTokenIdentifier.AccessMode.class), null, null));
TestWritable.testWritable(generateTokenId(sm, block3,
EnumSet.noneOf(BlockTokenIdentifier.AccessMode.class),
- StorageType.EMPTY_ARRAY));
+ StorageType.EMPTY_ARRAY, null));
}
@Test
@@ -215,35 +214,36 @@ public class TestBlockToken {
private static void checkAccess(BlockTokenSecretManager m,
Token<BlockTokenIdentifier> t, ExtendedBlock blk,
- BlockTokenIdentifier.AccessMode mode) throws SecretManager.InvalidToken {
- m.checkAccess(t, null, blk, mode, new StorageType[]{ StorageType.DEFAULT });
+ BlockTokenIdentifier.AccessMode mode, StorageType[] storageTypes,
+ String[] storageIds) throws SecretManager.InvalidToken {
+ m.checkAccess(t, null, blk, mode, storageTypes, storageIds);
}
private void tokenGenerationAndVerification(BlockTokenSecretManager master,
- BlockTokenSecretManager slave, StorageType... storageTypes)
- throws Exception {
+ BlockTokenSecretManager slave, StorageType[] storageTypes,
+ String[] storageIds) throws Exception {
// single-mode tokens
for (BlockTokenIdentifier.AccessMode mode : BlockTokenIdentifier.AccessMode
.values()) {
// generated by master
Token<BlockTokenIdentifier> token1 = master.generateToken(block1,
- EnumSet.of(mode), storageTypes);
- checkAccess(master, token1, block1, mode);
- checkAccess(slave, token1, block1, mode);
+ EnumSet.of(mode), storageTypes, storageIds);
+ checkAccess(master, token1, block1, mode, storageTypes, storageIds);
+ checkAccess(slave, token1, block1, mode, storageTypes, storageIds);
// generated by slave
Token<BlockTokenIdentifier> token2 = slave.generateToken(block2,
- EnumSet.of(mode), storageTypes);
- checkAccess(master, token2, block2, mode);
- checkAccess(slave, token2, block2, mode);
+ EnumSet.of(mode), storageTypes, storageIds);
+ checkAccess(master, token2, block2, mode, storageTypes, storageIds);
+ checkAccess(slave, token2, block2, mode, storageTypes, storageIds);
}
// multi-mode tokens
Token<BlockTokenIdentifier> mtoken = master.generateToken(block3,
EnumSet.allOf(BlockTokenIdentifier.AccessMode.class),
- storageTypes);
+ storageTypes, storageIds);
for (BlockTokenIdentifier.AccessMode mode : BlockTokenIdentifier.AccessMode
.values()) {
- checkAccess(master, mtoken, block3, mode);
- checkAccess(slave, mtoken, block3, mode);
+ checkAccess(master, mtoken, block3, mode, storageTypes, storageIds);
+ checkAccess(slave, mtoken, block3, mode, storageTypes, storageIds);
}
}
@@ -259,18 +259,18 @@ public class TestBlockToken {
ExportedBlockKeys keys = masterHandler.exportKeys();
slaveHandler.addKeys(keys);
tokenGenerationAndVerification(masterHandler, slaveHandler,
- StorageType.DEFAULT);
- tokenGenerationAndVerification(masterHandler, slaveHandler, null);
+ new StorageType[]{StorageType.DEFAULT}, null);
+ tokenGenerationAndVerification(masterHandler, slaveHandler, null, null);
// key updating
masterHandler.updateKeys();
tokenGenerationAndVerification(masterHandler, slaveHandler,
- StorageType.DEFAULT);
- tokenGenerationAndVerification(masterHandler, slaveHandler, null);
+ new StorageType[]{StorageType.DEFAULT}, null);
+ tokenGenerationAndVerification(masterHandler, slaveHandler, null, null);
keys = masterHandler.exportKeys();
slaveHandler.addKeys(keys);
tokenGenerationAndVerification(masterHandler, slaveHandler,
- StorageType.DEFAULT);
- tokenGenerationAndVerification(masterHandler, slaveHandler, null);
+ new StorageType[]{StorageType.DEFAULT}, null);
+ tokenGenerationAndVerification(masterHandler, slaveHandler, null, null);
}
@Test
@@ -315,7 +315,7 @@ public class TestBlockToken {
enableProtobuf);
Token<BlockTokenIdentifier> token = sm.generateToken(block3,
EnumSet.allOf(BlockTokenIdentifier.AccessMode.class),
- new StorageType[]{StorageType.DEFAULT});
+ new StorageType[]{StorageType.DEFAULT}, new String[0]);
final Server server = createMockDatanode(sm, token, conf);
@@ -365,7 +365,7 @@ public class TestBlockToken {
enableProtobuf);
Token<BlockTokenIdentifier> token = sm.generateToken(block3,
EnumSet.allOf(BlockTokenIdentifier.AccessMode.class),
- new StorageType[]{StorageType.DEFAULT});
+ new StorageType[]{StorageType.DEFAULT}, new String[0]);
final Server server = createMockDatanode(sm, token, conf);
server.start();
@@ -451,19 +451,23 @@ public class TestBlockToken {
ExportedBlockKeys keys = masterHandler.exportKeys();
bpMgr.addKeys(bpid, keys);
+ String[] storageIds = new String[] {"DS-9001"};
tokenGenerationAndVerification(masterHandler, bpMgr.get(bpid),
- StorageType.DEFAULT);
- tokenGenerationAndVerification(masterHandler, bpMgr.get(bpid), null);
+ new StorageType[]{StorageType.DEFAULT}, storageIds);
+ tokenGenerationAndVerification(masterHandler, bpMgr.get(bpid), null,
+ null);
// Test key updating
masterHandler.updateKeys();
tokenGenerationAndVerification(masterHandler, bpMgr.get(bpid),
- StorageType.DEFAULT);
- tokenGenerationAndVerification(masterHandler, bpMgr.get(bpid), null);
+ new StorageType[]{StorageType.DEFAULT}, storageIds);
+ tokenGenerationAndVerification(masterHandler, bpMgr.get(bpid), null,
+ null);
keys = masterHandler.exportKeys();
bpMgr.addKeys(bpid, keys);
tokenGenerationAndVerification(masterHandler, bpMgr.get(bpid),
- StorageType.DEFAULT);
- tokenGenerationAndVerification(masterHandler, bpMgr.get(bpid), null);
+ new StorageType[]{StorageType.DEFAULT}, new String[]{"DS-9001"});
+ tokenGenerationAndVerification(masterHandler, bpMgr.get(bpid), null,
+ null);
}
}
@@ -540,7 +544,7 @@ public class TestBlockToken {
useProto);
Token<BlockTokenIdentifier> token = sm.generateToken(block1,
EnumSet.noneOf(BlockTokenIdentifier.AccessMode.class),
- new StorageType[]{StorageType.DEFAULT});
+ new StorageType[]{StorageType.DEFAULT}, new String[0]);
final byte[] tokenBytes = token.getIdentifier();
BlockTokenIdentifier legacyToken = new BlockTokenIdentifier();
BlockTokenIdentifier protobufToken = new BlockTokenIdentifier();
@@ -605,7 +609,7 @@ public class TestBlockToken {
useProto);
Token<BlockTokenIdentifier> token = sm.generateToken(block1,
EnumSet.noneOf(BlockTokenIdentifier.AccessMode.class),
- StorageType.EMPTY_ARRAY);
+ StorageType.EMPTY_ARRAY, new String[0]);
final byte[] tokenBytes = token.getIdentifier();
BlockTokenIdentifier legacyToken = new BlockTokenIdentifier();
BlockTokenIdentifier protobufToken = new BlockTokenIdentifier();
@@ -699,7 +703,8 @@ public class TestBlockToken {
*/
BlockTokenIdentifier identifier = new BlockTokenIdentifier("user",
"blockpool", 123, EnumSet.allOf(BlockTokenIdentifier.AccessMode.class),
- new StorageType[]{StorageType.DISK, StorageType.ARCHIVE}, true);
+ new StorageType[]{StorageType.DISK, StorageType.ARCHIVE},
+ new String[] {"fake-storage-id"}, true);
Calendar cal = new GregorianCalendar();
cal.set(2017, 1, 9, 0, 12, 35);
long datetime = cal.getTimeInMillis();
@@ -749,7 +754,8 @@ public class TestBlockToken {
new StorageType[]{StorageType.RAM_DISK, StorageType.SSD,
StorageType.DISK, StorageType.ARCHIVE};
BlockTokenIdentifier ident = new BlockTokenIdentifier("user", "bpool",
- 123, accessModes, storageTypes, useProto);
+ 123, accessModes, storageTypes, new String[] {"fake-storage-id"},
+ useProto);
ident.setExpiryDate(1487080345L);
BlockTokenIdentifier ret = writeAndReadBlockToken(ident);
assertEquals(ret.getExpiryDate(), 1487080345L);
@@ -760,6 +766,7 @@ public class TestBlockToken {
assertEquals(ret.getAccessModes(),
EnumSet.allOf(BlockTokenIdentifier.AccessMode.class));
assertArrayEquals(ret.getStorageTypes(), storageTypes);
+ assertArrayEquals(ret.getStorageIds(), new String[] {"fake-storage-id"});
}
@Test
@@ -767,5 +774,4 @@ public class TestBlockToken {
testBlockTokenSerialization(false);
testBlockTokenSerialization(true);
}
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java
index 6810a0b..c9ff572 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java
@@ -389,7 +389,7 @@ public abstract class BlockReportTestBase {
// Create a bogus new block which will not be present on the namenode.
ExtendedBlock b = new ExtendedBlock(
poolId, rand.nextLong(), 1024L, rand.nextLong());
- dn.getFSDataset().createRbw(StorageType.DEFAULT, b, false);
+ dn.getFSDataset().createRbw(StorageType.DEFAULT, null, b, false);
DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId);
StorageBlockReport[] reports = getBlockReports(dn, poolId, false, false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index cd3befd..18b4922 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -1023,21 +1023,22 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
@Override // FsDatasetSpi
public synchronized ReplicaHandler createRbw(
- StorageType storageType, ExtendedBlock b,
+ StorageType storageType, String storageId, ExtendedBlock b,
boolean allowLazyPersist) throws IOException {
- return createTemporary(storageType, b);
+ return createTemporary(storageType, storageId, b);
}
@Override // FsDatasetSpi
public synchronized ReplicaHandler createTemporary(
- StorageType storageType, ExtendedBlock b) throws IOException {
+ StorageType storageType, String storageId, ExtendedBlock b)
+ throws IOException {
if (isValidBlock(b)) {
- throw new ReplicaAlreadyExistsException("Block " + b +
- " is valid, and cannot be written to.");
- }
+ throw new ReplicaAlreadyExistsException("Block " + b +
+ " is valid, and cannot be written to.");
+ }
if (isValidRbw(b)) {
- throw new ReplicaAlreadyExistsException("Block " + b +
- " is being written, and cannot be written to.");
+ throw new ReplicaAlreadyExistsException("Block " + b +
+ " is being written, and cannot be written to.");
}
final Map<Block, BInfo> map = getMap(b.getBlockPoolId());
BInfo binfo = new BInfo(b.getBlockPoolId(), b.getLocalBlock(), true);
@@ -1419,7 +1420,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
@Override
public ReplicaInfo moveBlockAcrossStorage(ExtendedBlock block,
- StorageType targetStorageType) throws IOException {
+ StorageType targetStorageType, String storageId) throws IOException {
// TODO Auto-generated method stub
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
index 579252b..311d5a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
@@ -647,7 +647,7 @@ public class TestBlockRecovery {
if(LOG.isDebugEnabled()) {
LOG.debug("Running " + GenericTestUtils.getMethodName());
}
- dn.data.createRbw(StorageType.DEFAULT, block, false);
+ dn.data.createRbw(StorageType.DEFAULT, null, block, false);
BlockRecoveryWorker.RecoveryTaskContiguous RecoveryTaskContiguous =
recoveryWorker.new RecoveryTaskContiguous(rBlock);
try {
@@ -673,7 +673,7 @@ public class TestBlockRecovery {
LOG.debug("Running " + GenericTestUtils.getMethodName());
}
ReplicaInPipeline replicaInfo = dn.data.createRbw(
- StorageType.DEFAULT, block, false).getReplica();
+ StorageType.DEFAULT, null, block, false).getReplica();
ReplicaOutputStreams streams = null;
try {
streams = replicaInfo.createStreams(true,
@@ -972,7 +972,7 @@ public class TestBlockRecovery {
// Register this thread as the writer for the recoveringBlock.
LOG.debug("slowWriter creating rbw");
ReplicaHandler replicaHandler =
- spyDN.data.createRbw(StorageType.DISK, block, false);
+ spyDN.data.createRbw(StorageType.DISK, null, block, false);
replicaHandler.close();
LOG.debug("slowWriter created rbw");
// Tell the parent thread to start progressing.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
index f811bd8..8992d47 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
@@ -394,7 +394,7 @@ public class TestBlockReplacement {
DataOutputStream out = new DataOutputStream(sock.getOutputStream());
new Sender(out).replaceBlock(block, targetStorageType,
BlockTokenSecretManager.DUMMY_TOKEN, source.getDatanodeUuid(),
- sourceProxy);
+ sourceProxy, null);
out.flush();
// receiveResponse
DataInputStream reply = new DataInputStream(sock.getInputStream());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataXceiverLazyPersistHint.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataXceiverLazyPersistHint.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataXceiverLazyPersistHint.java
index b2bfe49..8fda664 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataXceiverLazyPersistHint.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataXceiverLazyPersistHint.java
@@ -129,7 +129,7 @@ public class TestDataXceiverLazyPersistHint {
DataChecksum.newDataChecksum(DataChecksum.Type.NULL, 0),
CachingStrategy.newDefaultStrategy(),
lazyPersist,
- false, null);
+ false, null, null, new String[0]);
}
// Helper functions to setup the mock objects.
@@ -151,7 +151,7 @@ public class TestDataXceiverLazyPersistHint {
any(BlockConstructionStage.class), anyLong(), anyLong(), anyLong(),
anyString(), any(DatanodeInfo.class), any(DataNode.class),
any(DataChecksum.class), any(CachingStrategy.class),
- captor.capture(), anyBoolean());
+ captor.capture(), anyBoolean(), any(String.class));
doReturn(mock(DataOutputStream.class)).when(xceiverSpy)
.getBufferedOutputStream();
return xceiverSpy;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java
index cd86720..38e4287 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java
@@ -167,7 +167,8 @@ public class TestDiskError {
BlockTokenSecretManager.DUMMY_TOKEN, "",
new DatanodeInfo[0], new StorageType[0], null,
BlockConstructionStage.PIPELINE_SETUP_CREATE, 1, 0L, 0L, 0L,
- checksum, CachingStrategy.newDefaultStrategy(), false, false, null);
+ checksum, CachingStrategy.newDefaultStrategy(), false, false,
+ null, null, new String[0]);
out.flush();
// close the connection before sending the content of the block
@@ -274,7 +275,7 @@ public class TestDiskError {
dn1.getDatanodeId());
dn0.transferBlock(block, new DatanodeInfo[]{dnd1},
- new StorageType[]{StorageType.DISK});
+ new StorageType[]{StorageType.DISK}, new String[0]);
// Sleep for 1 second so the DataTrasnfer daemon can start transfer.
try {
Thread.sleep(1000);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
index 4e724bc7..2e69595 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
@@ -81,7 +81,7 @@ public class TestSimulatedFSDataset {
// we pass expected len as zero, - fsdataset should use the sizeof actual
// data written
ReplicaInPipeline bInfo = fsdataset.createRbw(
- StorageType.DEFAULT, b, false).getReplica();
+ StorageType.DEFAULT, null, b, false).getReplica();
ReplicaOutputStreams out = bInfo.createStreams(true,
DataChecksum.newDataChecksum(DataChecksum.Type.CRC32, 512));
try {
@@ -368,7 +368,7 @@ public class TestSimulatedFSDataset {
ExtendedBlock block = new ExtendedBlock(newbpid,1);
try {
// it will throw an exception if the block pool is not found
- fsdataset.createTemporary(StorageType.DEFAULT, block);
+ fsdataset.createTemporary(StorageType.DEFAULT, null, block);
} catch (IOException ioe) {
// JUnit does not capture exception in non-main thread,
// so cache it and then let main thread throw later.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
index 62ef731..2e439d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
@@ -138,14 +138,15 @@ public class ExternalDatasetImpl implements FsDatasetSpi<ExternalVolumeImpl> {
}
@Override
- public ReplicaHandler createTemporary(StorageType t, ExtendedBlock b)
+ public ReplicaHandler createTemporary(StorageType t, String i,
+ ExtendedBlock b)
throws IOException {
return new ReplicaHandler(new ExternalReplicaInPipeline(), null);
}
@Override
- public ReplicaHandler createRbw(StorageType t, ExtendedBlock b, boolean tf)
- throws IOException {
+ public ReplicaHandler createRbw(StorageType storageType, String id,
+ ExtendedBlock b, boolean tf) throws IOException {
return new ReplicaHandler(new ExternalReplicaInPipeline(), null);
}
@@ -332,7 +333,8 @@ public class ExternalDatasetImpl implements FsDatasetSpi<ExternalVolumeImpl> {
}
@Override
- public ReplicaInfo moveBlockAcrossStorage(ExtendedBlock block, StorageType targetStorageType) throws IOException {
+ public ReplicaInfo moveBlockAcrossStorage(ExtendedBlock block,
+ StorageType targetStorageType, String storageId) throws IOException {
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestAvailableSpaceVolumeChoosingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestAvailableSpaceVolumeChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestAvailableSpaceVolumeChoosingPolicy.java
index 9414a0e..24a43e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestAvailableSpaceVolumeChoosingPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestAvailableSpaceVolumeChoosingPolicy.java
@@ -89,10 +89,12 @@ public class TestAvailableSpaceVolumeChoosingPolicy {
// than the threshold of 1MB.
volumes.add(Mockito.mock(FsVolumeSpi.class));
Mockito.when(volumes.get(1).getAvailable()).thenReturn(1024L * 1024L * 3);
-
- Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes, 100));
- Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes, 100));
- Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes, 100));
+ Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes, 100,
+ null));
+ Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes, 100,
+ null));
+ Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes, 100,
+ null));
}
@Test(timeout=60000)
@@ -115,21 +117,29 @@ public class TestAvailableSpaceVolumeChoosingPolicy {
// Third volume, again with 3MB free space.
volumes.add(Mockito.mock(FsVolumeSpi.class));
Mockito.when(volumes.get(2).getAvailable()).thenReturn(1024L * 1024L * 3);
-
+
// We should alternate assigning between the two volumes with a lot of free
// space.
initPolicy(policy, 1.0f);
- Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes, 100));
- Assert.assertEquals(volumes.get(2), policy.chooseVolume(volumes, 100));
- Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes, 100));
- Assert.assertEquals(volumes.get(2), policy.chooseVolume(volumes, 100));
+ Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes, 100,
+ null));
+ Assert.assertEquals(volumes.get(2), policy.chooseVolume(volumes, 100,
+ null));
+ Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes, 100,
+ null));
+ Assert.assertEquals(volumes.get(2), policy.chooseVolume(volumes, 100,
+ null));
// All writes should be assigned to the volume with the least free space.
initPolicy(policy, 0.0f);
- Assert.assertEquals(volumes.get(0), policy.chooseVolume(volumes, 100));
- Assert.assertEquals(volumes.get(0), policy.chooseVolume(volumes, 100));
- Assert.assertEquals(volumes.get(0), policy.chooseVolume(volumes, 100));
- Assert.assertEquals(volumes.get(0), policy.chooseVolume(volumes, 100));
+ Assert.assertEquals(volumes.get(0), policy.chooseVolume(volumes, 100,
+ null));
+ Assert.assertEquals(volumes.get(0), policy.chooseVolume(volumes, 100,
+ null));
+ Assert.assertEquals(volumes.get(0), policy.chooseVolume(volumes, 100,
+ null));
+ Assert.assertEquals(volumes.get(0), policy.chooseVolume(volumes, 100,
+ null));
}
@Test(timeout=60000)
@@ -156,22 +166,30 @@ public class TestAvailableSpaceVolumeChoosingPolicy {
// Fourth volume, again with 3MB free space.
volumes.add(Mockito.mock(FsVolumeSpi.class));
Mockito.when(volumes.get(3).getAvailable()).thenReturn(1024L * 1024L * 3);
-
+
// We should alternate assigning between the two volumes with a lot of free
// space.
initPolicy(policy, 1.0f);
- Assert.assertEquals(volumes.get(2), policy.chooseVolume(volumes, 100));
- Assert.assertEquals(volumes.get(3), policy.chooseVolume(volumes, 100));
- Assert.assertEquals(volumes.get(2), policy.chooseVolume(volumes, 100));
- Assert.assertEquals(volumes.get(3), policy.chooseVolume(volumes, 100));
+ Assert.assertEquals(volumes.get(2), policy.chooseVolume(volumes, 100,
+ null));
+ Assert.assertEquals(volumes.get(3), policy.chooseVolume(volumes, 100,
+ null));
+ Assert.assertEquals(volumes.get(2), policy.chooseVolume(volumes, 100,
+ null));
+ Assert.assertEquals(volumes.get(3), policy.chooseVolume(volumes, 100,
+ null));
// We should alternate assigning between the two volumes with less free
// space.
initPolicy(policy, 0.0f);
- Assert.assertEquals(volumes.get(0), policy.chooseVolume(volumes, 100));
- Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes, 100));
- Assert.assertEquals(volumes.get(0), policy.chooseVolume(volumes, 100));
- Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes, 100));
+ Assert.assertEquals(volumes.get(0), policy.chooseVolume(volumes, 100,
+ null));
+ Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes, 100,
+ null));
+ Assert.assertEquals(volumes.get(0), policy.chooseVolume(volumes, 100,
+ null));
+ Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes, 100,
+ null));
}
@Test(timeout=60000)
@@ -190,13 +208,14 @@ public class TestAvailableSpaceVolumeChoosingPolicy {
// than the threshold of 1MB.
volumes.add(Mockito.mock(FsVolumeSpi.class));
Mockito.when(volumes.get(1).getAvailable()).thenReturn(1024L * 1024L * 3);
-
+
// All writes should be assigned to the volume with the least free space.
// However, if the volume with the least free space doesn't have enough
// space to accept the replica size, and another volume does have enough
// free space, that should be chosen instead.
initPolicy(policy, 0.0f);
- Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes, 1024L * 1024L * 2));
+ Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes,
+ 1024L * 1024L * 2, null));
}
@Test(timeout=60000)
@@ -220,10 +239,11 @@ public class TestAvailableSpaceVolumeChoosingPolicy {
.thenReturn(1024L * 1024L * 3)
.thenReturn(1024L * 1024L * 3)
.thenReturn(1024L * 1024L * 1); // After the third check, return 1MB.
-
+
// Should still be able to get a volume for the replica even though the
// available space on the second volume changed.
- Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes, 100));
+ Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes,
+ 100, null));
}
@Test(timeout=60000)
@@ -271,12 +291,12 @@ public class TestAvailableSpaceVolumeChoosingPolicy {
Mockito.when(volume.getAvailable()).thenReturn(1024L * 1024L * 3);
volumes.add(volume);
}
-
+
initPolicy(policy, preferencePercent);
long lowAvailableSpaceVolumeSelected = 0;
long highAvailableSpaceVolumeSelected = 0;
for (int i = 0; i < RANDOMIZED_ITERATIONS; i++) {
- FsVolumeSpi volume = policy.chooseVolume(volumes, 100);
+ FsVolumeSpi volume = policy.chooseVolume(volumes, 100, null);
for (int j = 0; j < volumes.size(); j++) {
// Note how many times the first low available volume was selected
if (volume == volumes.get(j) && j == 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestRoundRobinVolumeChoosingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestRoundRobinVolumeChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestRoundRobinVolumeChoosingPolicy.java
index 9b3047f..44e2a30 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestRoundRobinVolumeChoosingPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestRoundRobinVolumeChoosingPolicy.java
@@ -50,20 +50,21 @@ public class TestRoundRobinVolumeChoosingPolicy {
// Second volume, with 200 bytes of space.
volumes.add(Mockito.mock(FsVolumeSpi.class));
Mockito.when(volumes.get(1).getAvailable()).thenReturn(200L);
-
+
// Test two rounds of round-robin choosing
- Assert.assertEquals(volumes.get(0), policy.chooseVolume(volumes, 0));
- Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes, 0));
- Assert.assertEquals(volumes.get(0), policy.chooseVolume(volumes, 0));
- Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes, 0));
+ Assert.assertEquals(volumes.get(0), policy.chooseVolume(volumes, 0, null));
+ Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes, 0, null));
+ Assert.assertEquals(volumes.get(0), policy.chooseVolume(volumes, 0, null));
+ Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes, 0, null));
// The first volume has only 100L space, so the policy should
// wisely choose the second one in case we ask for more.
- Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes, 150));
+ Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes, 150,
+ null));
// Fail if no volume can be chosen?
try {
- policy.chooseVolume(volumes, Long.MAX_VALUE);
+ policy.chooseVolume(volumes, Long.MAX_VALUE, null);
Assert.fail();
} catch (IOException e) {
// Passed.
@@ -93,7 +94,7 @@ public class TestRoundRobinVolumeChoosingPolicy {
int blockSize = 700;
try {
- policy.chooseVolume(volumes, blockSize);
+ policy.chooseVolume(volumes, blockSize, null);
Assert.fail("expected to throw DiskOutOfSpaceException");
} catch(DiskOutOfSpaceException e) {
Assert.assertEquals("Not returnig the expected message",
@@ -137,21 +138,21 @@ public class TestRoundRobinVolumeChoosingPolicy {
Mockito.when(ssdVolumes.get(1).getAvailable()).thenReturn(100L);
Assert.assertEquals(diskVolumes.get(0),
- policy.chooseVolume(diskVolumes, 0));
+ policy.chooseVolume(diskVolumes, 0, null));
// Independent Round-Robin for different storage type
Assert.assertEquals(ssdVolumes.get(0),
- policy.chooseVolume(ssdVolumes, 0));
+ policy.chooseVolume(ssdVolumes, 0, null));
// Take block size into consideration
Assert.assertEquals(ssdVolumes.get(0),
- policy.chooseVolume(ssdVolumes, 150L));
+ policy.chooseVolume(ssdVolumes, 150L, null));
Assert.assertEquals(diskVolumes.get(1),
- policy.chooseVolume(diskVolumes, 0));
+ policy.chooseVolume(diskVolumes, 0, null));
Assert.assertEquals(diskVolumes.get(0),
- policy.chooseVolume(diskVolumes, 50L));
+ policy.chooseVolume(diskVolumes, 50L, null));
try {
- policy.chooseVolume(diskVolumes, 200L);
+ policy.chooseVolume(diskVolumes, 200L, null);
Assert.fail("Should throw an DiskOutOfSpaceException before this!");
} catch (DiskOutOfSpaceException e) {
// Pass.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
index 905c3f0..3293561 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
@@ -259,7 +259,7 @@ public class TestFsDatasetImpl {
String bpid = BLOCK_POOL_IDS[NUM_BLOCKS % BLOCK_POOL_IDS.length];
ExtendedBlock eb = new ExtendedBlock(bpid, i);
try (ReplicaHandler replica =
- dataset.createRbw(StorageType.DEFAULT, eb, false)) {
+ dataset.createRbw(StorageType.DEFAULT, null, eb, false)) {
}
}
final String[] dataDirs =
@@ -566,7 +566,7 @@ public class TestFsDatasetImpl {
class ResponderThread extends Thread {
public void run() {
try (ReplicaHandler replica = dataset
- .createRbw(StorageType.DEFAULT, eb, false)) {
+ .createRbw(StorageType.DEFAULT, null, eb, false)) {
LOG.info("CreateRbw finished");
startFinalizeLatch.countDown();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
index 83c15ca..ee3a79f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
@@ -101,7 +101,7 @@ public class TestFsVolumeList {
}
for (int i = 0; i < 10; i++) {
try (FsVolumeReference ref =
- volumeList.getNextVolume(StorageType.DEFAULT, 128)) {
+ volumeList.getNextVolume(StorageType.DEFAULT, null, 128)) {
// volume No.2 will not be chosen.
assertNotEquals(ref.getVolume(), volumes.get(1));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
index da53cae..11525ed 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
@@ -353,7 +353,7 @@ public class TestWriteToReplica {
}
try {
- dataSet.createRbw(StorageType.DEFAULT, blocks[FINALIZED], false);
+ dataSet.createRbw(StorageType.DEFAULT, null, blocks[FINALIZED], false);
Assert.fail("Should not have created a replica that's already " +
"finalized " + blocks[FINALIZED]);
} catch (ReplicaAlreadyExistsException e) {
@@ -371,7 +371,7 @@ public class TestWriteToReplica {
}
try {
- dataSet.createRbw(StorageType.DEFAULT, blocks[TEMPORARY], false);
+ dataSet.createRbw(StorageType.DEFAULT, null, blocks[TEMPORARY], false);
Assert.fail("Should not have created a replica that had created as " +
"temporary " + blocks[TEMPORARY]);
} catch (ReplicaAlreadyExistsException e) {
@@ -381,7 +381,7 @@ public class TestWriteToReplica {
0L, blocks[RBW].getNumBytes()); // expect to be successful
try {
- dataSet.createRbw(StorageType.DEFAULT, blocks[RBW], false);
+ dataSet.createRbw(StorageType.DEFAULT, null, blocks[RBW], false);
Assert.fail("Should not have created a replica that had created as RBW " +
blocks[RBW]);
} catch (ReplicaAlreadyExistsException e) {
@@ -397,7 +397,7 @@ public class TestWriteToReplica {
}
try {
- dataSet.createRbw(StorageType.DEFAULT, blocks[RWR], false);
+ dataSet.createRbw(StorageType.DEFAULT, null, blocks[RWR], false);
Assert.fail("Should not have created a replica that was waiting to be " +
"recovered " + blocks[RWR]);
} catch (ReplicaAlreadyExistsException e) {
@@ -413,7 +413,7 @@ public class TestWriteToReplica {
}
try {
- dataSet.createRbw(StorageType.DEFAULT, blocks[RUR], false);
+ dataSet.createRbw(StorageType.DEFAULT, null, blocks[RUR], false);
Assert.fail("Should not have created a replica that was under recovery " +
blocks[RUR]);
} catch (ReplicaAlreadyExistsException e) {
@@ -430,49 +430,49 @@ public class TestWriteToReplica {
e.getMessage().contains(ReplicaNotFoundException.NON_EXISTENT_REPLICA));
}
- dataSet.createRbw(StorageType.DEFAULT, blocks[NON_EXISTENT], false);
+ dataSet.createRbw(StorageType.DEFAULT, null, blocks[NON_EXISTENT], false);
}
private void testWriteToTemporary(FsDatasetImpl dataSet, ExtendedBlock[] blocks) throws IOException {
try {
- dataSet.createTemporary(StorageType.DEFAULT, blocks[FINALIZED]);
+ dataSet.createTemporary(StorageType.DEFAULT, null, blocks[FINALIZED]);
Assert.fail("Should not have created a temporary replica that was " +
"finalized " + blocks[FINALIZED]);
} catch (ReplicaAlreadyExistsException e) {
}
try {
- dataSet.createTemporary(StorageType.DEFAULT, blocks[TEMPORARY]);
+ dataSet.createTemporary(StorageType.DEFAULT, null, blocks[TEMPORARY]);
Assert.fail("Should not have created a replica that had created as" +
"temporary " + blocks[TEMPORARY]);
} catch (ReplicaAlreadyExistsException e) {
}
try {
- dataSet.createTemporary(StorageType.DEFAULT, blocks[RBW]);
+ dataSet.createTemporary(StorageType.DEFAULT, null, blocks[RBW]);
Assert.fail("Should not have created a replica that had created as RBW " +
blocks[RBW]);
} catch (ReplicaAlreadyExistsException e) {
}
try {
- dataSet.createTemporary(StorageType.DEFAULT, blocks[RWR]);
+ dataSet.createTemporary(StorageType.DEFAULT, null, blocks[RWR]);
Assert.fail("Should not have created a replica that was waiting to be " +
"recovered " + blocks[RWR]);
} catch (ReplicaAlreadyExistsException e) {
}
try {
- dataSet.createTemporary(StorageType.DEFAULT, blocks[RUR]);
+ dataSet.createTemporary(StorageType.DEFAULT, null, blocks[RUR]);
Assert.fail("Should not have created a replica that was under recovery " +
blocks[RUR]);
} catch (ReplicaAlreadyExistsException e) {
}
- dataSet.createTemporary(StorageType.DEFAULT, blocks[NON_EXISTENT]);
+ dataSet.createTemporary(StorageType.DEFAULT, null, blocks[NON_EXISTENT]);
try {
- dataSet.createTemporary(StorageType.DEFAULT, blocks[NON_EXISTENT]);
+ dataSet.createTemporary(StorageType.DEFAULT, null, blocks[NON_EXISTENT]);
Assert.fail("Should not have created a replica that had already been "
+ "created " + blocks[NON_EXISTENT]);
} catch (Exception e) {
@@ -485,7 +485,8 @@ public class TestWriteToReplica {
blocks[NON_EXISTENT].setGenerationStamp(newGenStamp);
try {
ReplicaInPipeline replicaInfo =
- dataSet.createTemporary(StorageType.DEFAULT, blocks[NON_EXISTENT]).getReplica();
+ dataSet.createTemporary(StorageType.DEFAULT, null,
+ blocks[NON_EXISTENT]).getReplica();
Assert.assertTrue(replicaInfo.getGenerationStamp() == newGenStamp);
Assert.assertTrue(
replicaInfo.getBlockId() == blocks[NON_EXISTENT].getBlockId());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3954cca/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeStorageDirectives.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeStorageDirectives.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeStorageDirectives.java
new file mode 100644
index 0000000..e0f7426
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeStorageDirectives.java
@@ -0,0 +1,330 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.namenode;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.ReconfigurationException;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.*;
+import org.apache.hadoop.hdfs.protocol.*;
+import org.apache.hadoop.hdfs.server.blockmanagement.*;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.RoundRobinVolumeChoosingPolicy;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.VolumeChoosingPolicy;
+import org.apache.hadoop.net.Node;
+import org.junit.After;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test to ensure that the StorageType and StorageID sent from Namenode
+ * to DFSClient are respected.
+ */
+public class TestNamenodeStorageDirectives {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(TestNamenodeStorageDirectives.class);
+
+ private static final int BLOCK_SIZE = 512;
+
+ private MiniDFSCluster cluster;
+
+ @After
+ public void tearDown() {
+ shutdown();
+ }
+
+ private void startDFSCluster(int numNameNodes, int numDataNodes,
+ int storagePerDataNode, StorageType[][] storageTypes)
+ throws IOException {
+ startDFSCluster(numNameNodes, numDataNodes, storagePerDataNode,
+ storageTypes, RoundRobinVolumeChoosingPolicy.class,
+ BlockPlacementPolicyDefault.class);
+ }
+
+ private void startDFSCluster(int numNameNodes, int numDataNodes,
+ int storagePerDataNode, StorageType[][] storageTypes,
+ Class<? extends VolumeChoosingPolicy> volumeChoosingPolicy,
+ Class<? extends BlockPlacementPolicy> blockPlacementPolicy) throws
+ IOException {
+ shutdown();
+ Configuration conf = new Configuration();
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
+
+ /*
+ * Lower the DN heartbeat, DF rate, and recheck interval to one second
+ * so state about failures and datanode death propagates faster.
+ */
+ conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_DF_INTERVAL_KEY, 1000);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY,
+ 1000);
+ /* Allow 1 volume failure */
+ conf.setInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY, 1);
+ conf.setTimeDuration(DFSConfigKeys.DFS_DATANODE_DISK_CHECK_MIN_GAP_KEY,
+ 0, TimeUnit.MILLISECONDS);
+ conf.setClass(
+ DFSConfigKeys.DFS_DATANODE_FSDATASET_VOLUME_CHOOSING_POLICY_KEY,
+ volumeChoosingPolicy, VolumeChoosingPolicy.class);
+ conf.setClass(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
+ blockPlacementPolicy, BlockPlacementPolicy.class);
+
+ MiniDFSNNTopology nnTopology =
+ MiniDFSNNTopology.simpleFederatedTopology(numNameNodes);
+
+ cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(nnTopology)
+ .numDataNodes(numDataNodes)
+ .storagesPerDatanode(storagePerDataNode)
+ .storageTypes(storageTypes)
+ .build();
+ cluster.waitActive();
+ }
+
+ private void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+
+ private void createFile(Path path, int numBlocks, short replicateFactor)
+ throws IOException, InterruptedException, TimeoutException {
+ createFile(0, path, numBlocks, replicateFactor);
+ }
+
+ private void createFile(int fsIdx, Path path, int numBlocks,
+ short replicateFactor)
+ throws IOException, TimeoutException, InterruptedException {
+ final int seed = 0;
+ final DistributedFileSystem fs = cluster.getFileSystem(fsIdx);
+ DFSTestUtil.createFile(fs, path, BLOCK_SIZE * numBlocks,
+ replicateFactor, seed);
+ DFSTestUtil.waitReplication(fs, path, replicateFactor);
+ }
+
+ private boolean verifyFileReplicasOnStorageType(Path path, int numBlocks,
+ StorageType storageType) throws IOException {
+ MiniDFSCluster.NameNodeInfo info = cluster.getNameNodeInfos()[0];
+ InetSocketAddress addr = info.nameNode.getServiceRpcAddress();
+ assert addr.getPort() != 0;
+ DFSClient client = new DFSClient(addr, cluster.getConfiguration(0));
+
+ FileSystem fs = cluster.getFileSystem();
+
+ if (!fs.exists(path)) {
+ LOG.info("verifyFileReplicasOnStorageType: file {} does not exist", path);
+ return false;
+ }
+ long fileLength = client.getFileInfo(path.toString()).getLen();
+ int foundBlocks = 0;
+ LocatedBlocks locatedBlocks =
+ client.getLocatedBlocks(path.toString(), 0, fileLength);
+ for (LocatedBlock locatedBlock : locatedBlocks.getLocatedBlocks()) {
+ for (StorageType st : locatedBlock.getStorageTypes()) {
+ if (st == storageType) {
+ foundBlocks++;
+ }
+ }
+ }
+
+ LOG.info("Found {}/{} blocks on StorageType {}",
+ foundBlocks, numBlocks, storageType);
+ final boolean isValid = foundBlocks >= numBlocks;
+ return isValid;
+ }
+
+ private void testStorageTypes(StorageType[][] storageTypes,
+ String storagePolicy, StorageType[] expectedStorageTypes,
+ StorageType[] unexpectedStorageTypes) throws ReconfigurationException,
+ InterruptedException, TimeoutException, IOException {
+ final int numDataNodes = storageTypes.length;
+ final int storagePerDataNode = storageTypes[0].length;
+ startDFSCluster(1, numDataNodes, storagePerDataNode, storageTypes);
+ cluster.getFileSystem(0).setStoragePolicy(new Path("/"), storagePolicy);
+ Path testFile = new Path("/test");
+ final short replFactor = 2;
+ final int numBlocks = 10;
+ createFile(testFile, numBlocks, replFactor);
+
+ for (StorageType storageType: expectedStorageTypes) {
+ assertTrue(verifyFileReplicasOnStorageType(testFile, numBlocks,
+ storageType));
+ }
+
+ for (StorageType storageType: unexpectedStorageTypes) {
+ assertFalse(verifyFileReplicasOnStorageType(testFile, numBlocks,
+ storageType));
+ }
+ }
+
+ /**
+ * Verify that writing to SSD and DISK will write to the correct Storage
+ * Types.
+ * @throws IOException
+ */
+ @Test(timeout=60000)
+ public void testTargetStorageTypes() throws ReconfigurationException,
+ InterruptedException, TimeoutException, IOException {
+ // DISK and not anything else.
+ testStorageTypes(new StorageType[][]{
+ {StorageType.SSD, StorageType.DISK},
+ {StorageType.SSD, StorageType.DISK}},
+ "ONE_SSD",
+ new StorageType[]{StorageType.SSD, StorageType.DISK},
+ new StorageType[]{StorageType.RAM_DISK, StorageType.ARCHIVE});
+ // only on SSD.
+ testStorageTypes(new StorageType[][]{
+ {StorageType.SSD, StorageType.DISK},
+ {StorageType.SSD, StorageType.DISK}},
+ "ALL_SSD",
+ new StorageType[]{StorageType.SSD},
+ new StorageType[]{StorageType.RAM_DISK, StorageType.DISK,
+ StorageType.ARCHIVE});
+ // only on SSD.
+ testStorageTypes(new StorageType[][]{
+ {StorageType.SSD, StorageType.DISK, StorageType.DISK},
+ {StorageType.SSD, StorageType.DISK, StorageType.DISK},
+ {StorageType.DISK, StorageType.DISK, StorageType.DISK}},
+ "ALL_SSD",
+ new StorageType[]{StorageType.SSD},
+ new StorageType[]{StorageType.RAM_DISK, StorageType.DISK,
+ StorageType.ARCHIVE});
+
+ // DISK and not anything else.
+ testStorageTypes(new StorageType[][] {
+ {StorageType.RAM_DISK, StorageType.SSD},
+ {StorageType.SSD, StorageType.DISK},
+ {StorageType.SSD, StorageType.DISK}},
+ "HOT",
+ new StorageType[]{StorageType.DISK},
+ new StorageType[] {StorageType.RAM_DISK, StorageType.SSD,
+ StorageType.ARCHIVE});
+
+ testStorageTypes(new StorageType[][] {
+ {StorageType.RAM_DISK, StorageType.SSD},
+ {StorageType.SSD, StorageType.DISK},
+ {StorageType.ARCHIVE, StorageType.ARCHIVE},
+ {StorageType.ARCHIVE, StorageType.ARCHIVE}},
+ "WARM",
+ new StorageType[]{StorageType.DISK, StorageType.ARCHIVE},
+ new StorageType[]{StorageType.RAM_DISK, StorageType.SSD});
+
+ testStorageTypes(new StorageType[][] {
+ {StorageType.RAM_DISK, StorageType.SSD},
+ {StorageType.SSD, StorageType.DISK},
+ {StorageType.ARCHIVE, StorageType.ARCHIVE},
+ {StorageType.ARCHIVE, StorageType.ARCHIVE}},
+ "COLD",
+ new StorageType[]{StorageType.ARCHIVE},
+ new StorageType[]{StorageType.RAM_DISK, StorageType.SSD,
+ StorageType.DISK});
+
+ // We wait for Lasy Persist to write to disk.
+ testStorageTypes(new StorageType[][] {
+ {StorageType.RAM_DISK, StorageType.SSD},
+ {StorageType.SSD, StorageType.DISK},
+ {StorageType.SSD, StorageType.DISK}},
+ "LAZY_PERSIST",
+ new StorageType[]{StorageType.DISK},
+ new StorageType[]{StorageType.RAM_DISK, StorageType.SSD,
+ StorageType.ARCHIVE});
+ }
+
+ /**
+ * A VolumeChoosingPolicy test stub used to verify that the storageId passed
+ * in is indeed in the list of volumes.
+ * @param <V>
+ */
+ private static class TestVolumeChoosingPolicy<V extends FsVolumeSpi>
+ extends RoundRobinVolumeChoosingPolicy<V> {
+ static String expectedStorageId;
+
+ @Override
+ public V chooseVolume(List<V> volumes, long replicaSize, String storageId)
+ throws IOException {
+ assertEquals(expectedStorageId, storageId);
+ return super.chooseVolume(volumes, replicaSize, storageId);
+ }
+ }
+
+ private static class TestBlockPlacementPolicy
+ extends BlockPlacementPolicyDefault {
+ static DatanodeStorageInfo[] dnStorageInfosToReturn;
+
+ @Override
+ public DatanodeStorageInfo[] chooseTarget(String srcPath, int numOfReplicas,
+ Node writer, List<DatanodeStorageInfo> chosenNodes,
+ boolean returnChosenNodes, Set<Node> excludedNodes, long blocksize,
+ final BlockStoragePolicy storagePolicy, EnumSet<AddBlockFlag> flags) {
+ return dnStorageInfosToReturn;
+ }
+ }
+
+ private DatanodeStorageInfo getDatanodeStorageInfo(int dnIndex)
+ throws UnregisteredNodeException {
+ if (cluster == null) {
+ return null;
+ }
+ DatanodeID dnId = cluster.getDataNodes().get(dnIndex).getDatanodeId();
+ DatanodeManager dnManager = cluster.getNamesystem()
+ .getBlockManager().getDatanodeManager();
+ return dnManager.getDatanode(dnId).getStorageInfos()[0];
+ }
+
+ @Test(timeout=60000)
+ public void testStorageIDBlockPlacementSpecific()
+ throws ReconfigurationException, InterruptedException, TimeoutException,
+ IOException {
+ final StorageType[][] storageTypes = {
+ {StorageType.DISK, StorageType.DISK},
+ {StorageType.DISK, StorageType.DISK},
+ {StorageType.DISK, StorageType.DISK},
+ {StorageType.DISK, StorageType.DISK},
+ {StorageType.DISK, StorageType.DISK},
+ };
+ final int numDataNodes = storageTypes.length;
+ final int storagePerDataNode = storageTypes[0].length;
+ startDFSCluster(1, numDataNodes, storagePerDataNode, storageTypes,
+ TestVolumeChoosingPolicy.class, TestBlockPlacementPolicy.class);
+ Path testFile = new Path("/test");
+ final short replFactor = 1;
+ final int numBlocks = 10;
+ DatanodeStorageInfo dnInfoToUse = getDatanodeStorageInfo(0);
+ TestBlockPlacementPolicy.dnStorageInfosToReturn =
+ new DatanodeStorageInfo[] {dnInfoToUse};
+ TestVolumeChoosingPolicy.expectedStorageId = dnInfoToUse.getStorageID();
+ //file creation invokes both BlockPlacementPolicy and VolumeChoosingPolicy,
+ //and will test that the storage ids match
+ createFile(testFile, numBlocks, replFactor);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: HADOOP-14373. License error in
org.apache.hadoop.metrics2.util.Servers. Contributed by hu xiaodong.
Posted by in...@apache.org.
HADOOP-14373. License error in org.apache.hadoop.metrics2.util.Servers. Contributed by hu xiaodong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a5fa9546
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a5fa9546
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a5fa9546
Branch: refs/heads/HDFS-10467
Commit: a5fa954684f517bdf0a50f3fa7a0314f24cc7049
Parents: 06ffeb8
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed May 10 05:29:47 2017 -0500
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed May 10 05:29:47 2017 -0500
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/metrics2/util/Servers.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5fa9546/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/Servers.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/Servers.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/Servers.java
index aa5d715..19a64a9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/Servers.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/Servers.java
@@ -1,6 +1,4 @@
-/*
- * Util.java
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: HDFS-11530. Use HDFS specific
network topology to choose datanode in BlockPlacementPolicyDefault.
Contributed by Yiqun Lin and Chen Liang.
Posted by in...@apache.org.
HDFS-11530. Use HDFS specific network topology to choose datanode in BlockPlacementPolicyDefault. Contributed by Yiqun Lin and Chen Liang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/97c2e576
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/97c2e576
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/97c2e576
Branch: refs/heads/HDFS-10467
Commit: 97c2e576c91c2316c2b52bfc948bae9bff8ca49f
Parents: 3082552
Author: Yiqun Lin <yq...@apache.org>
Authored: Fri May 5 11:54:50 2017 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Fri May 5 11:54:50 2017 +0800
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +
.../hadoop/hdfs/net/DFSNetworkTopology.java | 24 +++-
.../hadoop/hdfs/net/DFSTopologyNodeImpl.java | 137 +++++++++++++++++++
.../BlockPlacementPolicyDefault.java | 36 ++++-
.../blockmanagement/DatanodeDescriptor.java | 36 ++++-
.../server/blockmanagement/DatanodeManager.java | 19 ++-
.../src/main/resources/hdfs-default.xml | 8 ++
.../TestDefaultBlockPlacementPolicy.java | 46 +++++++
8 files changed, 302 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97c2e576/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 0ca344c..b95c7e6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -1085,6 +1085,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
"httpfs.buffer.size";
public static final int HTTP_BUFFER_SIZE_DEFAULT = 4096;
+ public static final String DFS_USE_DFS_NETWORK_TOPOLOGY_KEY =
+ "dfs.use.dfs.network.topology";
+ public static final boolean DFS_USE_DFS_NETWORK_TOPOLOGY_DEFAULT = false;
+
// dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry
@Deprecated
public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97c2e576/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
index 259e275..e74cdec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
@@ -19,8 +19,10 @@ package org.apache.hadoop.hdfs.net;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.net.Node;
@@ -204,10 +206,24 @@ public class DFSNetworkTopology extends NetworkTopology {
}
if (excludedNodes != null) {
for (Node excludedNode : excludedNodes) {
- // all excluded nodes should be DatanodeDescriptor
- Preconditions.checkArgument(excludedNode instanceof DatanodeDescriptor);
- availableCount -= ((DatanodeDescriptor) excludedNode)
- .hasStorageType(type) ? 1 : 0;
+ if (excludedNode instanceof DatanodeDescriptor) {
+ availableCount -= ((DatanodeDescriptor) excludedNode)
+ .hasStorageType(type) ? 1 : 0;
+ } else if (excludedNode instanceof DFSTopologyNodeImpl) {
+ availableCount -= ((DFSTopologyNodeImpl) excludedNode)
+ .getSubtreeStorageCount(type);
+ } else if (excludedNode instanceof DatanodeInfo) {
+ // find out the corresponding DatanodeDescriptor object, beacuse
+ // we need to get its storage type info.
+ // could be expensive operation, fortunately the size of excluded
+ // nodes set is supposed to be very small.
+ String nodeLocation = excludedNode.getNetworkLocation()
+ + "/" + excludedNode.getName();
+ DatanodeDescriptor dn = (DatanodeDescriptor)getNode(nodeLocation);
+ availableCount -= dn.hasStorageType(type)? 1 : 0;
+ } else {
+ LOG.error("Unexpected node type: {}.", excludedNode.getClass());
+ }
}
}
if (availableCount <= 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97c2e576/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSTopologyNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSTopologyNodeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSTopologyNodeImpl.java
index 6d80db5..002f4fc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSTopologyNodeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSTopologyNodeImpl.java
@@ -18,11 +18,14 @@
package org.apache.hadoop.hdfs.net;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.net.InnerNode;
import org.apache.hadoop.net.InnerNodeImpl;
import org.apache.hadoop.net.Node;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.EnumMap;
import java.util.EnumSet;
@@ -36,6 +39,9 @@ import java.util.HashMap;
*/
public class DFSTopologyNodeImpl extends InnerNodeImpl {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(DFSTopologyNodeImpl.class);
+
static final InnerNodeImpl.Factory FACTORY
= new DFSTopologyNodeImpl.Factory();
@@ -127,8 +133,68 @@ public class DFSTopologyNodeImpl extends InnerNodeImpl {
}
}
+ /**
+ * Called when add() is called to add a node that already exist.
+ *
+ * In normal execution, nodes are added only once and this should not happen.
+ * However if node restarts, we may run into the case where the same node
+ * tries to add itself again with potentially different storage type info.
+ * In this case this method will update the meta data according to the new
+ * storage info.
+ *
+ * Note that it is important to also update all the ancestors if we do have
+ * updated the local node storage info.
+ *
+ * @param dnDescriptor the node that is added another time, with potentially
+ * different storage types.
+ */
+ private void updateExistingDatanode(DatanodeDescriptor dnDescriptor) {
+ if (childrenStorageInfo.containsKey(dnDescriptor.getName())) {
+ // all existing node should have an entry in childrenStorageInfo
+ boolean same = dnDescriptor.getStorageTypes().size()
+ == childrenStorageInfo.get(dnDescriptor.getName()).keySet().size();
+ for (StorageType type :
+ childrenStorageInfo.get(dnDescriptor.getName()).keySet()) {
+ same = same && dnDescriptor.hasStorageType(type);
+ }
+ if (same) {
+ // if the storage type hasn't been changed, do nothing.
+ return;
+ }
+ // not same means we need to update the storage info.
+ DFSTopologyNodeImpl parent = (DFSTopologyNodeImpl)getParent();
+ for (StorageType type :
+ childrenStorageInfo.get(dnDescriptor.getName()).keySet()) {
+ if (!dnDescriptor.hasStorageType(type)) {
+ // remove this type, because the new storage info does not have it.
+ // also need to remove decrement the count for all the ancestors.
+ // since this is the parent of n, where n is a datanode,
+ // the map must have 1 as the value of all keys
+ childrenStorageInfo.get(dnDescriptor.getName()).remove(type);
+ decStorageTypeCount(type);
+ if (parent != null) {
+ parent.childRemoveStorage(getName(), type);
+ }
+ }
+ }
+ for (StorageType type : dnDescriptor.getStorageTypes()) {
+ if (!childrenStorageInfo.get(dnDescriptor.getName())
+ .containsKey(type)) {
+ // there is a new type in new storage info, add this locally,
+ // as well as all ancestors.
+ childrenStorageInfo.get(dnDescriptor.getName()).put(type, 1);
+ incStorageTypeCount(type);
+ if (parent != null) {
+ parent.childAddStorage(getName(), type);
+ }
+ }
+ }
+ }
+ }
+
@Override
public boolean add(Node n) {
+ LOG.debug("adding node {}", n.getName());
if (!isAncestor(n)) {
throw new IllegalArgumentException(n.getName()
+ ", which is located at " + n.getNetworkLocation()
@@ -149,6 +215,7 @@ public class DFSTopologyNodeImpl extends InnerNodeImpl {
for(int i=0; i<children.size(); i++) {
if (children.get(i).getName().equals(n.getName())) {
children.set(i, n);
+ updateExistingDatanode(dnDescriptor);
return false;
}
}
@@ -227,6 +294,7 @@ public class DFSTopologyNodeImpl extends InnerNodeImpl {
@Override
public boolean remove(Node n) {
+ LOG.debug("removing node {}", n.getName());
if (!isAncestor(n)) {
throw new IllegalArgumentException(n.getName()
+ ", which is located at " + n.getNetworkLocation()
@@ -299,4 +367,73 @@ public class DFSTopologyNodeImpl extends InnerNodeImpl {
return isRemoved;
}
}
+
+ /**
+ * Called by a child node of the current node to increment a storage count.
+ *
+ * lock is needed as different datanodes may call recursively to modify
+ * the same parent.
+ * TODO : this may not happen at all, depending on how heartheat is processed
+ * @param childName the name of the child that tries to add the storage type
+ * @param type the type being incremented.
+ */
+ public synchronized void childAddStorage(
+ String childName, StorageType type) {
+ LOG.debug("child add storage: {}:{}", childName, type);
+ // childrenStorageInfo should definitely contain this node already
+ // because updateStorage is called after node added
+ Preconditions.checkArgument(childrenStorageInfo.containsKey(childName));
+ EnumMap<StorageType, Integer> typeCount =
+ childrenStorageInfo.get(childName);
+ if (typeCount.containsKey(type)) {
+ typeCount.put(type, typeCount.get(type) + 1);
+ } else {
+ // Please be aware that, the counts are always "number of datanodes in
+ // this subtree" rather than "number of storages in this storage".
+ // so if the caller is a datanode, it should always be this branch rather
+ // than the +1 branch above. This depends on the caller in
+ // DatanodeDescriptor to make sure only when a *new* storage type is added
+ // it calls this. (should not call this when a already existing storage
+ // is added).
+ // but no such restriction for inner nodes.
+ typeCount.put(type, 1);
+ }
+ if (storageTypeCounts.containsKey(type)) {
+ storageTypeCounts.put(type, storageTypeCounts.get(type) + 1);
+ } else {
+ storageTypeCounts.put(type, 1);
+ }
+ if (getParent() != null) {
+ ((DFSTopologyNodeImpl)getParent()).childAddStorage(getName(), type);
+ }
+ }
+
+ /**
+ * Called by a child node of the current node to decrement a storage count.
+ *
+ * @param childName the name of the child removing a storage type.
+ * @param type the type being removed.
+ */
+ public synchronized void childRemoveStorage(
+ String childName, StorageType type) {
+ LOG.debug("child remove storage: {}:{}", childName, type);
+ Preconditions.checkArgument(childrenStorageInfo.containsKey(childName));
+ EnumMap<StorageType, Integer> typeCount =
+ childrenStorageInfo.get(childName);
+ Preconditions.checkArgument(typeCount.containsKey(type));
+ if (typeCount.get(type) > 1) {
+ typeCount.put(type, typeCount.get(type) - 1);
+ } else {
+ typeCount.remove(type);
+ }
+ Preconditions.checkArgument(storageTypeCounts.containsKey(type));
+ if (storageTypeCounts.get(type) > 1) {
+ storageTypeCounts.put(type, storageTypeCounts.get(type) - 1);
+ } else {
+ storageTypeCounts.remove(type);
+ }
+ if (getParent() != null) {
+ ((DFSTopologyNodeImpl)getParent()).childRemoveStorage(getName(), type);
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97c2e576/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index 7676334..a245f0c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hdfs.AddBlockFlag;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.net.DFSNetworkTopology;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.net.Node;
@@ -713,7 +714,22 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
boolean badTarget = false;
DatanodeStorageInfo firstChosen = null;
while (numOfReplicas > 0) {
- DatanodeDescriptor chosenNode = chooseDataNode(scope, excludedNodes);
+ // the storage type that current node has
+ StorageType includeType = null;
+ DatanodeDescriptor chosenNode = null;
+ if (clusterMap instanceof DFSNetworkTopology) {
+ for (StorageType type : storageTypes.keySet()) {
+ chosenNode = chooseDataNode(scope, excludedNodes, type);
+
+ if (chosenNode != null) {
+ includeType = type;
+ break;
+ }
+ }
+ } else {
+ chosenNode = chooseDataNode(scope, excludedNodes);
+ }
+
if (chosenNode == null) {
break;
}
@@ -729,6 +745,13 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
for (Iterator<Map.Entry<StorageType, Integer>> iter = storageTypes
.entrySet().iterator(); iter.hasNext();) {
Map.Entry<StorageType, Integer> entry = iter.next();
+
+ // If there is one storage type the node has already contained,
+ // then no need to loop through other storage type.
+ if (includeType != null && entry.getKey() != includeType) {
+ continue;
+ }
+
storage = chooseStorage4Block(
chosenNode, blocksize, results, entry.getKey());
if (storage != null) {
@@ -782,6 +805,17 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
}
/**
+ * Choose a datanode from the given <i>scope</i> with specified
+ * storage type.
+ * @return the chosen node, if there is any.
+ */
+ protected DatanodeDescriptor chooseDataNode(final String scope,
+ final Collection<Node> excludedNodes, StorageType type) {
+ return (DatanodeDescriptor) ((DFSNetworkTopology) clusterMap)
+ .chooseRandomWithStorageTypeTwoTrial(scope, excludedNodes, type);
+ }
+
+ /**
* Choose a good storage of given storage type from datanode, and add it to
* the result list.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97c2e576/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index d0583b3..4b87fd4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@ -35,6 +35,7 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.net.DFSTopologyNodeImpl;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -494,7 +495,16 @@ public class DatanodeDescriptor extends DatanodeInfo {
// blocks.
for (final DatanodeStorageInfo storageInfo : excessStorages.values()) {
if (storageInfo.numBlocks() == 0) {
- storageMap.remove(storageInfo.getStorageID());
+ DatanodeStorageInfo info =
+ storageMap.remove(storageInfo.getStorageID());
+ if (!hasStorageType(info.getStorageType())) {
+ // we removed a storage, and as result there is no more such storage
+ // type, inform the parent about this.
+ if (getParent() instanceof DFSTopologyNodeImpl) {
+ ((DFSTopologyNodeImpl) getParent()).childRemoveStorage(getName(),
+ info.getStorageType());
+ }
+ }
LOG.info("Removed storage {} from DataNode {}", storageInfo, this);
} else {
// This can occur until all block reports are received.
@@ -911,9 +921,20 @@ public class DatanodeDescriptor extends DatanodeInfo {
DatanodeStorageInfo updateStorage(DatanodeStorage s) {
synchronized (storageMap) {
DatanodeStorageInfo storage = storageMap.get(s.getStorageID());
+ DFSTopologyNodeImpl parent = null;
+ if (getParent() instanceof DFSTopologyNodeImpl) {
+ parent = (DFSTopologyNodeImpl) getParent();
+ }
+
if (storage == null) {
LOG.info("Adding new storage ID {} for DN {}", s.getStorageID(),
getXferAddr());
+ StorageType type = s.getStorageType();
+ if (!hasStorageType(type) && parent != null) {
+ // we are about to add a type this node currently does not have,
+ // inform the parent that a new type is added to this datanode
+ parent.childAddStorage(getName(), s.getStorageType());
+ }
storage = new DatanodeStorageInfo(this, s);
storageMap.put(s.getStorageID(), storage);
} else if (storage.getState() != s.getState() ||
@@ -921,8 +942,21 @@ public class DatanodeDescriptor extends DatanodeInfo {
// For backwards compatibility, make sure that the type and
// state are updated. Some reports from older datanodes do
// not include these fields so we may have assumed defaults.
+ StorageType oldType = storage.getStorageType();
+ StorageType newType = s.getStorageType();
+ if (oldType != newType && !hasStorageType(newType) && parent != null) {
+ // we are about to add a type this node currently does not have
+ // inform the parent that a new type is added to this datanode
+ // if old == new, nothing's changed. don't bother
+ parent.childAddStorage(getName(), newType);
+ }
storage.updateFromStorage(s);
storageMap.put(storage.getStorageID(), storage);
+ if (oldType != newType && !hasStorageType(oldType) && parent != null) {
+ // there is no more old type storage on this datanode, inform parent
+ // about this change.
+ parent.childRemoveStorage(getName(), oldType);
+ }
}
return storage;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97c2e576/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index a61aa78..7dcc9fd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.net.DFSNetworkTopology;
import org.apache.hadoop.hdfs.protocol.*;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
@@ -186,6 +187,11 @@ public class DatanodeManager {
*/
private final boolean dataNodeDiskStatsEnabled;
+ /**
+ * If we use DfsNetworkTopology to choose nodes for placing replicas.
+ */
+ private final boolean useDfsNetworkTopology;
+
@Nullable
private final SlowPeerTracker slowPeerTracker;
@Nullable
@@ -205,8 +211,17 @@ public class DatanodeManager {
final Configuration conf) throws IOException {
this.namesystem = namesystem;
this.blockManager = blockManager;
-
- networktopology = NetworkTopology.getInstance(conf);
+
+ // TODO: Enables DFSNetworkTopology by default after more stress
+ // testings/validations.
+ this.useDfsNetworkTopology = conf.getBoolean(
+ DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_KEY,
+ DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_DEFAULT);
+ if (useDfsNetworkTopology) {
+ networktopology = DFSNetworkTopology.getInstance(conf);
+ } else {
+ networktopology = NetworkTopology.getInstance(conf);
+ }
this.heartbeatManager = new HeartbeatManager(namesystem, blockManager, conf);
this.decomManager = new DecommissionManager(namesystem, blockManager,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97c2e576/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 0f33b70..f0f2220 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4505,4 +4505,12 @@
</description>
</property>
+ <property>
+ <name>dfs.use.dfs.network.topology</name>
+ <value>false</value>
+ <description>
+ Enables DFSNetworkTopology to choose nodes for placing replicas.
+ </description>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97c2e576/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDefaultBlockPlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDefaultBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDefaultBlockPlacementPolicy.java
index 0931ff4..eab1199 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDefaultBlockPlacementPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDefaultBlockPlacementPolicy.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.net.DFSNetworkTopology;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
@@ -98,6 +99,51 @@ public class TestDefaultBlockPlacementPolicy {
}
/**
+ * Verify local node selection with using DFSNetworkTopology.
+ */
+ @Test
+ public void testPlacementWithDFSNetworkTopology() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ final String[] racks = {"/RACK0", "/RACK0", "/RACK2", "/RACK3", "/RACK2"};
+ final String[] hosts = {"/host0", "/host1", "/host2", "/host3", "/host4"};
+
+ // enables DFSNetworkTopology
+ conf.setBoolean(DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_KEY, true);
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE);
+ conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY,
+ DEFAULT_BLOCK_SIZE / 2);
+
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(5).racks(racks)
+ .hosts(hosts).build();
+ cluster.waitActive();
+ nameNodeRpc = cluster.getNameNodeRpc();
+ namesystem = cluster.getNamesystem();
+
+ DatanodeManager dm = namesystem.getBlockManager().getDatanodeManager();
+ assertTrue(dm.getNetworkTopology() instanceof DFSNetworkTopology);
+
+ String clientMachine = "/host3";
+ String clientRack = "/RACK3";
+ String src = "/test";
+ // Create the file with client machine
+ HdfsFileStatus fileStatus = namesystem.startFile(src, perm, clientMachine,
+ clientMachine, EnumSet.of(CreateFlag.CREATE), true, REPLICATION_FACTOR,
+ DEFAULT_BLOCK_SIZE, null, null, false);
+ LocatedBlock locatedBlock = nameNodeRpc.addBlock(src, clientMachine, null,
+ null, fileStatus.getFileId(), null, null);
+
+ assertEquals("Block should be allocated sufficient locations",
+ REPLICATION_FACTOR, locatedBlock.getLocations().length);
+ assertEquals("First datanode should be rack local", clientRack,
+ locatedBlock.getLocations()[0].getNetworkLocation());
+ nameNodeRpc.abandonBlock(locatedBlock.getBlock(), fileStatus.getFileId(),
+ src, clientMachine);
+ }
+
+ /**
* Verify decommissioned nodes should not be selected.
*/
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: YARN-6281. Cleanup when AMRMProxy
fails to initialize a new interceptor chain. (Botong Huang via Subru)
Posted by in...@apache.org.
YARN-6281. Cleanup when AMRMProxy fails to initialize a new interceptor chain. (Botong Huang via Subru)
(cherry picked from commit 57a9afbd45b7ef8e6021cc58f96bd0074bf1389d)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/749e5c09
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/749e5c09
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/749e5c09
Branch: refs/heads/HDFS-10467
Commit: 749e5c09b9990590c282ea944e24735b795351fc
Parents: cd9ff27
Author: Subru Krishnan <su...@apache.org>
Authored: Fri Mar 10 18:13:29 2017 -0800
Committer: Subru Krishnan <su...@apache.org>
Committed: Mon May 8 16:55:47 2017 -0700
----------------------------------------------------------------------
.../nodemanager/amrmproxy/AMRMProxyService.java | 25 +++++++++++------
.../amrmproxy/BaseAMRMProxyTest.java | 21 +++++++++-----
.../amrmproxy/TestAMRMProxyService.java | 29 ++++++++++++++++++++
3 files changed, 59 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/749e5c09/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
index 2696bca..aeb3be8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
@@ -319,11 +319,16 @@ public class AMRMProxyService extends AbstractService implements
+ " ApplicationId:" + applicationAttemptId + " for the user: "
+ user);
- RequestInterceptor interceptorChain =
- this.createRequestInterceptorChain();
- interceptorChain.init(createApplicationMasterContext(
- applicationAttemptId, user, amrmToken, localToken));
- chainWrapper.init(interceptorChain, applicationAttemptId);
+ try {
+ RequestInterceptor interceptorChain =
+ this.createRequestInterceptorChain();
+ interceptorChain.init(createApplicationMasterContext(this.nmContext,
+ applicationAttemptId, user, amrmToken, localToken));
+ chainWrapper.init(interceptorChain, applicationAttemptId);
+ } catch (Exception e) {
+ this.applPipelineMap.remove(applicationAttemptId.getApplicationId());
+ throw e;
+ }
}
/**
@@ -339,8 +344,10 @@ public class AMRMProxyService extends AbstractService implements
this.applPipelineMap.remove(applicationId);
if (pipeline == null) {
- LOG.info("Request to stop an application that does not exist. Id:"
- + applicationId);
+ LOG.info(
+ "No interceptor pipeline for application {},"
+ + " likely because its AM is not run in this node.",
+ applicationId);
} else {
// Remove the appAttempt in AMRMTokenSecretManager
this.secretManager
@@ -413,11 +420,11 @@ public class AMRMProxyService extends AbstractService implements
}
private AMRMProxyApplicationContext createApplicationMasterContext(
- ApplicationAttemptId applicationAttemptId, String user,
+ Context context, ApplicationAttemptId applicationAttemptId, String user,
Token<AMRMTokenIdentifier> amrmToken,
Token<AMRMTokenIdentifier> localToken) {
AMRMProxyApplicationContextImpl appContext =
- new AMRMProxyApplicationContextImpl(this.nmContext, getConfig(),
+ new AMRMProxyApplicationContextImpl(context, getConfig(),
applicationAttemptId, user, amrmToken, localToken);
return appContext;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/749e5c09/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
index 7f96947..6f5009e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
@@ -121,9 +121,9 @@ public abstract class BaseAMRMProxyTest {
+ MockRequestInterceptor.class.getName());
this.dispatcher = new AsyncDispatcher();
- this.dispatcher.init(conf);
+ this.dispatcher.init(this.conf);
this.dispatcher.start();
- this.amrmProxyService = createAndStartAMRMProxyService();
+ createAndStartAMRMProxyService(this.conf);
}
@After
@@ -137,12 +137,19 @@ public abstract class BaseAMRMProxyTest {
return threadpool;
}
- protected MockAMRMProxyService createAndStartAMRMProxyService() {
- MockAMRMProxyService svc =
+ protected Configuration getConf() {
+ return this.conf;
+ }
+
+ protected void createAndStartAMRMProxyService(Configuration config) {
+ // Stop the existing instance first if not null
+ if (this.amrmProxyService != null) {
+ this.amrmProxyService.stop();
+ }
+ this.amrmProxyService =
new MockAMRMProxyService(new NullContext(), dispatcher);
- svc.init(conf);
- svc.start();
- return svc;
+ this.amrmProxyService.init(config);
+ this.amrmProxyService.start();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/749e5c09/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java
index 837278c..fa17f26 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java
@@ -21,9 +21,11 @@ package org.apache.hadoop.yarn.server.nodemanager.amrmproxy;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse;
@@ -34,6 +36,7 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.nodemanager.amrmproxy.AMRMProxyService.RequestInterceptorChainWrapper;
import org.apache.hadoop.yarn.util.Records;
@@ -95,6 +98,32 @@ public class TestAMRMProxyService extends BaseAMRMProxyTest {
}
/**
+ * Tests the case when interceptor pipeline initialization fails.
+ */
+ @Test
+ public void testInterceptorInitFailure() {
+ Configuration conf = this.getConf();
+ // Override with a bad interceptor configuration
+ conf.set(YarnConfiguration.AMRM_PROXY_INTERCEPTOR_CLASS_PIPELINE,
+ "class.that.does.not.exist");
+
+ // Reinitialize instance with the new config
+ createAndStartAMRMProxyService(conf);
+ int testAppId = 1;
+ try {
+ registerApplicationMaster(testAppId);
+ Assert.fail("Should not reach here. Expecting an exception thrown");
+ } catch (Exception e) {
+ Map<ApplicationId, RequestInterceptorChainWrapper> pipelines =
+ getAMRMProxyService().getPipelines();
+ ApplicationId id = getApplicationId(testAppId);
+ Assert.assertTrue(
+ "The interceptor pipeline should be removed if initializtion fails",
+ pipelines.get(id) == null);
+ }
+ }
+
+ /**
* Tests the registration of multiple application master serially one at a
* time.
*
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: HADOOP-14298.
TestHadoopArchiveLogsRunner fails. Contribute dby Akira Ajisaka.
Posted by in...@apache.org.
HADOOP-14298. TestHadoopArchiveLogsRunner fails. Contribute dby Akira Ajisaka.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8065129d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8065129d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8065129d
Branch: refs/heads/HDFS-10467
Commit: 8065129d87e17f7d4424b1b1619c2e4acabee199
Parents: d6eed5a
Author: Andrew Wang <wa...@apache.org>
Authored: Sun May 7 13:59:15 2017 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Sun May 7 13:59:15 2017 -0700
----------------------------------------------------------------------
hadoop-tools/hadoop-archive-logs/pom.xml | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8065129d/hadoop-tools/hadoop-archive-logs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-archive-logs/pom.xml b/hadoop-tools/hadoop-archive-logs/pom.xml
index 8fdad6ea..b45ad98 100644
--- a/hadoop-tools/hadoop-archive-logs/pom.xml
+++ b/hadoop-tools/hadoop-archive-logs/pom.xml
@@ -62,6 +62,11 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-tests</artifactId>
<type>test-jar</type>
<scope>test</scope>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: HADOOP-14382 Remove usages of
MoreObjects.toStringHelper. Contributed by Andrew Wang
Posted by in...@apache.org.
HADOOP-14382 Remove usages of MoreObjects.toStringHelper.
Contributed by Andrew Wang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4e6bbd04
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4e6bbd04
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4e6bbd04
Branch: refs/heads/HDFS-10467
Commit: 4e6bbd049dead7008942bda09dfd54542c407f48
Parents: 97c2e57
Author: Steve Loughran <st...@apache.org>
Authored: Fri May 5 13:03:09 2017 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri May 5 13:03:09 2017 +0100
----------------------------------------------------------------------
.../apache/hadoop/metrics2/AbstractMetric.java | 16 +++++++++-------
.../org/apache/hadoop/metrics2/MetricsTag.java | 13 +++++++------
.../metrics2/impl/AbstractMetricsRecord.java | 16 ++++++++--------
.../org/apache/hadoop/metrics2/impl/MsInfo.java | 9 +++++----
.../hadoop/metrics2/lib/MetricsInfoImpl.java | 11 +++++++----
.../hadoop/metrics2/lib/MetricsRegistry.java | 20 +++++++++++---------
.../hadoop/metrics2/source/JvmMetricsInfo.java | 9 +++++----
.../hadoop/metrics2/util/MetricsCache.java | 18 +++++++++---------
.../hadoop/metrics2/impl/TestKafkaMetrics.java | 8 +++++---
9 files changed, 66 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e6bbd04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/AbstractMetric.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/AbstractMetric.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/AbstractMetric.java
index 0605156..e2574f6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/AbstractMetric.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/AbstractMetric.java
@@ -18,13 +18,14 @@
package org.apache.hadoop.metrics2;
-import com.google.common.base.MoreObjects;
import com.google.common.base.Objects;
-import static com.google.common.base.Preconditions.*;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import java.util.StringJoiner;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
/**
* The immutable metric
*/
@@ -84,10 +85,11 @@ public abstract class AbstractMetric implements MetricsInfo {
return Objects.hashCode(info, value());
}
- @Override public String toString() {
- return MoreObjects.toStringHelper(this)
- .add("info", info)
- .add("value", value())
+ @Override
+ public String toString() {
+ return new StringJoiner(", ", this.getClass().getSimpleName() + "{", "}")
+ .add("info=" + info)
+ .add("value=" + value())
.toString();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e6bbd04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsTag.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsTag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsTag.java
index 68b0737..db8a5d9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsTag.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsTag.java
@@ -18,13 +18,14 @@
package org.apache.hadoop.metrics2;
-import com.google.common.base.MoreObjects;
import com.google.common.base.Objects;
-import static com.google.common.base.Preconditions.*;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import java.util.StringJoiner;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
/**
* Immutable tag for metrics (for grouping on host/queue/username etc.)
*/
@@ -81,9 +82,9 @@ public class MetricsTag implements MetricsInfo {
}
@Override public String toString() {
- return MoreObjects.toStringHelper(this)
- .add("info", info)
- .add("value", value())
+ return new StringJoiner(", ", this.getClass().getSimpleName() + "{", "}")
+ .add("info=" + info)
+ .add("value=" + value())
.toString();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e6bbd04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/AbstractMetricsRecord.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/AbstractMetricsRecord.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/AbstractMetricsRecord.java
index fec29c2..a4632c6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/AbstractMetricsRecord.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/AbstractMetricsRecord.java
@@ -18,12 +18,12 @@
package org.apache.hadoop.metrics2.impl;
-import com.google.common.base.MoreObjects;
import com.google.common.base.Objects;
import com.google.common.collect.Iterables;
-
import org.apache.hadoop.metrics2.MetricsRecord;
+import java.util.StringJoiner;
+
abstract class AbstractMetricsRecord implements MetricsRecord {
@Override public boolean equals(Object obj) {
@@ -44,12 +44,12 @@ abstract class AbstractMetricsRecord implements MetricsRecord {
}
@Override public String toString() {
- return MoreObjects.toStringHelper(this)
- .add("timestamp", timestamp())
- .add("name", name())
- .add("description", description())
- .add("tags", tags())
- .add("metrics", Iterables.toString(metrics()))
+ return new StringJoiner(", ", this.getClass().getSimpleName() + "{", "}")
+ .add("timestamp=" + timestamp())
+ .add("name=" + name())
+ .add("description=" + description())
+ .add("tags=" + tags())
+ .add("metrics=" + Iterables.toString(metrics()))
.toString();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e6bbd04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MsInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MsInfo.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MsInfo.java
index 5de7edc..0bf5c78 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MsInfo.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MsInfo.java
@@ -18,11 +18,11 @@
package org.apache.hadoop.metrics2.impl;
-import com.google.common.base.MoreObjects;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.metrics2.MetricsInfo;
+import java.util.StringJoiner;
+
/**
* Metrics system related metrics info instances
*/
@@ -48,8 +48,9 @@ public enum MsInfo implements MetricsInfo {
}
@Override public String toString() {
- return MoreObjects.toStringHelper(this)
- .add("name", name()).add("description", desc)
+ return new StringJoiner(", ", this.getClass().getSimpleName() + "{", "}")
+ .add("name=" + name())
+ .add("description=" + desc)
.toString();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e6bbd04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsInfoImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsInfoImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsInfoImpl.java
index 054f211..e3adc82 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsInfoImpl.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsInfoImpl.java
@@ -18,11 +18,13 @@
package org.apache.hadoop.metrics2.lib;
-import com.google.common.base.MoreObjects;
import com.google.common.base.Objects;
-import static com.google.common.base.Preconditions.*;
import org.apache.hadoop.metrics2.MetricsInfo;
+import java.util.StringJoiner;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
/**
* Making implementing metric info a little easier
*/
@@ -56,8 +58,9 @@ class MetricsInfoImpl implements MetricsInfo {
}
@Override public String toString() {
- return MoreObjects.toStringHelper(this)
- .add("name", name).add("description", description)
+ return new StringJoiner(", ", this.getClass().getSimpleName() + "{", "}")
+ .add("name=" + name)
+ .add("description=" + description)
.toString();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e6bbd04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java
index 7070869..9727954 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java
@@ -18,20 +18,19 @@
package org.apache.hadoop.metrics2.lib;
-import java.util.Collection;
-import java.util.Map;
-
import com.google.common.collect.Maps;
-import com.google.common.base.MoreObjects;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.metrics2.MetricsInfo;
import org.apache.hadoop.metrics2.MetricsException;
+import org.apache.hadoop.metrics2.MetricsInfo;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.MetricsTag;
import org.apache.hadoop.metrics2.impl.MsInfo;
+import java.util.Collection;
+import java.util.Map;
+import java.util.StringJoiner;
+
/**
* An optional metrics registry class for creating and maintaining a
* collection of MetricsMutables, making writing metrics source easier.
@@ -440,9 +439,12 @@ public class MetricsRegistry {
}
}
- @Override public String toString() {
- return MoreObjects.toStringHelper(this)
- .add("info", metricsInfo).add("tags", tags()).add("metrics", metrics())
+ @Override
+ public String toString() {
+ return new StringJoiner(", ", this.getClass().getSimpleName() + "{", "}")
+ .add("info=" + metricsInfo.toString())
+ .add("tags=" + tags())
+ .add("metrics=" + metrics())
.toString();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e6bbd04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetricsInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetricsInfo.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetricsInfo.java
index 59a79fd..8da6785 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetricsInfo.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetricsInfo.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.metrics2.source;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.metrics2.MetricsInfo;
-import com.google.common.base.MoreObjects;
+import java.util.StringJoiner;
/**
* JVM and logging related metrics info instances
@@ -60,8 +60,9 @@ public enum JvmMetricsInfo implements MetricsInfo {
@Override public String description() { return desc; }
@Override public String toString() {
- return MoreObjects.toStringHelper(this)
- .add("name", name()).add("description", desc)
- .toString();
+ return new StringJoiner(", ", this.getClass().getSimpleName() + "{", "}")
+ .add("name=" + name())
+ .add("description=" + desc)
+ .toString();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e6bbd04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MetricsCache.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MetricsCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MetricsCache.java
index 753e307..cfd126c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MetricsCache.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MetricsCache.java
@@ -18,11 +18,7 @@
package org.apache.hadoop.metrics2.util;
-import java.util.Collection;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Set;
-
+import com.google.common.collect.Maps;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -31,8 +27,11 @@ import org.apache.hadoop.metrics2.AbstractMetric;
import org.apache.hadoop.metrics2.MetricsRecord;
import org.apache.hadoop.metrics2.MetricsTag;
-import com.google.common.base.MoreObjects;
-import com.google.common.collect.Maps;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.StringJoiner;
/**
* A metrics cache for sinks that don't support sparse updates.
@@ -127,8 +126,9 @@ public class MetricsCache {
}
@Override public String toString() {
- return MoreObjects.toStringHelper(this)
- .add("tags", tags).add("metrics", metrics)
+ return new StringJoiner(", ", this.getClass().getSimpleName() + "{", "}")
+ .add("tags=" + tags)
+ .add("metrics=" + metrics)
.toString();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e6bbd04/hadoop-tools/hadoop-kafka/src/test/java/org/apache/hadoop/metrics2/impl/TestKafkaMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-kafka/src/test/java/org/apache/hadoop/metrics2/impl/TestKafkaMetrics.java b/hadoop-tools/hadoop-kafka/src/test/java/org/apache/hadoop/metrics2/impl/TestKafkaMetrics.java
index bee6aaa..cd40ac8 100644
--- a/hadoop-tools/hadoop-kafka/src/test/java/org/apache/hadoop/metrics2/impl/TestKafkaMetrics.java
+++ b/hadoop-tools/hadoop-kafka/src/test/java/org/apache/hadoop/metrics2/impl/TestKafkaMetrics.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.metrics2.impl;
-import com.google.common.base.MoreObjects;
import com.google.common.collect.Lists;
import org.apache.commons.configuration2.SubsetConfiguration;
import org.apache.hadoop.metrics2.AbstractMetric;
@@ -40,6 +39,7 @@ import org.slf4j.LoggerFactory;
import java.net.InetAddress;
import java.text.SimpleDateFormat;
import java.util.Date;
+import java.util.StringJoiner;
import java.util.concurrent.Future;
import static org.junit.Assert.assertEquals;
@@ -74,8 +74,10 @@ public class TestKafkaMetrics {
@Override
public String toString() {
- return MoreObjects.toStringHelper(this).add("name", name())
- .add("description", desc).toString();
+ return new StringJoiner(", ", this.getClass().getSimpleName() + "{", "}")
+ .add("name=" + name())
+ .add("description=" + desc)
+ .toString();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org