You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2017/09/07 20:20:04 UTC
[01/40] hadoop git commit: YARN-7148. TestLogsCLI fails in trunk and
branch-2 and javadoc error. Contributed by Xuan Gong. [Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/HDFS-10467 3f7df14a2 -> 504b639b1 (forced update)
YARN-7148. TestLogsCLI fails in trunk and branch-2 and javadoc error. Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/63231a2a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/63231a2a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/63231a2a
Branch: refs/heads/HDFS-10467
Commit: 63231a2a3008794c5ebcbc9d3855638fc5f28216
Parents: 1f3bc63
Author: Junping Du <ju...@apache.org>
Authored: Wed Sep 6 13:19:34 2017 -0700
Committer: Junping Du <ju...@apache.org>
Committed: Wed Sep 6 13:19:34 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/client/cli/TestLogsCLI.java | 16 ++--
.../logaggregation/AggregatedLogFormat.java | 78 ++++++++++----------
.../logaggregation/LogAggregationUtils.java | 32 ++++----
.../yarn/logaggregation/LogCLIHelpers.java | 41 ++++++++--
.../LogAggregationFileController.java | 11 +--
.../tfile/LogAggregationTFileController.java | 1 -
6 files changed, 103 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63231a2a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
index 509a790..d58732e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
@@ -285,7 +285,7 @@ public class TestLogsCLI {
@Test (timeout = 15000)
public void testFetchFinishedApplictionLogs() throws Exception {
String remoteLogRootDir = "target/logs/";
- Configuration configuration = new Configuration();
+ Configuration configuration = new YarnConfiguration();
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
@@ -646,7 +646,7 @@ public class TestLogsCLI {
any(ContainerId.class));
// create local logs
- Configuration configuration = new Configuration();
+ Configuration configuration = new YarnConfiguration();
FileSystem fs = FileSystem.get(configuration);
String rootLogDir = "target/LocalLogs";
Path rootLogDirPath = new Path(rootLogDir);
@@ -790,7 +790,7 @@ public class TestLogsCLI {
UserGroupInformation testUgi = UserGroupInformation
.createRemoteUser(testUser);
- Configuration configuration = new Configuration();
+ Configuration configuration = new YarnConfiguration();
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
@@ -921,7 +921,7 @@ public class TestLogsCLI {
public void testLogsCLIWithInvalidArgs() throws Exception {
String localDir = "target/SaveLogs";
Path localPath = new Path(localDir);
- Configuration configuration = new Configuration();
+ Configuration configuration = new YarnConfiguration();
FileSystem fs = FileSystem.get(configuration);
ApplicationId appId = ApplicationId.newInstance(0, 1);
YarnClient mockYarnClient =
@@ -992,7 +992,7 @@ public class TestLogsCLI {
String localDir = "target/SaveLogs";
Path localPath = new Path(localDir);
- Configuration configuration = new Configuration();
+ Configuration configuration = new YarnConfiguration();
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
@@ -1083,7 +1083,7 @@ public class TestLogsCLI {
@Test (timeout = 15000)
public void testPrintContainerLogMetadata() throws Exception {
String remoteLogRootDir = "target/logs/";
- Configuration configuration = new Configuration();
+ Configuration configuration = new YarnConfiguration();
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
@@ -1188,7 +1188,7 @@ public class TestLogsCLI {
@Test (timeout = 15000)
public void testListNodeInfo() throws Exception {
String remoteLogRootDir = "target/logs/";
- Configuration configuration = new Configuration();
+ Configuration configuration = new YarnConfiguration();
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
@@ -1239,7 +1239,7 @@ public class TestLogsCLI {
@Test (timeout = 15000)
public void testFetchApplictionLogsHar() throws Exception {
String remoteLogRootDir = "target/logs/";
- Configuration configuration = new Configuration();
+ Configuration configuration = new YarnConfiguration();
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63231a2a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
index 3c1dcdc..9bec147 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
@@ -583,9 +583,9 @@ public class AggregatedLogFormat {
/**
* Returns the owner of the application.
- *
+ *
* @return the application owner.
- * @throws IOException
+ * @throws IOException if we can not get the application owner.
*/
public String getApplicationOwner() throws IOException {
TFile.Reader.Scanner ownerScanner = null;
@@ -610,9 +610,9 @@ public class AggregatedLogFormat {
/**
* Returns ACLs for the application. An empty map is returned if no ACLs are
* found.
- *
+ *
* @return a map of the Application ACLs.
- * @throws IOException
+ * @throws IOException if we can not get the application acls.
*/
public Map<ApplicationAccessType, String> getApplicationAcls()
throws IOException {
@@ -656,9 +656,10 @@ public class AggregatedLogFormat {
/**
* Read the next key and return the value-stream.
*
- * @param key
- * @return the valueStream if there are more keys or null otherwise.
- * @throws IOException
+ * @param key the log key
+ * @return the valueStream if there are more keys or null otherwise
+ * @throws IOException if we can not get the dataInputStream
+ * for the next key
*/
public DataInputStream next(LogKey key) throws IOException {
if (!this.atBeginning) {
@@ -683,10 +684,10 @@ public class AggregatedLogFormat {
* Get a ContainerLogsReader to read the logs for
* the specified container.
*
- * @param containerId
+ * @param containerId the containerId
* @return object to read the container's logs or null if the
* logs could not be found
- * @throws IOException
+ * @throws IOException if we can not get the container log reader.
*/
@Private
public ContainerLogsReader getContainerLogsReader(
@@ -752,10 +753,10 @@ public class AggregatedLogFormat {
/**
* Writes all logs for a single container to the provided writer.
- * @param valueStream
- * @param writer
- * @param logUploadedTime
- * @throws IOException
+ * @param valueStream the valueStream
+ * @param writer the log writer
+ * @param logUploadedTime the time stamp
+ * @throws IOException if we can not read the container logs.
*/
public static void readAcontainerLogs(DataInputStream valueStream,
Writer writer, long logUploadedTime) throws IOException {
@@ -780,9 +781,9 @@ public class AggregatedLogFormat {
/**
* Writes all logs for a single container to the provided writer.
- * @param valueStream
- * @param writer
- * @throws IOException
+ * @param valueStream the value stream
+ * @param writer the log writer
+ * @throws IOException if we can not read the container logs.
*/
public static void readAcontainerLogs(DataInputStream valueStream,
Writer writer) throws IOException {
@@ -849,10 +850,11 @@ public class AggregatedLogFormat {
* Keep calling this till you get a {@link EOFException} for getting logs of
* all types for a single container.
*
- * @param valueStream
- * @param out
- * @param logUploadedTime
- * @throws IOException
+ * @param valueStream the value stream
+ * @param out the print stream
+ * @param logUploadedTime the time stamp
+ * @throws IOException if we can not read the container log by specifying
+ * the container log type.
*/
public static void readAContainerLogsForALogType(
DataInputStream valueStream, PrintStream out, long logUploadedTime)
@@ -864,11 +866,11 @@ public class AggregatedLogFormat {
* Keep calling this till you get a {@link EOFException} for getting logs of
* all types for a single container for the specific bytes.
*
- * @param valueStream
- * @param out
- * @param logUploadedTime
- * @param bytes
- * @throws IOException
+ * @param valueStream the value stream
+ * @param out the output print stream
+ * @param logUploadedTime the log upload time stamp
+ * @param bytes the output size of the log
+ * @throws IOException if we can not read the container log
*/
public static void readAContainerLogsForALogType(
DataInputStream valueStream, PrintStream out, long logUploadedTime,
@@ -880,9 +882,9 @@ public class AggregatedLogFormat {
* Keep calling this till you get a {@link EOFException} for getting logs of
* all types for a single container.
*
- * @param valueStream
- * @param out
- * @throws IOException
+ * @param valueStream the value stream
+ * @param out the output print stream
+ * @throws IOException if we can not read the container log
*/
public static void readAContainerLogsForALogType(
DataInputStream valueStream, PrintStream out)
@@ -893,11 +895,11 @@ public class AggregatedLogFormat {
/**
* Keep calling this till you get a {@link EOFException} for getting logs of
* the specific types for a single container.
- * @param valueStream
- * @param out
- * @param logUploadedTime
- * @param logType
- * @throws IOException
+ * @param valueStream the value stream
+ * @param out the output print stream
+ * @param logUploadedTime the log uploaded time stamp
+ * @param logType the given log type
+ * @throws IOException if we can not read the container logs
*/
public static int readContainerLogsForALogType(
DataInputStream valueStream, PrintStream out, long logUploadedTime,
@@ -909,11 +911,11 @@ public class AggregatedLogFormat {
/**
* Keep calling this till you get a {@link EOFException} for getting logs of
* the specific types for a single container.
- * @param valueStream
- * @param out
- * @param logUploadedTime
- * @param logType
- * @throws IOException
+ * @param valueStream the value stream
+ * @param out the output print stream
+ * @param logUploadedTime the log uploaded time stamp
+ * @param logType the given log type
+ * @throws IOException if we can not read the container logs
*/
public static int readContainerLogsForALogType(
DataInputStream valueStream, PrintStream out, long logUploadedTime,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63231a2a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java
index e8a28de..6d04c29 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java
@@ -38,11 +38,11 @@ public class LogAggregationUtils {
/**
* Constructs the full filename for an application's log file per node.
- * @param remoteRootLogDir
- * @param appId
- * @param user
- * @param nodeId
- * @param suffix
+ * @param remoteRootLogDir the aggregated remote root log dir
+ * @param appId the application Id
+ * @param user the application owner
+ * @param nodeId the node id
+ * @param suffix the log dir suffix
* @return the remote log file.
*/
public static Path getRemoteNodeLogFileForApp(Path remoteRootLogDir,
@@ -53,10 +53,10 @@ public class LogAggregationUtils {
/**
* Gets the remote app log dir.
- * @param remoteRootLogDir
- * @param appId
- * @param user
- * @param suffix
+ * @param remoteRootLogDir the aggregated log remote root log dir
+ * @param appId the application id
+ * @param user the application owner
+ * @param suffix the log directory suffix
* @return the remote application specific log dir.
*/
public static Path getRemoteAppLogDir(Path remoteRootLogDir,
@@ -67,9 +67,9 @@ public class LogAggregationUtils {
/**
* Gets the remote suffixed log dir for the user.
- * @param remoteRootLogDir
- * @param user
- * @param suffix
+ * @param remoteRootLogDir the aggregated log remote root log dir
+ * @param user the application owner
+ * @param suffix the log dir suffix
* @return the remote suffixed log dir.
*/
public static Path getRemoteLogSuffixedDir(Path remoteRootLogDir,
@@ -83,8 +83,8 @@ public class LogAggregationUtils {
/**
* Gets the remote log user dir.
- * @param remoteRootLogDir
- * @param user
+ * @param remoteRootLogDir the aggregated log remote root log dir
+ * @param user the application owner
* @return the remote per user log dir.
*/
public static Path getRemoteLogUserDir(Path remoteRootLogDir, String user) {
@@ -93,7 +93,7 @@ public class LogAggregationUtils {
/**
* Returns the suffix component of the log dir.
- * @param conf
+ * @param conf the configuration
* @return the suffix which will be appended to the user log dir.
*/
public static String getRemoteNodeLogDirSuffix(Configuration conf) {
@@ -104,7 +104,7 @@ public class LogAggregationUtils {
/**
* Converts a nodeId to a form used in the app log file name.
- * @param nodeId
+ * @param nodeId the nodeId
* @return the node string to be used to construct the file name.
*/
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63231a2a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
index 03acb33..0068eae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
@@ -128,8 +128,17 @@ public class LogCLIHelpers implements Configurable {
@VisibleForTesting
public int dumpAContainerLogsForLogType(ContainerLogsRequest options,
boolean outputFailure) throws IOException {
- boolean foundAnyLogs = this.getFileController(options.getAppId(),
- options.getAppOwner()).readAggregatedLogs(options, null);
+ LogAggregationFileController fc = null;
+ try {
+ fc = this.getFileController(
+ options.getAppId(), options.getAppOwner());
+ } catch (IOException ex) {
+ System.err.println(ex);
+ }
+ boolean foundAnyLogs = false;
+ if (fc != null) {
+ foundAnyLogs = fc.readAggregatedLogs(options, null);
+ }
if (!foundAnyLogs) {
if (outputFailure) {
containerLogNotFound(options.getContainerId());
@@ -142,9 +151,17 @@ public class LogCLIHelpers implements Configurable {
@Private
public int dumpAContainerLogsForLogTypeWithoutNodeId(
ContainerLogsRequest options) throws IOException {
- boolean foundAnyLogs = getFileController(options.getAppId(),
- options.getAppOwner()).readAggregatedLogs(
- options, null);
+ LogAggregationFileController fc = null;
+ try {
+ fc = this.getFileController(
+ options.getAppId(), options.getAppOwner());
+ } catch (IOException ex) {
+ System.err.println(ex);
+ }
+ boolean foundAnyLogs = false;
+ if (fc != null) {
+ foundAnyLogs = fc.readAggregatedLogs(options, null);
+ }
if (!foundAnyLogs) {
containerLogNotFound(options.getContainerId());
return -1;
@@ -155,9 +172,17 @@ public class LogCLIHelpers implements Configurable {
@Private
public int dumpAllContainersLogs(ContainerLogsRequest options)
throws IOException {
- boolean foundAnyLogs = getFileController(options.getAppId(),
- options.getAppOwner()).readAggregatedLogs(
- options, null);
+ LogAggregationFileController fc = null;
+ try {
+ fc = this.getFileController(
+ options.getAppId(), options.getAppOwner());
+ } catch (IOException ex) {
+ System.err.println(ex);
+ }
+ boolean foundAnyLogs = false;
+ if (fc != null) {
+ foundAnyLogs = fc.readAggregatedLogs(options, null);
+ }
if (!foundAnyLogs) {
emptyLogDir(LogAggregationUtils.getRemoteAppLogDir(
conf, options.getAppId(), options.getAppOwner())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63231a2a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
index 87344a7..aafdb66 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
@@ -201,6 +201,7 @@ public abstract class LogAggregationFileController {
* Output container log.
* @param logRequest {@link ContainerLogsRequest}
* @param os the output stream
+ * @return true if we can read the aggregated logs successfully
* @throws IOException if we can not access the log file.
*/
public abstract boolean readAggregatedLogs(ContainerLogsRequest logRequest,
@@ -228,9 +229,9 @@ public abstract class LogAggregationFileController {
/**
* Returns the owner of the application.
*
- * @param the aggregatedLog path.
- * @return the application owner.
- * @throws IOException
+ * @param aggregatedLogPath the aggregatedLog path
+ * @return the application owner
+ * @throws IOException if we can not get the application owner
*/
public abstract String getApplicationOwner(Path aggregatedLogPath)
throws IOException;
@@ -239,9 +240,9 @@ public abstract class LogAggregationFileController {
* Returns ACLs for the application. An empty map is returned if no ACLs are
* found.
*
- * @param the aggregatedLog path.
+ * @param aggregatedLogPath the aggregatedLog path.
* @return a map of the Application ACLs.
- * @throws IOException
+ * @throws IOException if we can not get the application acls
*/
public abstract Map<ApplicationAccessType, String> getApplicationAcls(
Path aggregatedLogPath) throws IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63231a2a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java
index d2038e2..aaed538 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java
@@ -172,7 +172,6 @@ public class LogAggregationTFileController
byte[] buf = new byte[65535];
while (nodeFiles != null && nodeFiles.hasNext()) {
final FileStatus thisNodeFile = nodeFiles.next();
- LOG.error(thisNodeFile.getPath().toString());
String nodeName = thisNodeFile.getPath().getName();
if (nodeName.equals(appId + ".har")) {
Path p = new Path("har:///"
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/40] hadoop git commit: HDFS-11546. Federation Router RPC server.
Contributed by Jason Kace and Inigo Goiri.
Posted by in...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e96a2c7f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
index ee6f57d..2875750 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.util.Time;
/**
* In-memory cache/mock of a namenode and file resolver. Stores the most
- * recently updated NN information for each nameservice and block pool. Also
+ * recently updated NN information for each nameservice and block pool. It also
* stores a virtual mount table for resolving global namespace paths to local NN
* paths.
*/
@@ -51,82 +51,93 @@ public class MockResolver
implements ActiveNamenodeResolver, FileSubclusterResolver {
private Map<String, List<? extends FederationNamenodeContext>> resolver =
- new HashMap<String, List<? extends FederationNamenodeContext>>();
- private Map<String, List<RemoteLocation>> locations =
- new HashMap<String, List<RemoteLocation>>();
- private Set<FederationNamespaceInfo> namespaces =
- new HashSet<FederationNamespaceInfo>();
+ new HashMap<>();
+ private Map<String, List<RemoteLocation>> locations = new HashMap<>();
+ private Set<FederationNamespaceInfo> namespaces = new HashSet<>();
private String defaultNamespace = null;
+
public MockResolver(Configuration conf, StateStoreService store) {
this.cleanRegistrations();
}
- public void addLocation(String mount, String nameservice, String location) {
- RemoteLocation remoteLocation = new RemoteLocation(nameservice, location);
- List<RemoteLocation> locationsList = locations.get(mount);
+ public void addLocation(String mount, String nsId, String location) {
+ List<RemoteLocation> locationsList = this.locations.get(mount);
if (locationsList == null) {
- locationsList = new LinkedList<RemoteLocation>();
- locations.put(mount, locationsList);
+ locationsList = new LinkedList<>();
+ this.locations.put(mount, locationsList);
}
+
+ final RemoteLocation remoteLocation = new RemoteLocation(nsId, location);
if (!locationsList.contains(remoteLocation)) {
locationsList.add(remoteLocation);
}
if (this.defaultNamespace == null) {
- this.defaultNamespace = nameservice;
+ this.defaultNamespace = nsId;
}
}
public synchronized void cleanRegistrations() {
- this.resolver =
- new HashMap<String, List<? extends FederationNamenodeContext>>();
- this.namespaces = new HashSet<FederationNamespaceInfo>();
+ this.resolver = new HashMap<>();
+ this.namespaces = new HashSet<>();
}
@Override
public void updateActiveNamenode(
- String ns, InetSocketAddress successfulAddress) {
+ String nsId, InetSocketAddress successfulAddress) {
String address = successfulAddress.getHostName() + ":" +
successfulAddress.getPort();
- String key = ns;
+ String key = nsId;
if (key != null) {
// Update the active entry
@SuppressWarnings("unchecked")
- List<FederationNamenodeContext> iterator =
- (List<FederationNamenodeContext>) resolver.get(key);
- for (FederationNamenodeContext namenode : iterator) {
+ List<FederationNamenodeContext> namenodes =
+ (List<FederationNamenodeContext>) this.resolver.get(key);
+ for (FederationNamenodeContext namenode : namenodes) {
if (namenode.getRpcAddress().equals(address)) {
MockNamenodeContext nn = (MockNamenodeContext) namenode;
nn.setState(FederationNamenodeServiceState.ACTIVE);
break;
}
}
- Collections.sort(iterator, new NamenodePriorityComparator());
+ // This operation modifies the list so we need to be careful
+ synchronized(namenodes) {
+ Collections.sort(namenodes, new NamenodePriorityComparator());
+ }
}
}
@Override
public List<? extends FederationNamenodeContext>
getNamenodesForNameserviceId(String nameserviceId) {
- return resolver.get(nameserviceId);
+ // Return a copy of the list because it is updated periodically
+ List<? extends FederationNamenodeContext> namenodes =
+ this.resolver.get(nameserviceId);
+ return Collections.unmodifiableList(new ArrayList<>(namenodes));
}
@Override
public List<? extends FederationNamenodeContext> getNamenodesForBlockPoolId(
String blockPoolId) {
- return resolver.get(blockPoolId);
+ // Return a copy of the list because it is updated periodically
+ List<? extends FederationNamenodeContext> namenodes =
+ this.resolver.get(blockPoolId);
+ return Collections.unmodifiableList(new ArrayList<>(namenodes));
}
private static class MockNamenodeContext
implements FederationNamenodeContext {
+
+ private String namenodeId;
+ private String nameserviceId;
+
private String webAddress;
private String rpcAddress;
private String serviceAddress;
private String lifelineAddress;
- private String namenodeId;
- private String nameserviceId;
+
private FederationNamenodeServiceState state;
private long dateModified;
@@ -197,6 +208,7 @@ public class MockResolver
@Override
public synchronized boolean registerNamenode(NamenodeStatusReport report)
throws IOException {
+
MockNamenodeContext context = new MockNamenodeContext(
report.getRpcAddress(), report.getServiceAddress(),
report.getLifelineAddress(), report.getWebAddress(),
@@ -205,13 +217,14 @@ public class MockResolver
String nsId = report.getNameserviceId();
String bpId = report.getBlockPoolId();
String cId = report.getClusterId();
+
@SuppressWarnings("unchecked")
List<MockNamenodeContext> existingItems =
- (List<MockNamenodeContext>) resolver.get(nsId);
+ (List<MockNamenodeContext>) this.resolver.get(nsId);
if (existingItems == null) {
- existingItems = new ArrayList<MockNamenodeContext>();
- resolver.put(bpId, existingItems);
- resolver.put(nsId, existingItems);
+ existingItems = new ArrayList<>();
+ this.resolver.put(bpId, existingItems);
+ this.resolver.put(nsId, existingItems);
}
boolean added = false;
for (int i=0; i<existingItems.size() && !added; i++) {
@@ -227,7 +240,7 @@ public class MockResolver
Collections.sort(existingItems, new NamenodePriorityComparator());
FederationNamespaceInfo info = new FederationNamespaceInfo(bpId, cId, nsId);
- namespaces.add(info);
+ this.namespaces.add(info);
return true;
}
@@ -238,16 +251,13 @@ public class MockResolver
@Override
public PathLocation getDestinationForPath(String path) throws IOException {
- String finalPath = null;
- String nameservice = null;
- Set<String> namespaceSet = new HashSet<String>();
- LinkedList<RemoteLocation> remoteLocations =
- new LinkedList<RemoteLocation>();
- for(String key : this.locations.keySet()) {
- if(path.startsWith(key)) {
+ Set<String> namespaceSet = new HashSet<>();
+ List<RemoteLocation> remoteLocations = new LinkedList<>();
+ for (String key : this.locations.keySet()) {
+ if (path.startsWith(key)) {
for (RemoteLocation location : this.locations.get(key)) {
- finalPath = location.getDest() + path.substring(key.length());
- nameservice = location.getNameserviceId();
+ String finalPath = location.getDest() + path.substring(key.length());
+ String nameservice = location.getNameserviceId();
RemoteLocation remoteLocation =
new RemoteLocation(nameservice, finalPath);
remoteLocations.add(remoteLocation);
@@ -265,7 +275,7 @@ public class MockResolver
@Override
public List<String> getMountPoints(String path) throws IOException {
- List<String> mounts = new ArrayList<String>();
+ List<String> mounts = new ArrayList<>();
if (path.equals("/")) {
// Mounts only supported under root level
for (String mount : this.locations.keySet()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e96a2c7f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
index 16d624c..39fcf7a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.federation;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
/**
* Constructs a router configuration with individual features enabled/disabled.
@@ -26,15 +27,32 @@ public class RouterConfigBuilder {
private Configuration conf;
+ private boolean enableRpcServer = false;
+
public RouterConfigBuilder(Configuration configuration) {
this.conf = configuration;
}
public RouterConfigBuilder() {
- this.conf = new Configuration();
+ this.conf = new Configuration(false);
+ }
+
+ public RouterConfigBuilder all() {
+ this.enableRpcServer = true;
+ return this;
+ }
+
+ public RouterConfigBuilder rpc(boolean enable) {
+ this.enableRpcServer = enable;
+ return this;
+ }
+
+ public RouterConfigBuilder rpc() {
+ return this.rpc(true);
}
public Configuration build() {
+ conf.setBoolean(DFSConfigKeys.DFS_ROUTER_RPC_ENABLE, this.enableRpcServer);
return conf;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e96a2c7f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
index 55d04ad..4031b7f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
@@ -17,27 +17,44 @@
*/
package org.apache.hadoop.hdfs.server.federation;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_BIND_HOST_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HANDLER_COUNT_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_RPC_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_RPC_BIND_HOST_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMENODES;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.addDirectory;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.waitNamenodeRegistered;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
import java.io.IOException;
+import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map.Entry;
import java.util.Random;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.hdfs.DFSClient;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -46,16 +63,49 @@ import org.apache.hadoop.hdfs.MiniDFSNNTopology.NNConf;
import org.apache.hadoop.hdfs.MiniDFSNNTopology.NSConf;
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport;
import org.apache.hadoop.hdfs.server.federation.router.Router;
+import org.apache.hadoop.hdfs.server.namenode.FSImage;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.Service.STATE;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
- * Test utility to mimic a federated HDFS cluster with a router.
+ * Test utility to mimic a federated HDFS cluster with multiple routers.
*/
public class RouterDFSCluster {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RouterDFSCluster.class);
+
+ public static final String TEST_STRING = "teststring";
+ public static final String TEST_DIR = "testdir";
+ public static final String TEST_FILE = "testfile";
+
+
+ /** Nameservices in the federated cluster. */
+ private List<String> nameservices;
+ /** Namenodes in the federated cluster. */
+ private List<NamenodeContext> namenodes;
+ /** Routers in the federated cluster. */
+ private List<RouterContext> routers;
+ /** If the Namenodes are in high availability.*/
+ private boolean highAvailability;
+
+ /** Mini cluster. */
+ private MiniDFSCluster cluster;
+
+ /** Router configuration overrides. */
+ private Configuration routerOverrides;
+ /** Namenode configuration overrides. */
+ private Configuration namenodeOverrides;
+
+
/**
* Router context.
*/
@@ -69,13 +119,14 @@ public class RouterDFSCluster {
private Configuration conf;
private URI fileSystemUri;
- public RouterContext(Configuration conf, String ns, String nn)
+ public RouterContext(Configuration conf, String nsId, String nnId)
throws URISyntaxException {
- this.namenodeId = nn;
- this.nameserviceId = ns;
this.conf = conf;
- router = new Router();
- router.init(conf);
+ this.nameserviceId = nsId;
+ this.namenodeId = nnId;
+
+ this.router = new Router();
+ this.router.init(conf);
}
public Router getRouter() {
@@ -99,18 +150,30 @@ public class RouterDFSCluster {
}
public void initRouter() throws URISyntaxException {
+ // Store the bound points for the router interfaces
+ InetSocketAddress rpcAddress = router.getRpcServerAddress();
+ if (rpcAddress != null) {
+ this.rpcPort = rpcAddress.getPort();
+ this.fileSystemUri =
+ URI.create("hdfs://" + NetUtils.getHostPortString(rpcAddress));
+ // Override the default FS to point to the router RPC
+ DistributedFileSystem.setDefaultUri(conf, fileSystemUri);
+ try {
+ this.fileContext = FileContext.getFileContext(conf);
+ } catch (UnsupportedFileSystemException e) {
+ this.fileContext = null;
+ }
+ }
}
- public DistributedFileSystem getFileSystem() throws IOException {
- DistributedFileSystem fs =
- (DistributedFileSystem) DistributedFileSystem.get(conf);
- return fs;
+ public FileSystem getFileSystem() throws IOException {
+ return DistributedFileSystem.get(conf);
}
public DFSClient getClient(UserGroupInformation user)
throws IOException, URISyntaxException, InterruptedException {
- LOG.info("Connecting to router at " + fileSystemUri);
+ LOG.info("Connecting to router at {}", fileSystemUri);
return user.doAs(new PrivilegedExceptionAction<DFSClient>() {
@Override
public DFSClient run() throws IOException {
@@ -120,9 +183,8 @@ public class RouterDFSCluster {
}
public DFSClient getClient() throws IOException, URISyntaxException {
-
if (client == null) {
- LOG.info("Connecting to router at " + fileSystemUri);
+ LOG.info("Connecting to router at {}", fileSystemUri);
client = new DFSClient(fileSystemUri, conf);
}
return client;
@@ -130,9 +192,10 @@ public class RouterDFSCluster {
}
/**
- * Namenode context.
+ * Namenode context in the federated cluster.
*/
public class NamenodeContext {
+ private Configuration conf;
private NameNode namenode;
private String nameserviceId;
private String namenodeId;
@@ -143,14 +206,13 @@ public class RouterDFSCluster {
private int httpPort;
private URI fileSystemUri;
private int index;
- private Configuration conf;
private DFSClient client;
- public NamenodeContext(Configuration conf, String ns, String nn,
- int index) {
+ public NamenodeContext(
+ Configuration conf, String nsId, String nnId, int index) {
this.conf = conf;
- this.namenodeId = nn;
- this.nameserviceId = ns;
+ this.nameserviceId = nsId;
+ this.namenodeId = nnId;
this.index = index;
}
@@ -170,20 +232,19 @@ public class RouterDFSCluster {
return this.fileContext;
}
- public void setNamenode(NameNode n) throws URISyntaxException {
- namenode = n;
+ public void setNamenode(NameNode nn) throws URISyntaxException {
+ this.namenode = nn;
- // Store the bound ports and override the default FS with the local NN's
- // RPC
- rpcPort = n.getNameNodeAddress().getPort();
- servicePort = n.getServiceRpcAddress().getPort();
- lifelinePort = n.getServiceRpcAddress().getPort();
- httpPort = n.getHttpAddress().getPort();
- fileSystemUri = new URI("hdfs://" + namenode.getHostAndPort());
- DistributedFileSystem.setDefaultUri(conf, fileSystemUri);
+ // Store the bound ports and override the default FS with the local NN RPC
+ this.rpcPort = nn.getNameNodeAddress().getPort();
+ this.servicePort = nn.getServiceRpcAddress().getPort();
+ this.lifelinePort = nn.getServiceRpcAddress().getPort();
+ this.httpPort = nn.getHttpAddress().getPort();
+ this.fileSystemUri = new URI("hdfs://" + namenode.getHostAndPort());
+ DistributedFileSystem.setDefaultUri(this.conf, this.fileSystemUri);
try {
- this.fileContext = FileContext.getFileContext(conf);
+ this.fileContext = FileContext.getFileContext(this.conf);
} catch (UnsupportedFileSystemException e) {
this.fileContext = null;
}
@@ -205,10 +266,8 @@ public class RouterDFSCluster {
return namenode.getHttpAddress().getHostName() + ":" + httpPort;
}
- public DistributedFileSystem getFileSystem() throws IOException {
- DistributedFileSystem fs =
- (DistributedFileSystem) DistributedFileSystem.get(conf);
- return fs;
+ public FileSystem getFileSystem() throws IOException {
+ return DistributedFileSystem.get(conf);
}
public void resetClient() {
@@ -218,7 +277,7 @@ public class RouterDFSCluster {
public DFSClient getClient(UserGroupInformation user)
throws IOException, URISyntaxException, InterruptedException {
- LOG.info("Connecting to namenode at " + fileSystemUri);
+ LOG.info("Connecting to namenode at {}", fileSystemUri);
return user.doAs(new PrivilegedExceptionAction<DFSClient>() {
@Override
public DFSClient run() throws IOException {
@@ -229,7 +288,7 @@ public class RouterDFSCluster {
public DFSClient getClient() throws IOException, URISyntaxException {
if (client == null) {
- LOG.info("Connecting to namenode at " + fileSystemUri);
+ LOG.info("Connecting to namenode at {}", fileSystemUri);
client = new DFSClient(fileSystemUri, conf);
}
return client;
@@ -244,36 +303,20 @@ public class RouterDFSCluster {
}
}
- public static final String NAMENODE1 = "nn0";
- public static final String NAMENODE2 = "nn1";
- public static final String NAMENODE3 = "nn2";
- public static final String TEST_STRING = "teststring";
- public static final String TEST_DIR = "testdir";
- public static final String TEST_FILE = "testfile";
-
- private List<String> nameservices;
- private List<RouterContext> routers;
- private List<NamenodeContext> namenodes;
- private static final Log LOG = LogFactory.getLog(RouterDFSCluster.class);
- private MiniDFSCluster cluster;
- private boolean highAvailability;
-
- protected static final int DEFAULT_HEARTBEAT_INTERVAL = 5;
- protected static final int DEFAULT_CACHE_INTERVAL_SEC = 5;
- private Configuration routerOverrides;
- private Configuration namenodeOverrides;
-
- private static final String NAMENODES = NAMENODE1 + "," + NAMENODE2;
-
- public RouterDFSCluster(boolean ha, int numNameservices) {
- this(ha, numNameservices, 2);
- }
-
public RouterDFSCluster(boolean ha, int numNameservices, int numNamenodes) {
this.highAvailability = ha;
configureNameservices(numNameservices, numNamenodes);
}
+ public RouterDFSCluster(boolean ha, int numNameservices) {
+ this(ha, numNameservices, 2);
+ }
+
+ /**
+ * Add configuration settings to override default Router settings.
+ *
+ * @param conf Router configuration overrides.
+ */
public void addRouterOverrides(Configuration conf) {
if (this.routerOverrides == null) {
this.routerOverrides = conf;
@@ -282,6 +325,11 @@ public class RouterDFSCluster {
}
}
+ /**
+ * Add configuration settings to override default Namenode settings.
+ *
+ * @param conf Namenode configuration overrides.
+ */
public void addNamenodeOverrides(Configuration conf) {
if (this.namenodeOverrides == null) {
this.namenodeOverrides = conf;
@@ -290,124 +338,134 @@ public class RouterDFSCluster {
}
}
- public Configuration generateNamenodeConfiguration(
- String defaultNameserviceId) {
- Configuration c = new HdfsConfiguration();
+ /**
+ * Generate the configuration for a client.
+ *
+ * @param nsId Nameservice identifier.
+ * @return New namenode configuration.
+ */
+ public Configuration generateNamenodeConfiguration(String nsId) {
+ Configuration conf = new HdfsConfiguration();
- c.set(DFSConfigKeys.DFS_NAMESERVICES, getNameservicesKey());
- c.set("fs.defaultFS", "hdfs://" + defaultNameserviceId);
+ conf.set(DFS_NAMESERVICES, getNameservicesKey());
+ conf.set(FS_DEFAULT_NAME_KEY, "hdfs://" + nsId);
for (String ns : nameservices) {
if (highAvailability) {
- c.set(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "." + ns, NAMENODES);
+ conf.set(
+ DFS_HA_NAMENODES_KEY_PREFIX + "." + ns,
+ NAMENODES[0] + "," + NAMENODES[1]);
}
for (NamenodeContext context : getNamenodes(ns)) {
String suffix = context.getConfSuffix();
- c.set(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + suffix,
+ conf.set(DFS_NAMENODE_RPC_ADDRESS_KEY + "." + suffix,
"127.0.0.1:" + context.rpcPort);
- c.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY + "." + suffix,
+ conf.set(DFS_NAMENODE_HTTP_ADDRESS_KEY + "." + suffix,
"127.0.0.1:" + context.httpPort);
- c.set(DFSConfigKeys.DFS_NAMENODE_RPC_BIND_HOST_KEY + "." + suffix,
+ conf.set(DFS_NAMENODE_RPC_BIND_HOST_KEY + "." + suffix,
"0.0.0.0");
}
}
- if (namenodeOverrides != null) {
- c.addResource(namenodeOverrides);
+ if (this.namenodeOverrides != null) {
+ conf.addResource(this.namenodeOverrides);
}
- return c;
+ return conf;
}
+ /**
+ * Generate the configuration for a client.
+ *
+ * @return New configuration for a client.
+ */
public Configuration generateClientConfiguration() {
- Configuration conf = new HdfsConfiguration();
- conf.addResource(generateNamenodeConfiguration(getNameservices().get(0)));
+ Configuration conf = new HdfsConfiguration(false);
+ String ns0 = getNameservices().get(0);
+ conf.addResource(generateNamenodeConfiguration(ns0));
return conf;
}
- public Configuration generateRouterConfiguration(String localNameserviceId,
- String localNamenodeId) throws IOException {
- Configuration conf = new HdfsConfiguration();
- conf.addResource(generateNamenodeConfiguration(localNameserviceId));
+ /**
+ * Generate the configuration for a Router.
+ *
+ * @param nsId Nameservice identifier.
+ * @param nnId Namenode identifier.
+ * @return New configuration for a Router.
+ */
+ public Configuration generateRouterConfiguration(String nsId, String nnId) {
+
+ Configuration conf = new HdfsConfiguration(false);
+ conf.addResource(generateNamenodeConfiguration(nsId));
+
+ conf.setInt(DFS_ROUTER_HANDLER_COUNT_KEY, 10);
+ conf.set(DFS_ROUTER_RPC_ADDRESS_KEY, "127.0.0.1:0");
+ conf.set(DFS_ROUTER_RPC_BIND_HOST_KEY, "0.0.0.0");
+
+ conf.set(DFS_ROUTER_DEFAULT_NAMESERVICE, nameservices.get(0));
// Use mock resolver classes
- conf.set(DFSConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS,
- MockResolver.class.getCanonicalName());
- conf.set(DFSConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
- MockResolver.class.getCanonicalName());
+ conf.setClass(FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS,
+ MockResolver.class, ActiveNamenodeResolver.class);
+ conf.setClass(FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
+ MockResolver.class, FileSubclusterResolver.class);
// Set the nameservice ID for the default NN monitor
- conf.set(DFSConfigKeys.DFS_NAMESERVICE_ID, localNameserviceId);
-
- if (localNamenodeId != null) {
- conf.set(DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY, localNamenodeId);
+ conf.set(DFS_NAMESERVICE_ID, nsId);
+ if (nnId != null) {
+ conf.set(DFS_HA_NAMENODE_ID_KEY, nnId);
}
- StringBuilder routerBuilder = new StringBuilder();
- for (String ns : nameservices) {
- for (NamenodeContext context : getNamenodes(ns)) {
- String suffix = context.getConfSuffix();
-
- if (routerBuilder.length() != 0) {
- routerBuilder.append(",");
- }
- routerBuilder.append(suffix);
+ // Add custom overrides if available
+ if (this.routerOverrides != null) {
+ for (Entry<String, String> entry : this.routerOverrides) {
+ String confKey = entry.getKey();
+ String confValue = entry.getValue();
+ conf.set(confKey, confValue);
}
}
-
return conf;
}
public void configureNameservices(int numNameservices, int numNamenodes) {
- nameservices = new ArrayList<String>();
- for (int i = 0; i < numNameservices; i++) {
- nameservices.add("ns" + i);
- }
- namenodes = new ArrayList<NamenodeContext>();
- int index = 0;
- for (String ns : nameservices) {
- Configuration nnConf = generateNamenodeConfiguration(ns);
- if (highAvailability) {
- NamenodeContext context =
- new NamenodeContext(nnConf, ns, NAMENODE1, index);
- namenodes.add(context);
- index++;
-
- if (numNamenodes > 1) {
- context = new NamenodeContext(nnConf, ns, NAMENODE2, index + 1);
- namenodes.add(context);
- index++;
- }
+ this.nameservices = new ArrayList<>();
+ this.namenodes = new ArrayList<>();
- if (numNamenodes > 2) {
- context = new NamenodeContext(nnConf, ns, NAMENODE3, index + 1);
- namenodes.add(context);
- index++;
- }
+ NamenodeContext context = null;
+ int nnIndex = 0;
+ for (int i=0; i<numNameservices; i++) {
+ String ns = "ns" + i;
+ this.nameservices.add("ns" + i);
+ Configuration nnConf = generateNamenodeConfiguration(ns);
+ if (!highAvailability) {
+ context = new NamenodeContext(nnConf, ns, null, nnIndex++);
+ this.namenodes.add(context);
} else {
- NamenodeContext context = new NamenodeContext(nnConf, ns, null, index);
- namenodes.add(context);
- index++;
+ for (int j=0; j<numNamenodes; j++) {
+ context = new NamenodeContext(nnConf, ns, NAMENODES[j], nnIndex++);
+ this.namenodes.add(context);
+ }
}
}
}
public String getNameservicesKey() {
- StringBuilder ns = new StringBuilder();
- for (int i = 0; i < nameservices.size(); i++) {
- if (i > 0) {
- ns.append(",");
+ StringBuilder sb = new StringBuilder();
+ for (String nsId : this.nameservices) {
+ if (sb.length() > 0) {
+ sb.append(",");
}
- ns.append(nameservices.get(i));
+ sb.append(nsId);
}
- return ns.toString();
+ return sb.toString();
}
public String getRandomNameservice() {
Random r = new Random();
- return nameservices.get(r.nextInt(nameservices.size()));
+ int randIndex = r.nextInt(nameservices.size());
+ return nameservices.get(randIndex);
}
public List<String> getNameservices() {
@@ -415,7 +473,7 @@ public class RouterDFSCluster {
}
public List<NamenodeContext> getNamenodes(String nameservice) {
- ArrayList<NamenodeContext> nns = new ArrayList<NamenodeContext>();
+ List<NamenodeContext> nns = new ArrayList<>();
for (NamenodeContext c : namenodes) {
if (c.nameserviceId.equals(nameservice)) {
nns.add(c);
@@ -426,23 +484,23 @@ public class RouterDFSCluster {
public NamenodeContext getRandomNamenode() {
Random rand = new Random();
- return namenodes.get(rand.nextInt(namenodes.size()));
+ int i = rand.nextInt(this.namenodes.size());
+ return this.namenodes.get(i);
}
public List<NamenodeContext> getNamenodes() {
- return namenodes;
+ return this.namenodes;
}
public boolean isHighAvailability() {
return highAvailability;
}
- public NamenodeContext getNamenode(String nameservice,
- String namenode) {
- for (NamenodeContext c : namenodes) {
+ public NamenodeContext getNamenode(String nameservice, String namenode) {
+ for (NamenodeContext c : this.namenodes) {
if (c.nameserviceId.equals(nameservice)) {
- if (namenode == null || c.namenodeId == null || namenode.isEmpty()
- || c.namenodeId.isEmpty()) {
+ if (namenode == null || namenode.isEmpty() ||
+ c.namenodeId == null || c.namenodeId.isEmpty()) {
return c;
} else if (c.namenodeId.equals(namenode)) {
return c;
@@ -453,7 +511,7 @@ public class RouterDFSCluster {
}
public List<RouterContext> getRouters(String nameservice) {
- ArrayList<RouterContext> nns = new ArrayList<RouterContext>();
+ List<RouterContext> nns = new ArrayList<>();
for (RouterContext c : routers) {
if (c.nameserviceId.equals(nameservice)) {
nns.add(c);
@@ -462,14 +520,13 @@ public class RouterDFSCluster {
return nns;
}
- public RouterContext getRouterContext(String nameservice,
- String namenode) {
+ public RouterContext getRouterContext(String nsId, String nnId) {
for (RouterContext c : routers) {
- if (namenode == null) {
+ if (nnId == null) {
return c;
}
- if (c.namenodeId.equals(namenode)
- && c.nameserviceId.equals(nameservice)) {
+ if (c.namenodeId.equals(nnId) &&
+ c.nameserviceId.equals(nsId)) {
return c;
}
}
@@ -485,10 +542,10 @@ public class RouterDFSCluster {
return routers;
}
- public RouterContext buildRouter(String nameservice, String namenode)
+ public RouterContext buildRouter(String nsId, String nnId)
throws URISyntaxException, IOException {
- Configuration config = generateRouterConfiguration(nameservice, namenode);
- RouterContext rc = new RouterContext(config, nameservice, namenode);
+ Configuration config = generateRouterConfiguration(nsId, nnId);
+ RouterContext rc = new RouterContext(config, nsId, nnId);
return rc;
}
@@ -500,10 +557,9 @@ public class RouterDFSCluster {
try {
MiniDFSNNTopology topology = new MiniDFSNNTopology();
for (String ns : nameservices) {
-
NSConf conf = new MiniDFSNNTopology.NSConf(ns);
if (highAvailability) {
- for(int i = 0; i < namenodes.size()/nameservices.size(); i++) {
+ for (int i=0; i<namenodes.size()/nameservices.size(); i++) {
NNConf nnConf = new MiniDFSNNTopology.NNConf("nn" + i);
conf.addNN(nnConf);
}
@@ -516,11 +572,15 @@ public class RouterDFSCluster {
topology.setFederation(true);
// Start mini DFS cluster
- Configuration nnConf = generateNamenodeConfiguration(nameservices.get(0));
+ String ns0 = nameservices.get(0);
+ Configuration nnConf = generateNamenodeConfiguration(ns0);
if (overrideConf != null) {
nnConf.addResource(overrideConf);
}
- cluster = new MiniDFSCluster.Builder(nnConf).nnTopology(topology).build();
+ cluster = new MiniDFSCluster.Builder(nnConf)
+ .numDataNodes(nameservices.size()*2)
+ .nnTopology(topology)
+ .build();
cluster.waitActive();
// Store NN pointers
@@ -530,28 +590,32 @@ public class RouterDFSCluster {
}
} catch (Exception e) {
- LOG.error("Cannot start Router DFS cluster: " + e.getMessage(), e);
- cluster.shutdown();
+ LOG.error("Cannot start Router DFS cluster: {}", e.getMessage(), e);
+ if (cluster != null) {
+ cluster.shutdown();
+ }
}
}
public void startRouters()
throws InterruptedException, URISyntaxException, IOException {
- // Create routers
- routers = new ArrayList<RouterContext>();
- for (String ns : nameservices) {
+ // Create one router per nameservice
+ this.routers = new ArrayList<>();
+ for (String ns : this.nameservices) {
for (NamenodeContext context : getNamenodes(ns)) {
- routers.add(buildRouter(ns, context.namenodeId));
+ RouterContext router = buildRouter(ns, context.namenodeId);
+ this.routers.add(router);
}
}
// Start all routers
- for (RouterContext router : routers) {
+ for (RouterContext router : this.routers) {
router.router.start();
}
+
// Wait until all routers are active and record their ports
- for (RouterContext router : routers) {
+ for (RouterContext router : this.routers) {
waitActive(router);
router.initRouter();
}
@@ -570,22 +634,21 @@ public class RouterDFSCluster {
}
Thread.sleep(1000);
}
- assertFalse(
- "Timeout waiting for " + router.router.toString() + " to activate.",
- true);
+ fail("Timeout waiting for " + router.router + " to activate");
}
-
public void registerNamenodes() throws IOException {
- for (RouterContext r : routers) {
+ for (RouterContext r : this.routers) {
ActiveNamenodeResolver resolver = r.router.getNamenodeResolver();
- for (NamenodeContext nn : namenodes) {
+ for (NamenodeContext nn : this.namenodes) {
// Generate a report
- NamenodeStatusReport report = new NamenodeStatusReport(nn.nameserviceId,
- nn.namenodeId, nn.getRpcAddress(), nn.getServiceAddress(),
+ NamenodeStatusReport report = new NamenodeStatusReport(
+ nn.nameserviceId, nn.namenodeId,
+ nn.getRpcAddress(), nn.getServiceAddress(),
nn.getLifelineAddress(), nn.getHttpAddress());
- report.setNamespaceInfo(nn.namenode.getNamesystem().getFSImage()
- .getStorage().getNamespaceInfo());
+ FSImage fsImage = nn.namenode.getNamesystem().getFSImage();
+ NamespaceInfo nsInfo = fsImage.getStorage().getNamespaceInfo();
+ report.setNamespaceInfo(nsInfo);
// Determine HA state from nn public state string
String nnState = nn.namenode.getState();
@@ -606,74 +669,97 @@ public class RouterDFSCluster {
public void waitNamenodeRegistration()
throws InterruptedException, IllegalStateException, IOException {
- for (RouterContext r : routers) {
- for (NamenodeContext nn : namenodes) {
- FederationTestUtils.waitNamenodeRegistered(
- r.router.getNamenodeResolver(), nn.nameserviceId, nn.namenodeId,
- null);
+ for (RouterContext r : this.routers) {
+ Router router = r.router;
+ for (NamenodeContext nn : this.namenodes) {
+ ActiveNamenodeResolver nnResolver = router.getNamenodeResolver();
+ waitNamenodeRegistered(
+ nnResolver, nn.nameserviceId, nn.namenodeId, null);
}
}
}
public void waitRouterRegistrationQuorum(RouterContext router,
- FederationNamenodeServiceState state, String nameservice, String namenode)
+ FederationNamenodeServiceState state, String nsId, String nnId)
throws InterruptedException, IOException {
- LOG.info("Waiting for NN - " + nameservice + ":" + namenode
- + " to transition to state - " + state);
- FederationTestUtils.waitNamenodeRegistered(
- router.router.getNamenodeResolver(), nameservice, namenode, state);
+ LOG.info("Waiting for NN {} {} to transition to {}", nsId, nnId, state);
+ ActiveNamenodeResolver nnResolver = router.router.getNamenodeResolver();
+ waitNamenodeRegistered(nnResolver, nsId, nnId, state);
}
- public String getFederatedPathForNameservice(String ns) {
- return "/" + ns;
+ /**
+ * Get the federated path for a nameservice.
+ * @param nsId Nameservice identifier.
+ * @return Path in the Router.
+ */
+ public String getFederatedPathForNS(String nsId) {
+ return "/" + nsId;
}
- public String getNamenodePathForNameservice(String ns) {
- return "/target-" + ns;
+ /**
+ * Get the namenode path for a nameservice.
+ * @param nsId Nameservice identifier.
+ * @return Path in the Namenode.
+ */
+ public String getNamenodePathForNS(String nsId) {
+ return "/target-" + nsId;
}
/**
- * @return example:
+ * Get the federated test directory for a nameservice.
+ * @param nsId Nameservice identifier.
+ * @return Example:
* <ul>
* <li>/ns0/testdir which maps to ns0->/target-ns0/testdir
* </ul>
*/
- public String getFederatedTestDirectoryForNameservice(String ns) {
- return getFederatedPathForNameservice(ns) + "/" + TEST_DIR;
+ public String getFederatedTestDirectoryForNS(String nsId) {
+ return getFederatedPathForNS(nsId) + "/" + TEST_DIR;
}
/**
+ * Get the namenode test directory for a nameservice.
+ * @param nsId Nameservice identifier.
* @return example:
* <ul>
* <li>/target-ns0/testdir
* </ul>
*/
- public String getNamenodeTestDirectoryForNameservice(String ns) {
- return getNamenodePathForNameservice(ns) + "/" + TEST_DIR;
+ public String getNamenodeTestDirectoryForNS(String nsId) {
+ return getNamenodePathForNS(nsId) + "/" + TEST_DIR;
}
/**
+ * Get the federated test file for a nameservice.
+ * @param nsId Nameservice identifier.
* @return example:
* <ul>
* <li>/ns0/testfile which maps to ns0->/target-ns0/testfile
* </ul>
*/
- public String getFederatedTestFileForNameservice(String ns) {
- return getFederatedPathForNameservice(ns) + "/" + TEST_FILE;
+ public String getFederatedTestFileForNS(String nsId) {
+ return getFederatedPathForNS(nsId) + "/" + TEST_FILE;
}
/**
+ * Get the namenode test file for a nameservice.
+ * @param nsId Nameservice identifier.
* @return example:
* <ul>
* <li>/target-ns0/testfile
* </ul>
*/
- public String getNamenodeTestFileForNameservice(String ns) {
- return getNamenodePathForNameservice(ns) + "/" + TEST_FILE;
+ public String getNamenodeTestFileForNS(String nsId) {
+ return getNamenodePathForNS(nsId) + "/" + TEST_FILE;
}
+ /**
+ * Stop the federated HDFS cluster.
+ */
public void shutdown() {
- cluster.shutdown();
+ if (cluster != null) {
+ cluster.shutdown();
+ }
if (routers != null) {
for (RouterContext context : routers) {
stopRouter(context);
@@ -681,9 +767,12 @@ public class RouterDFSCluster {
}
}
+ /**
+ * Stop a router.
+ * @param router Router context.
+ */
public void stopRouter(RouterContext router) {
try {
-
router.router.shutDown();
int loopCount = 0;
@@ -691,7 +780,7 @@ public class RouterDFSCluster {
loopCount++;
Thread.sleep(1000);
if (loopCount > 20) {
- LOG.error("Unable to shutdown router - " + router.rpcPort);
+ LOG.error("Cannot shutdown router {}", router.rpcPort);
break;
}
}
@@ -714,26 +803,28 @@ public class RouterDFSCluster {
for (String ns : getNameservices()) {
NamenodeContext context = getNamenode(ns, null);
if (!createTestDirectoriesNamenode(context)) {
- throw new IOException("Unable to create test directory for ns - " + ns);
+ throw new IOException("Cannot create test directory for ns " + ns);
}
}
}
public boolean createTestDirectoriesNamenode(NamenodeContext nn)
throws IOException {
- return FederationTestUtils.addDirectory(nn.getFileSystem(),
- getNamenodeTestDirectoryForNameservice(nn.nameserviceId));
+ FileSystem fs = nn.getFileSystem();
+ String testDir = getNamenodeTestDirectoryForNS(nn.nameserviceId);
+ return addDirectory(fs, testDir);
}
public void deleteAllFiles() throws IOException {
// Delete all files via the NNs and verify
for (NamenodeContext context : getNamenodes()) {
- FileStatus[] status = context.getFileSystem().listStatus(new Path("/"));
- for(int i = 0; i <status.length; i++) {
+ FileSystem fs = context.getFileSystem();
+ FileStatus[] status = fs.listStatus(new Path("/"));
+ for (int i = 0; i <status.length; i++) {
Path p = status[i].getPath();
- context.getFileSystem().delete(p, true);
+ fs.delete(p, true);
}
- status = context.getFileSystem().listStatus(new Path("/"));
+ status = fs.listStatus(new Path("/"));
assertEquals(status.length, 0);
}
}
@@ -754,14 +845,34 @@ public class RouterDFSCluster {
MockResolver resolver =
(MockResolver) r.router.getSubclusterResolver();
// create table entries
- for (String ns : nameservices) {
+ for (String nsId : nameservices) {
// Direct path
- resolver.addLocation(getFederatedPathForNameservice(ns), ns,
- getNamenodePathForNameservice(ns));
+ String routerPath = getFederatedPathForNS(nsId);
+ String nnPath = getNamenodePathForNS(nsId);
+ resolver.addLocation(routerPath, nsId, nnPath);
}
- // Root path goes to both NS1
- resolver.addLocation("/", nameservices.get(0), "/");
+ // Root path points to both first nameservice
+ String ns0 = nameservices.get(0);
+ resolver.addLocation("/", ns0, "/");
+ }
+ }
+
+ public MiniDFSCluster getCluster() {
+ return cluster;
+ }
+
+ /**
+ * Wait until the federated cluster is up and ready.
+ * @throws IOException If we cannot wait for the cluster to be up.
+ */
+ public void waitClusterUp() throws IOException {
+ cluster.waitClusterUp();
+ registerNamenodes();
+ try {
+ waitNamenodeRegistration();
+ } catch (Exception e) {
+ throw new IOException("Cannot wait for the namenodes", e);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e96a2c7f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
index 8c720c7..d8afb39 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.federation.router;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import java.io.IOException;
import java.net.URISyntaxException;
@@ -51,6 +52,10 @@ public class TestRouter {
conf.set(DFSConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
MockResolver.class.getCanonicalName());
+ // Bind to any available port
+ conf.set(DFSConfigKeys.DFS_ROUTER_RPC_BIND_HOST_KEY, "0.0.0.0");
+ conf.set(DFSConfigKeys.DFS_ROUTER_RPC_ADDRESS_KEY, "127.0.0.1:0");
+
// Simulate a co-located NN
conf.set(DFSConfigKeys.DFS_NAMESERVICES, "ns0");
conf.set("fs.defaultFS", "hdfs://" + "ns0");
@@ -90,7 +95,31 @@ public class TestRouter {
@Test
public void testRouterService() throws InterruptedException, IOException {
+ // Rpc only
+ testRouterStartup(new RouterConfigBuilder(conf).rpc().build());
+
// Run with all services
- testRouterStartup((new RouterConfigBuilder(conf)).build());
+ testRouterStartup(new RouterConfigBuilder(conf).all().build());
+ }
+
+ @Test
+ public void testRouterRestartRpcService() throws IOException {
+
+ // Start
+ Router router = new Router();
+ router.init(new RouterConfigBuilder(conf).rpc().build());
+ router.start();
+
+ // Verify RPC server is running
+ assertNotNull(router.getRpcServerAddress());
+ RouterRpcServer rpcServer = router.getRpcServer();
+ assertNotNull(rpcServer);
+ assertEquals(STATE.STARTED, rpcServer.getServiceState());
+
+ // Stop router and RPC server
+ router.stop();
+ assertEquals(STATE.STOPPED, rpcServer.getServiceState());
+
+ router.close();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e96a2c7f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
new file mode 100644
index 0000000..af506c9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
@@ -0,0 +1,869 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.addDirectory;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.countContents;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createFile;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.deleteFile;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.getFileStatus;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyFileExists;
+import static org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.TEST_STRING;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.CryptoProtocolVersion;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Options;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
+import org.apache.hadoop.io.EnumSetWritable;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.Service.STATE;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * The the RPC interface of the {@link Router} implemented by
+ * {@link RouterRpcServer}.
+ */
+public class TestRouterRpc {
+
+ /** Federated HDFS cluster. */
+ private static RouterDFSCluster cluster;
+
+ /** Random Router for this federated cluster. */
+ private RouterContext router;
+
+ /** Random nameservice in the federated cluster. */
+ private String ns;
+ /** First namenode in the nameservice. */
+ private NamenodeContext namenode;
+
+ /** Client interface to the Router. */
+ private ClientProtocol routerProtocol;
+ /** Client interface to the Namenode. */
+ private ClientProtocol nnProtocol;
+
+ /** Filesystem interface to the Router. */
+ private FileSystem routerFS;
+ /** Filesystem interface to the Namenode. */
+ private FileSystem nnFS;
+
+ /** File in the Router. */
+ private String routerFile;
+ /** File in the Namenode. */
+ private String nnFile;
+
+
+ @BeforeClass
+ public static void globalSetUp() throws Exception {
+ cluster = new RouterDFSCluster(false, 2);
+
+ // Start NNs and DNs and wait until ready
+ cluster.startCluster();
+
+ // Start routers with only an RPC service
+ cluster.addRouterOverrides((new RouterConfigBuilder()).rpc().build());
+ cluster.startRouters();
+
+ // Register and verify all NNs with all routers
+ cluster.registerNamenodes();
+ cluster.waitNamenodeRegistration();
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ cluster.shutdown();
+ }
+
+ @Before
+ public void testSetup() throws Exception {
+
+ // Create mock locations
+ cluster.installMockLocations();
+
+ // Delete all files via the NNs and verify
+ cluster.deleteAllFiles();
+
+ // Create test fixtures on NN
+ cluster.createTestDirectoriesNamenode();
+
+ // Wait to ensure NN has fully created its test directories
+ Thread.sleep(100);
+
+ // Pick a NS, namenode and router for this test
+ this.router = cluster.getRandomRouter();
+ this.ns = cluster.getRandomNameservice();
+ this.namenode = cluster.getNamenode(ns, null);
+
+ // Handles to the ClientProtocol interface
+ this.routerProtocol = router.getClient().getNamenode();
+ this.nnProtocol = namenode.getClient().getNamenode();
+
+ // Handles to the filesystem client
+ this.nnFS = namenode.getFileSystem();
+ this.routerFS = router.getFileSystem();
+
+ // Create a test file on the NN
+ Random r = new Random();
+ String randomFile = "testfile-" + r.nextInt();
+ this.nnFile =
+ cluster.getNamenodeTestDirectoryForNS(ns) + "/" + randomFile;
+ this.routerFile =
+ cluster.getFederatedTestDirectoryForNS(ns) + "/" + randomFile;
+
+ createFile(nnFS, nnFile, 32);
+ verifyFileExists(nnFS, nnFile);
+ }
+
+ @Test
+ public void testRpcService() throws IOException {
+ Router testRouter = new Router();
+ List<String> nss = cluster.getNameservices();
+ String ns0 = nss.get(0);
+ Configuration routerConfig = cluster.generateRouterConfiguration(ns0, null);
+ RouterRpcServer server = new RouterRpcServer(routerConfig, testRouter,
+ testRouter.getNamenodeResolver(), testRouter.getSubclusterResolver());
+ server.init(routerConfig);
+ assertEquals(STATE.INITED, server.getServiceState());
+ server.start();
+ assertEquals(STATE.STARTED, server.getServiceState());
+ server.stop();
+ assertEquals(STATE.STOPPED, server.getServiceState());
+ server.close();
+ testRouter.close();
+ }
+
+ protected RouterDFSCluster getCluster() {
+ return TestRouterRpc.cluster;
+ }
+
+ protected RouterContext getRouterContext() {
+ return this.router;
+ }
+
+ protected void setRouter(RouterContext r)
+ throws IOException, URISyntaxException {
+ this.router = r;
+ this.routerProtocol = r.getClient().getNamenode();
+ this.routerFS = r.getFileSystem();
+ }
+
+ protected FileSystem getRouterFileSystem() {
+ return this.routerFS;
+ }
+
+ protected FileSystem getNamenodeFileSystem() {
+ return this.nnFS;
+ }
+
+ protected ClientProtocol getRouterProtocol() {
+ return this.routerProtocol;
+ }
+
+ protected ClientProtocol getNamenodeProtocol() {
+ return this.nnProtocol;
+ }
+
+ protected NamenodeContext getNamenode() {
+ return this.namenode;
+ }
+
+ protected void setNamenodeFile(String filename) {
+ this.nnFile = filename;
+ }
+
+ protected String getNamenodeFile() {
+ return this.nnFile;
+ }
+
+ protected void setRouterFile(String filename) {
+ this.routerFile = filename;
+ }
+
+ protected String getRouterFile() {
+ return this.routerFile;
+ }
+
+ protected void setNamenode(NamenodeContext nn)
+ throws IOException, URISyntaxException {
+ this.namenode = nn;
+ this.nnProtocol = nn.getClient().getNamenode();
+ this.nnFS = nn.getFileSystem();
+ }
+
+ protected String getNs() {
+ return this.ns;
+ }
+
+ protected void setNs(String nameservice) {
+ this.ns = nameservice;
+ }
+
+ protected static void compareResponses(
+ ClientProtocol protocol1, ClientProtocol protocol2,
+ Method m, Object[] paramList) {
+
+ Object return1 = null;
+ Exception exception1 = null;
+ try {
+ return1 = m.invoke(protocol1, paramList);
+ } catch (Exception ex) {
+ exception1 = ex;
+ }
+
+ Object return2 = null;
+ Exception exception2 = null;
+ try {
+ return2 = m.invoke(protocol2, paramList);
+ } catch (Exception ex) {
+ exception2 = ex;
+ }
+
+ assertEquals(return1, return2);
+ if (exception1 == null && exception2 == null) {
+ return;
+ }
+
+ assertEquals(
+ exception1.getCause().getClass(),
+ exception2.getCause().getClass());
+ }
+
+ @Test
+ public void testProxyListFiles() throws IOException, InterruptedException,
+ URISyntaxException, NoSuchMethodException, SecurityException {
+
+ // Verify that the root listing is a union of the mount table destinations
+ // and the files stored at all nameservices mounted at the root (ns0 + ns1)
+ //
+ // / -->
+ // /ns0 (from mount table)
+ // /ns1 (from mount table)
+ // all items in / of ns0 (default NS)
+
+ // Collect the mount table entries from the root mount point
+ Set<String> requiredPaths = new TreeSet<>();
+ FileSubclusterResolver fileResolver =
+ router.getRouter().getSubclusterResolver();
+ for (String mount : fileResolver.getMountPoints("/")) {
+ requiredPaths.add(mount);
+ }
+
+ // Collect all files/dirs on the root path of the default NS
+ String defaultNs = cluster.getNameservices().get(0);
+ NamenodeContext nn = cluster.getNamenode(defaultNs, null);
+ FileStatus[] iterator = nn.getFileSystem().listStatus(new Path("/"));
+ for (FileStatus file : iterator) {
+ requiredPaths.add(file.getPath().getName());
+ }
+
+ // Fetch listing
+ DirectoryListing listing =
+ routerProtocol.getListing("/", HdfsFileStatus.EMPTY_NAME, false);
+ Iterator<String> requiredPathsIterator = requiredPaths.iterator();
+ // Match each path returned and verify order returned
+ for(HdfsFileStatus f : listing.getPartialListing()) {
+ String fileName = requiredPathsIterator.next();
+ String currentFile = f.getFullPath(new Path("/")).getName();
+ assertEquals(currentFile, fileName);
+ }
+
+ // Verify the total number of results found/matched
+ assertEquals(requiredPaths.size(), listing.getPartialListing().length);
+
+ // List a path that doesn't exist and validate error response with NN
+ // behavior.
+ Method m = ClientProtocol.class.getMethod(
+ "getListing", String.class, byte[].class, boolean.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(routerProtocol, nnProtocol, m,
+ new Object[] {badPath, HdfsFileStatus.EMPTY_NAME, false});
+ }
+
+ @Test
+ public void testProxyListFilesWithConflict()
+ throws IOException, InterruptedException {
+
+ // Add a directory to the namespace that conflicts with a mount point
+ NamenodeContext nn = cluster.getNamenode(ns, null);
+ FileSystem nnFs = nn.getFileSystem();
+ addDirectory(nnFs, cluster.getFederatedTestDirectoryForNS(ns));
+
+ FileSystem routerFs = router.getFileSystem();
+ int initialCount = countContents(routerFs, "/");
+
+ // Root file system now for NS X:
+ // / ->
+ // /ns0 (mount table)
+ // /ns1 (mount table)
+ // /target-ns0 (the target folder for the NS0 mapped to /
+ // /nsX (local directory that duplicates mount table)
+ int newCount = countContents(routerFs, "/");
+ assertEquals(initialCount, newCount);
+
+ // Verify that each root path is readable and contains one test directory
+ assertEquals(1, countContents(routerFs, cluster.getFederatedPathForNS(ns)));
+
+ // Verify that real folder for the ns contains a single test directory
+ assertEquals(1, countContents(nnFs, cluster.getNamenodePathForNS(ns)));
+
+ }
+
+ protected void testRename(RouterContext testRouter, String filename,
+ String renamedFile, boolean exceptionExpected) throws IOException {
+
+ createFile(testRouter.getFileSystem(), filename, 32);
+ // verify
+ verifyFileExists(testRouter.getFileSystem(), filename);
+ // rename
+ boolean exceptionThrown = false;
+ try {
+ DFSClient client = testRouter.getClient();
+ ClientProtocol clientProtocol = client.getNamenode();
+ clientProtocol.rename(filename, renamedFile);
+ } catch (Exception ex) {
+ exceptionThrown = true;
+ }
+ if (exceptionExpected) {
+ // Error was expected
+ assertTrue(exceptionThrown);
+ FileContext fileContext = testRouter.getFileContext();
+ assertTrue(fileContext.delete(new Path(filename), true));
+ } else {
+ // No error was expected
+ assertFalse(exceptionThrown);
+ // verify
+ assertTrue(verifyFileExists(testRouter.getFileSystem(), renamedFile));
+ // delete
+ FileContext fileContext = testRouter.getFileContext();
+ assertTrue(fileContext.delete(new Path(renamedFile), true));
+ }
+ }
+
+ protected void testRename2(RouterContext testRouter, String filename,
+ String renamedFile, boolean exceptionExpected) throws IOException {
+ createFile(testRouter.getFileSystem(), filename, 32);
+ // verify
+ verifyFileExists(testRouter.getFileSystem(), filename);
+ // rename
+ boolean exceptionThrown = false;
+ try {
+ DFSClient client = testRouter.getClient();
+ ClientProtocol clientProtocol = client.getNamenode();
+ clientProtocol.rename2(filename, renamedFile, new Options.Rename[] {});
+ } catch (Exception ex) {
+ exceptionThrown = true;
+ }
+ assertEquals(exceptionExpected, exceptionThrown);
+ if (exceptionExpected) {
+ // Error was expected
+ FileContext fileContext = testRouter.getFileContext();
+ assertTrue(fileContext.delete(new Path(filename), true));
+ } else {
+ // verify
+ assertTrue(verifyFileExists(testRouter.getFileSystem(), renamedFile));
+ // delete
+ FileContext fileContext = testRouter.getFileContext();
+ assertTrue(fileContext.delete(new Path(renamedFile), true));
+ }
+ }
+
+ @Test
+ public void testProxyRenameFiles() throws IOException, InterruptedException {
+
+ Thread.sleep(5000);
+ List<String> nss = cluster.getNameservices();
+ String ns0 = nss.get(0);
+ String ns1 = nss.get(1);
+
+ // Rename within the same namespace
+ // /ns0/testdir/testrename -> /ns0/testdir/testrename-append
+ String filename =
+ cluster.getFederatedTestDirectoryForNS(ns0) + "/testrename";
+ String renamedFile = filename + "-append";
+ testRename(router, filename, renamedFile, false);
+ testRename2(router, filename, renamedFile, false);
+
+ // Rename a file to a destination that is in a different namespace (fails)
+ filename = cluster.getFederatedTestDirectoryForNS(ns0) + "/testrename";
+ renamedFile = cluster.getFederatedTestDirectoryForNS(ns1) + "/testrename";
+ testRename(router, filename, renamedFile, true);
+ testRename2(router, filename, renamedFile, true);
+ }
+
+ @Test
+ public void testProxyChownFiles() throws Exception {
+
+ String newUsername = "TestUser";
+ String newGroup = "TestGroup";
+
+ // change owner
+ routerProtocol.setOwner(routerFile, newUsername, newGroup);
+
+ // Verify with NN
+ FileStatus file = getFileStatus(namenode.getFileSystem(), nnFile);
+ assertEquals(file.getOwner(), newUsername);
+ assertEquals(file.getGroup(), newGroup);
+
+ // Bad request and validate router response matches NN response.
+ Method m = ClientProtocol.class.getMethod("setOwner", String.class,
+ String.class, String.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(routerProtocol, nnProtocol, m,
+ new Object[] {badPath, newUsername, newGroup});
+ }
+
+ @Test
+ public void testProxyGetStats() throws Exception {
+
+ long[] combinedData = routerProtocol.getStats();
+
+ long[] individualData = new long[10];
+ for (String nameservice : cluster.getNameservices()) {
+ NamenodeContext n = cluster.getNamenode(nameservice, null);
+ DFSClient client = n.getClient();
+ ClientProtocol clientProtocol = client.getNamenode();
+ long[] data = clientProtocol.getStats();
+ for (int i = 0; i < data.length; i++) {
+ individualData[i] += data[i];
+ }
+ assert(data.length == combinedData.length);
+ }
+
+ for (int i = 0; i < combinedData.length && i < individualData.length; i++) {
+ if (i == ClientProtocol.GET_STATS_REMAINING_IDX) {
+ // Skip available storage as this fluctuates in mini cluster
+ continue;
+ }
+ assertEquals(combinedData[i], individualData[i]);
+ }
+ }
+
+ @Test
+ public void testProxyGetDatanodeReport() throws Exception {
+
+ DatanodeInfo[] combinedData =
+ routerProtocol.getDatanodeReport(DatanodeReportType.ALL);
+
+ Set<Integer> individualData = new HashSet<Integer>();
+ for (String nameservice : cluster.getNameservices()) {
+ NamenodeContext n = cluster.getNamenode(nameservice, null);
+ DFSClient client = n.getClient();
+ ClientProtocol clientProtocol = client.getNamenode();
+ DatanodeInfo[] data =
+ clientProtocol.getDatanodeReport(DatanodeReportType.ALL);
+ for (int i = 0; i < data.length; i++) {
+ // Collect unique DNs based on their xfer port
+ DatanodeInfo info = data[i];
+ individualData.add(info.getXferPort());
+ }
+ }
+ assertEquals(combinedData.length, individualData.size());
+ }
+
+ @Test
+ public void testProxyGetDatanodeStorageReport()
+ throws IOException, InterruptedException, URISyntaxException {
+
+ DatanodeStorageReport[] combinedData =
+ routerProtocol.getDatanodeStorageReport(DatanodeReportType.ALL);
+
+ Set<String> individualData = new HashSet<>();
+ for (String nameservice : cluster.getNameservices()) {
+ NamenodeContext n = cluster.getNamenode(nameservice, null);
+ DFSClient client = n.getClient();
+ ClientProtocol clientProtocol = client.getNamenode();
+ DatanodeStorageReport[] data =
+ clientProtocol.getDatanodeStorageReport(DatanodeReportType.ALL);
+ for (DatanodeStorageReport report : data) {
+ // Determine unique DN instances
+ DatanodeInfo dn = report.getDatanodeInfo();
+ individualData.add(dn.toString());
+ }
+ }
+ assertEquals(combinedData.length, individualData.size());
+ }
+
+ @Test
+ public void testProxyMkdir() throws Exception {
+
+ // Check the initial folders
+ FileStatus[] filesInitial = routerFS.listStatus(new Path("/"));
+
+ // Create a directory via the router at the root level
+ String dirPath = "/testdir";
+ FsPermission permission = new FsPermission("705");
+ routerProtocol.mkdirs(dirPath, permission, false);
+
+ // Verify the root listing has the item via the router
+ FileStatus[] files = routerFS.listStatus(new Path("/"));
+ assertEquals(Arrays.toString(files) + " should be " +
+ Arrays.toString(filesInitial) + " + " + dirPath,
+ filesInitial.length + 1, files.length);
+ assertTrue(verifyFileExists(routerFS, dirPath));
+
+ // Verify the directory is present in only 1 Namenode
+ int foundCount = 0;
+ for (NamenodeContext n : cluster.getNamenodes()) {
+ if (verifyFileExists(n.getFileSystem(), dirPath)) {
+ foundCount++;
+ }
+ }
+ assertEquals(1, foundCount);
+ assertTrue(deleteFile(routerFS, dirPath));
+
+ // Validate router failure response matches NN failure response.
+ Method m = ClientProtocol.class.getMethod("mkdirs", String.class,
+ FsPermission.class, boolean.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(routerProtocol, nnProtocol, m,
+ new Object[] {badPath, permission, false});
+ }
+
+ @Test
+ public void testProxyChmodFiles() throws Exception {
+
+ FsPermission permission = new FsPermission("444");
+
+ // change permissions
+ routerProtocol.setPermission(routerFile, permission);
+
+ // Validate permissions NN
+ FileStatus file = getFileStatus(namenode.getFileSystem(), nnFile);
+ assertEquals(permission, file.getPermission());
+
+ // Validate router failure response matches NN failure response.
+ Method m = ClientProtocol.class.getMethod(
+ "setPermission", String.class, FsPermission.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(routerProtocol, nnProtocol, m,
+ new Object[] {badPath, permission});
+ }
+
+ @Test
+ public void testProxySetReplication() throws Exception {
+
+ // Check current replication via NN
+ FileStatus file = getFileStatus(nnFS, nnFile);
+ assertEquals(1, file.getReplication());
+
+ // increment replication via router
+ routerProtocol.setReplication(routerFile, (short) 2);
+
+ // Verify via NN
+ file = getFileStatus(nnFS, nnFile);
+ assertEquals(2, file.getReplication());
+
+ // Validate router failure response matches NN failure response.
+ Method m = ClientProtocol.class.getMethod(
+ "setReplication", String.class, short.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(routerProtocol, nnProtocol, m,
+ new Object[] {badPath, (short) 2});
+ }
+
+ @Test
+ public void testProxyTruncateFile() throws Exception {
+
+ // Check file size via NN
+ FileStatus file = getFileStatus(nnFS, nnFile);
+ assertTrue(file.getLen() > 0);
+
+ // Truncate to 0 bytes via router
+ routerProtocol.truncate(routerFile, 0, "testclient");
+
+ // Verify via NN
+ file = getFileStatus(nnFS, nnFile);
+ assertEquals(0, file.getLen());
+
+ // Validate router failure response matches NN failure response.
+ Method m = ClientProtocol.class.getMethod(
+ "truncate", String.class, long.class, String.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(routerProtocol, nnProtocol, m,
+ new Object[] {badPath, (long) 0, "testclient"});
+ }
+
+ @Test
+ public void testProxyGetBlockLocations() throws Exception {
+
+ // Fetch block locations via router
+ LocatedBlocks locations =
+ routerProtocol.getBlockLocations(routerFile, 0, 1024);
+ assertEquals(1, locations.getLocatedBlocks().size());
+
+ // Validate router failure response matches NN failure response.
+ Method m = ClientProtocol.class.getMethod(
+ "getBlockLocations", String.class, long.class, long.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(routerProtocol, nnProtocol,
+ m, new Object[] {badPath, (long) 0, (long) 0});
+ }
+
+ @Test
+ public void testProxyStoragePolicy() throws Exception {
+
+ // Query initial policy via NN
+ HdfsFileStatus status = namenode.getClient().getFileInfo(nnFile);
+
+ // Set a random policy via router
+ BlockStoragePolicy[] policies = namenode.getClient().getStoragePolicies();
+ BlockStoragePolicy policy = policies[0];
+
+ while (policy.isCopyOnCreateFile()) {
+ // Pick a non copy on create policy
+ Random rand = new Random();
+ int randIndex = rand.nextInt(policies.length);
+ policy = policies[randIndex];
+ }
+ routerProtocol.setStoragePolicy(routerFile, policy.getName());
+
+ // Verify policy via NN
+ HdfsFileStatus newStatus = namenode.getClient().getFileInfo(nnFile);
+ assertTrue(newStatus.getStoragePolicy() == policy.getId());
+ assertTrue(newStatus.getStoragePolicy() != status.getStoragePolicy());
+
+ // Validate router failure response matches NN failure response.
+ Method m = ClientProtocol.class.getMethod("setStoragePolicy", String.class,
+ String.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(routerProtocol, nnProtocol,
+ m, new Object[] {badPath, "badpolicy"});
+ }
+
+ @Test
+ public void testProxyGetPreferedBlockSize() throws Exception {
+
+ // Query via NN and Router and verify
+ long namenodeSize = nnProtocol.getPreferredBlockSize(nnFile);
+ long routerSize = routerProtocol.getPreferredBlockSize(routerFile);
+ assertEquals(routerSize, namenodeSize);
+
+ // Validate router failure response matches NN failure response.
+ Method m = ClientProtocol.class.getMethod(
+ "getPreferredBlockSize", String.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(
+ routerProtocol, nnProtocol, m, new Object[] {badPath});
+ }
+
+ private void testConcat(
+ String source, String target, boolean failureExpected) {
+ boolean failure = false;
+ try {
+ // Concat test file with fill block length file via router
+ routerProtocol.concat(target, new String[] {source});
+ } catch (IOException ex) {
+ failure = true;
+ }
+ assertEquals(failureExpected, failure);
+ }
+
+ @Test
+ public void testProxyConcatFile() throws Exception {
+
+ // Create a stub file in the primary ns
+ String sameNameservice = ns;
+ String existingFile =
+ cluster.getFederatedTestDirectoryForNS(sameNameservice) +
+ "_concatfile";
+ int existingFileSize = 32;
+ createFile(routerFS, existingFile, existingFileSize);
+
+ // Identify an alternate nameservice that doesn't match the existing file
+ String alternateNameservice = null;
+ for (String n : cluster.getNameservices()) {
+ if (!n.equals(sameNameservice)) {
+ alternateNameservice = n;
+ break;
+ }
+ }
+
+ // Create new files, must be a full block to use concat. One file is in the
+ // same namespace as the target file, the other is in a different namespace.
+ String altRouterFile =
+ cluster.getFederatedTestDirectoryForNS(alternateNameservice) +
+ "_newfile";
+ String sameRouterFile =
+ cluster.getFederatedTestDirectoryForNS(sameNameservice) +
+ "_newfile";
+ createFile(routerFS, altRouterFile, DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
+ createFile(routerFS, sameRouterFile, DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
+
+ // Concat in different namespaces, fails
+ testConcat(existingFile, altRouterFile, true);
+
+ // Concat in same namespaces, succeeds
+ testConcat(existingFile, sameRouterFile, false);
+
+ // Check target file length
+ FileStatus status = getFileStatus(routerFS, sameRouterFile);
+ assertEquals(
+ existingFileSize + DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT,
+ status.getLen());
+
+ // Validate router failure response matches NN failure response.
+ Method m = ClientProtocol.class.getMethod(
+ "concat", String.class, String[].class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(routerProtocol, nnProtocol, m,
+ new Object[] {badPath, new String[] {routerFile}});
+ }
+
+ @Test
+ public void testProxyAppend() throws Exception {
+
+ // Append a test string via router
+ EnumSet<CreateFlag> createFlag = EnumSet.of(CreateFlag.APPEND);
+ DFSClient routerClient = getRouterContext().getClient();
+ HdfsDataOutputStream stream =
+ routerClient.append(routerFile, 1024, createFlag, null, null);
+ stream.writeBytes(TEST_STRING);
+ stream.close();
+
+ // Verify file size via NN
+ FileStatus status = getFileStatus(nnFS, nnFile);
+ assertTrue(status.getLen() > TEST_STRING.length());
+
+ // Validate router failure response matches NN failure response.
+ Method m = ClientProtocol.class.getMethod("append", String.class,
+ String.class, EnumSetWritable.class);
+ String badPath = "/unknownlocation/unknowndir";
+ EnumSetWritable<CreateFlag> createFlagWritable =
+ new EnumSetWritable<CreateFlag>(createFlag);
+ compareResponses(routerProtocol, nnProtocol, m,
+ new Object[] {badPath, "testClient", createFlagWritable});
+ }
+
+ @Test
+ public void testProxyGetAdditionalDatanode()
+ throws IOException, InterruptedException, URISyntaxException {
+
+ // Use primitive APIs to open a file, add a block, and get datanode location
+ EnumSet<CreateFlag> createFlag = EnumSet.of(CreateFlag.CREATE);
+ String clientName = getRouterContext().getClient().getClientName();
+ String newRouterFile = routerFile + "_additionalDatanode";
+ HdfsFileStatus status = routerProtocol.create(
+ newRouterFile, new FsPermission("777"), clientName,
+ new EnumSetWritable<CreateFlag>(createFlag), true, (short) 1,
+ (long) 1024, CryptoProtocolVersion.supported(), null);
+
+ // Add a block via router (requires client to have same lease)
+ LocatedBlock block = routerProtocol.addBlock(
+ newRouterFile, clientName, null, null,
+ status.getFileId(), null, null);
+
+ DatanodeInfo[] exclusions = new DatanodeInfo[0];
+ LocatedBlock newBlock = routerProtocol.getAdditionalDatanode(
+ newRouterFile, status.getFileId(), block.getBlock(),
+ block.getLocations(), block.getStorageIDs(), exclusions, 1, clientName);
+ assertNotNull(newBlock);
+ }
+
+ @Test
+ public void testProxyCreateFileAlternateUser()
+ throws IOException, URISyntaxException, InterruptedException {
+
+ // Create via Router
+ String routerDir = cluster.getFederatedTestDirectoryForNS(ns);
+ String namenodeDir = cluster.getNamenodeTestDirectoryForNS(ns);
+ String newRouterFile = routerDir + "/unknownuser";
+ String newNamenodeFile = namenodeDir + "/unknownuser";
+ String username = "unknownuser";
+
+ // Allow all user access to dir
+ namenode.getFileContext().setPermission(
+ new Path(namenodeDir), new FsPermission("777"));
+
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser(username);
+ DFSClient client = getRouterContext().getClient(ugi);
+ client.create(newRouterFile, true);
+
+ // Fetch via NN and check user
+ FileStatus status = getFileStatus(nnFS, newNamenodeFile);
+ assertEquals(status.getOwner(), username);
+ }
+
+ @Test
+ public void testProxyGetFileInfoAcessException() throws IOException {
+
+ UserGroupInformation ugi =
+ UserGroupInformation.createRemoteUser("unknownuser");
+
+ // List files from the NN and trap the exception
+ Exception nnFailure = null;
+ try {
+ String testFile = cluster.getNamenodeTestFileForNS(ns);
+ namenode.getClient(ugi).getLocatedBlocks(testFile, 0);
+ } catch (Exception e) {
+ nnFailure = e;
+ }
+ assertNotNull(nnFailure);
+
+ // List files from the router and trap the exception
+ Exception routerFailure = null;
+ try {
+ String testFile = cluster.getFederatedTestFileForNS(ns);
+ getRouterContext().getClient(ugi).getLocatedBlocks(testFile, 0);
+ } catch (Exception e) {
+ routerFailure = e;
+ }
+ assertNotNull(routerFailure);
+
+ assertEquals(routerFailure.getClass(), nnFailure.getClass());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e96a2c7f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
new file mode 100644
index 0000000..5489691
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createFile;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyFileExists;
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.server.federation.MockResolver;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+
+/**
+ * The the RPC interface of the {@link getRouter()} implemented by
+ * {@link RouterRpcServer}.
+ */
+public class TestRouterRpcMultiDestination extends TestRouterRpc {
+
+ @Override
+ public void testSetup() throws Exception {
+
+ RouterDFSCluster cluster = getCluster();
+
+ // Create mock locations
+ getCluster().installMockLocations();
+ List<RouterContext> routers = cluster.getRouters();
+
+ // Add extra location to the root mount / such that the root mount points:
+ // /
+ // ns0 -> /
+ // ns1 -> /
+ for (RouterContext rc : routers) {
+ Router router = rc.getRouter();
+ MockResolver resolver = (MockResolver) router.getSubclusterResolver();
+ resolver.addLocation("/", cluster.getNameservices().get(1), "/");
+ }
+
+ // Create a mount that points to 2 dirs in the same ns:
+ // /same
+ // ns0 -> /
+ // ns0 -> /target-ns0
+ for (RouterContext rc : routers) {
+ Router router = rc.getRouter();
+ MockResolver resolver = (MockResolver) router.getSubclusterResolver();
+ List<String> nss = cluster.getNameservices();
+ String ns0 = nss.get(0);
+ resolver.addLocation("/same", ns0, "/");
+ resolver.addLocation("/same", ns0, cluster.getNamenodePathForNS(ns0));
+ }
+
+ // Delete all files via the NNs and verify
+ cluster.deleteAllFiles();
+
+ // Create test fixtures on NN
+ cluster.createTestDirectoriesNamenode();
+
+ // Wait to ensure NN has fully created its test directories
+ Thread.sleep(100);
+
+ // Pick a NS, namenode and getRouter() for this test
+ RouterContext router = cluster.getRandomRouter();
+ this.setRouter(router);
+
+ String ns = cluster.getRandomNameservice();
+ this.setNs(ns);
+ this.setNamenode(cluster.getNamenode(ns, null));
+
+ // Create a test file on a single NN that is accessed via a getRouter() path
+ // with 2 destinations. All tests should failover to the alternate
+ // destination if the wrong NN is attempted first.
+ Random r = new Random();
+ String randomString = "testfile-" + r.nextInt();
+ setNamenodeFile("/" + randomString);
+ setRouterFile("/" + randomString);
+
+ FileSystem nnFs = getNamenodeFileSystem();
+ FileSystem routerFs = getRouterFileSystem();
+ createFile(nnFs, getNamenodeFile(), 32);
+
+ verifyFileExists(nnFs, getNamenodeFile());
+ verifyFileExists(routerFs, getRouterFile());
+ }
+
+ private void testListing(String path) throws IOException {
+
+ // Collect the mount table entries for this path
+ Set<String> requiredPaths = new TreeSet<>();
+ RouterContext rc = getRouterContext();
+ Router router = rc.getRouter();
+ FileSubclusterResolver subclusterResolver = router.getSubclusterResolver();
+ for (String mount : subclusterResolver.getMountPoints(path)) {
+ requiredPaths.add(mount);
+ }
+
+ // Get files/dirs from the Namenodes
+ PathLocation location = subclusterResolver.getDestinationForPath(path);
+ for (RemoteLocation loc : location.getDestinations()) {
+ String nsId = loc.getNameserviceId();
+ String dest = loc.getDest();
+ NamenodeContext nn = getCluster().getNamenode(nsId, null);
+ FileSystem fs = nn.getFileSystem();
+ FileStatus[] files = fs.listStatus(new Path(dest));
+ for (FileStatus file : files) {
+ String pathName = file.getPath().getName();
+ requiredPaths.add(pathName);
+ }
+ }
+
+ // Get files/dirs from the Router
+ DirectoryListing listing =
+ getRouterProtocol().getListing(path, HdfsFileStatus.EMPTY_NAME, false);
+ Iterator<String> requiredPathsIterator = requiredPaths.iterator();
+
+ // Match each path returned and verify order returned
+ HdfsFileStatus[] partialListing = listing.getPartialListing();
+ for (HdfsFileStatus fileStatus : listing.getPartialListing()) {
+ String fileName = requiredPathsIterator.next();
+ String currentFile = fileStatus.getFullPath(new Path(path)).getName();
+ assertEquals(currentFile, fileName);
+ }
+
+ // Verify the total number of results found/matched
+ assertEquals(
+ requiredPaths + " doesn't match " + Arrays.toString(partialListing),
+ requiredPaths.size(), partialListing.length);
+ }
+
+ @Override
+ public void testProxyListFiles() throws IOException, InterruptedException,
+ URISyntaxException, NoSuchMethodException, SecurityException {
+
+ // Verify that the root listing is a union of the mount table destinations
+ // and the files stored at all nameservices mounted at the root (ns0 + ns1)
+ // / -->
+ // /ns0 (from mount table)
+ // /ns1 (from mount table)
+ // /same (from the mount table)
+ // all items in / of ns0 from mapping of / -> ns0:::/)
+ // all items in / of ns1 from mapping of / -> ns1:::/)
+ testListing("/");
+
+ // Verify that the "/same" mount point lists the contents of both dirs in
+ // the same ns
+ // /same -->
+ // /target-ns0 (from root of ns0)
+ // /testdir (from contents of /target-ns0)
+ testListing("/same");
+
+ // List a non-existing path and validate error response with NN behavior
+ ClientProtocol namenodeProtocol =
+ getCluster().getRandomNamenode().getClient().getNamenode();
+ Method m = ClientProtocol.class.getMethod(
+ "getListing", String.class, byte[].class, boolean.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(getRouterProtocol(), namenodeProtocol, m,
+ new Object[] {badPath, HdfsFileStatus.EMPTY_NAME, false});
+ }
+
+ @Override
+ public void testProxyRenameFiles() throws IOException, InterruptedException {
+
+ super.testProxyRenameFiles();
+
+ List<String> nss = getCluster().getNameservices();
+ String ns0 = nss.get(0);
+ String ns1 = nss.get(1);
+
+ // Rename a file from ns0 into the root (mapped to both ns0 and ns1)
+ String testDir0 = getCluster().getFederatedTestDirectoryForNS(ns0);
+ String filename0 = testDir0 + "/testrename";
+ String renamedFile = "/testrename";
+ testRename(getRouterContext(), filename0, renamedFile, false);
+ testRename2(getRouterContext(), filename0, renamedFile, false);
+
+ // Rename a file from ns1 into the root (mapped to both ns0 and ns1)
+ String testDir1 = getCluster().getFederatedTestDirectoryForNS(ns1);
+ String filename1 = testDir1 + "/testrename";
+ testRename(getRouterContext(), filename1, renamedFile, false);
+ testRename2(getRouterContext(), filename1, renamedFile, false);
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/40] hadoop git commit: HADOOP-14827. Allow StopWatch to accept a
Timer parameter for tests. Contributed by Erik Krogen
Posted by in...@apache.org.
HADOOP-14827. Allow StopWatch to accept a Timer parameter for tests. Contributed by Erik Krogen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dd814946
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dd814946
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dd814946
Branch: refs/heads/HDFS-10467
Commit: dd814946f68d52a9b1627ac4dd61f9ab093423ae
Parents: 704267c
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Sep 6 16:04:30 2017 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Sep 6 16:04:30 2017 -0500
----------------------------------------------------------------------
.../java/org/apache/hadoop/util/StopWatch.java | 17 ++++++++++++++---
1 file changed, 14 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd814946/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StopWatch.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StopWatch.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StopWatch.java
index b9d0d0b..c0eedf6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StopWatch.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StopWatch.java
@@ -25,11 +25,22 @@ import java.util.concurrent.TimeUnit;
* A simplified StopWatch implementation which can measure times in nanoseconds.
*/
public class StopWatch implements Closeable {
+ private final Timer timer;
private boolean isStarted;
private long startNanos;
private long currentElapsedNanos;
public StopWatch() {
+ this(new Timer());
+ }
+
+ /**
+ * Used for tests to be able to create a StopWatch which does not follow real
+ * time.
+ * @param timer The timer to base this StopWatch's timekeeping off of.
+ */
+ public StopWatch(Timer timer) {
+ this.timer = timer;
}
/**
@@ -49,7 +60,7 @@ public class StopWatch implements Closeable {
throw new IllegalStateException("StopWatch is already running");
}
isStarted = true;
- startNanos = System.nanoTime();
+ startNanos = timer.monotonicNowNanos();
return this;
}
@@ -61,7 +72,7 @@ public class StopWatch implements Closeable {
if (!isStarted) {
throw new IllegalStateException("StopWatch is already stopped");
}
- long now = System.nanoTime();
+ long now = timer.monotonicNowNanos();
isStarted = false;
currentElapsedNanos += now - startNanos;
return this;
@@ -90,7 +101,7 @@ public class StopWatch implements Closeable {
*/
public long now() {
return isStarted ?
- System.nanoTime() - startNanos + currentElapsedNanos :
+ timer.monotonicNowNanos() - startNanos + currentElapsedNanos :
currentElapsedNanos;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/40] hadoop git commit: HDFS-12223. Rebasing HDFS-10467.
Contributed by Inigo Goiri.
Posted by in...@apache.org.
HDFS-12223. Rebasing HDFS-10467. Contributed by Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4634769e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4634769e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4634769e
Branch: refs/heads/HDFS-10467
Commit: 4634769e1a8972dc2201c61a99373965506acaff
Parents: e96a2c7
Author: Inigo Goiri <in...@apache.org>
Authored: Fri Jul 28 15:55:10 2017 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu Sep 7 13:19:50 2017 -0700
----------------------------------------------------------------------
.../federation/router/RouterRpcServer.java | 59 +++++++++++++++++---
1 file changed, 51 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4634769e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
index 4bae71e..eaaab39 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
@@ -64,8 +64,9 @@ import org.apache.hadoop.hdfs.AddBlockFlag;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.inotify.EventBatchList;
-import org.apache.hadoop.hdfs.protocol.AddingECPolicyResponse;
+import org.apache.hadoop.hdfs.protocol.AddECPolicyResponse;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.protocol.BlocksStats;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
@@ -75,6 +76,7 @@ import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.ECBlockGroupsStats;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@@ -85,6 +87,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
@@ -1736,13 +1739,6 @@ public class RouterRpcServer extends AbstractService implements ClientProtocol {
}
@Override // ClientProtocol
- public AddingECPolicyResponse[] addErasureCodingPolicies(
- ErasureCodingPolicy[] policies) throws IOException {
- checkOperation(OperationCategory.WRITE, false);
- return null;
- }
-
- @Override // ClientProtocol
public void unsetErasureCodingPolicy(String src) throws IOException {
checkOperation(OperationCategory.WRITE, false);
}
@@ -1808,6 +1804,53 @@ public class RouterRpcServer extends AbstractService implements ClientProtocol {
return null;
}
+ @Override
+ public AddECPolicyResponse[] addErasureCodingPolicies(
+ ErasureCodingPolicy[] arg0) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ return null;
+ }
+
+ @Override
+ public void removeErasureCodingPolicy(String arg0) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override
+ public void disableErasureCodingPolicy(String arg0) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override
+ public void enableErasureCodingPolicy(String arg0) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override
+ public ECBlockGroupsStats getECBlockGroupsStats() throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override
+ public HashMap<String, String> getErasureCodingCodecs() throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override
+ public BlocksStats getBlocksStats() throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override
+ public BatchedEntries<OpenFileEntry> listOpenFiles(long arg0)
+ throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
/**
* Locate the location with the matching block pool id.
*
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/40] hadoop git commit: HDFS-10630. Federation State Store FS
Implementation. Contributed by Jason Kace and Inigo Goiri.
Posted by in...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3384811c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
new file mode 100644
index 0000000..7f0b36a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
@@ -0,0 +1,483 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.Query;
+import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
+import org.junit.AfterClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Base tests for the driver. The particular implementations will use this to
+ * test their functionality.
+ */
+public class TestStateStoreDriverBase {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestStateStoreDriverBase.class);
+
+ private static StateStoreService stateStore;
+ private static Configuration conf;
+
+
+ /**
+ * Get the State Store driver.
+ * @return State Store driver.
+ */
+ protected StateStoreDriver getStateStoreDriver() {
+ return stateStore.getDriver();
+ }
+
+ @AfterClass
+ public static void tearDownCluster() {
+ if (stateStore != null) {
+ stateStore.stop();
+ }
+ }
+
+ /**
+ * Get a new State Store using this configuration.
+ *
+ * @param config Configuration for the State Store.
+ * @throws Exception If we cannot get the State Store.
+ */
+ public static void getStateStore(Configuration config) throws Exception {
+ conf = config;
+ stateStore = FederationStateStoreTestUtils.getStateStore(conf);
+ }
+
+ private <T extends BaseRecord> T generateFakeRecord(Class<T> recordClass)
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+
+ // TODO add record
+ return null;
+ }
+
+ /**
+ * Validate if a record is the same.
+ *
+ * @param original
+ * @param committed
+ * @param assertEquals Assert if the records are equal or just return.
+ * @return
+ * @throws IllegalArgumentException
+ * @throws IllegalAccessException
+ */
+ private boolean validateRecord(
+ BaseRecord original, BaseRecord committed, boolean assertEquals)
+ throws IllegalArgumentException, IllegalAccessException {
+
+ boolean ret = true;
+
+ Map<String, Class<?>> fields = getFields(original);
+ for (String key : fields.keySet()) {
+ if (key.equals("dateModified") ||
+ key.equals("dateCreated") ||
+ key.equals("proto")) {
+ // Fields are updated/set on commit and fetch and may not match
+ // the fields that are initialized in a non-committed object.
+ continue;
+ }
+ Object data1 = getField(original, key);
+ Object data2 = getField(committed, key);
+ if (assertEquals) {
+ assertEquals("Field " + key + " does not match", data1, data2);
+ } else if (!data1.equals(data2)) {
+ ret = false;
+ }
+ }
+
+ long now = stateStore.getDriver().getTime();
+ assertTrue(
+ committed.getDateCreated() <= now && committed.getDateCreated() > 0);
+ assertTrue(committed.getDateModified() >= committed.getDateCreated());
+
+ return ret;
+ }
+
+ public static void removeAll(StateStoreDriver driver) throws IOException {
+ // TODO add records to remove
+ }
+
+ public <T extends BaseRecord> void testInsert(
+ StateStoreDriver driver, Class<T> recordClass)
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+
+ assertTrue(driver.removeAll(recordClass));
+ QueryResult<T> records = driver.get(recordClass);
+ assertTrue(records.getRecords().isEmpty());
+
+ // Insert single
+ BaseRecord record = generateFakeRecord(recordClass);
+ driver.put(record, true, false);
+
+ // Verify
+ records = driver.get(recordClass);
+ assertEquals(1, records.getRecords().size());
+ validateRecord(record, records.getRecords().get(0), true);
+
+ // Insert multiple
+ List<T> insertList = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ T newRecord = generateFakeRecord(recordClass);
+ insertList.add(newRecord);
+ }
+ driver.putAll(insertList, true, false);
+
+ // Verify
+ records = driver.get(recordClass);
+ assertEquals(11, records.getRecords().size());
+ }
+
+ public <T extends BaseRecord> void testFetchErrors(StateStoreDriver driver,
+ Class<T> clazz) throws IllegalAccessException, IOException {
+
+ // Fetch empty list
+ driver.removeAll(clazz);
+ QueryResult<T> result0 = driver.get(clazz);
+ assertNotNull(result0);
+ List<T> records0 = result0.getRecords();
+ assertEquals(records0.size(), 0);
+
+ // Insert single
+ BaseRecord record = generateFakeRecord(clazz);
+ assertTrue(driver.put(record, true, false));
+
+ // Verify
+ QueryResult<T> result1 = driver.get(clazz);
+ List<T> records1 = result1.getRecords();
+ assertEquals(1, records1.size());
+ validateRecord(record, records1.get(0), true);
+
+ // Test fetch single object with a bad query
+ final T fakeRecord = generateFakeRecord(clazz);
+ final Query<T> query = new Query<T>(fakeRecord);
+ T getRecord = driver.get(clazz, query);
+ assertNull(getRecord);
+
+ // Test fetch multiple objects does not exist returns empty list
+ assertEquals(driver.getMultiple(clazz, query).size(), 0);
+ }
+
+ public <T extends BaseRecord> void testPut(
+ StateStoreDriver driver, Class<T> clazz)
+ throws IllegalArgumentException, ReflectiveOperationException,
+ IOException, SecurityException {
+
+ driver.removeAll(clazz);
+ QueryResult<T> records = driver.get(clazz);
+ assertTrue(records.getRecords().isEmpty());
+
+ // Insert multiple
+ List<T> insertList = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ T newRecord = generateFakeRecord(clazz);
+ insertList.add(newRecord);
+ }
+
+ // Verify
+ assertTrue(driver.putAll(insertList, false, true));
+ records = driver.get(clazz);
+ assertEquals(records.getRecords().size(), 10);
+
+ // Generate a new record with the same PK fields as an existing record
+ BaseRecord updatedRecord = generateFakeRecord(clazz);
+ BaseRecord existingRecord = records.getRecords().get(0);
+ Map<String, String> primaryKeys = existingRecord.getPrimaryKeys();
+ for (Entry<String, String> entry : primaryKeys.entrySet()) {
+ String key = entry.getKey();
+ String value = entry.getValue();
+ Class<?> fieldType = getFieldType(existingRecord, key);
+ Object field = fromString(value, fieldType);
+ assertTrue(setField(updatedRecord, key, field));
+ }
+
+ // Attempt an update of an existing entry, but it is not allowed.
+ assertFalse(driver.put(updatedRecord, false, true));
+
+ // Verify no update occurred, all original records are unchanged
+ QueryResult<T> newRecords = driver.get(clazz);
+ assertTrue(newRecords.getRecords().size() == 10);
+ assertEquals("A single entry was improperly updated in the store", 10,
+ countMatchingEntries(records.getRecords(), newRecords.getRecords()));
+
+ // Update the entry (allowing updates)
+ assertTrue(driver.put(updatedRecord, true, false));
+
+ // Verify that one entry no longer matches the original set
+ newRecords = driver.get(clazz);
+ assertEquals(10, newRecords.getRecords().size());
+ assertEquals(
+ "Record of type " + clazz + " not updated in the store", 9,
+ countMatchingEntries(records.getRecords(), newRecords.getRecords()));
+ }
+
+ private int countMatchingEntries(
+ Collection<? extends BaseRecord> committedList,
+ Collection<? extends BaseRecord> matchList) {
+
+ int matchingCount = 0;
+ for (BaseRecord committed : committedList) {
+ for (BaseRecord match : matchList) {
+ try {
+ if (match.getPrimaryKey().equals(committed.getPrimaryKey())) {
+ if (validateRecord(match, committed, false)) {
+ matchingCount++;
+ }
+ break;
+ }
+ } catch (Exception ex) {
+ }
+ }
+ }
+ return matchingCount;
+ }
+
+ public <T extends BaseRecord> void testRemove(
+ StateStoreDriver driver, Class<T> clazz)
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+
+ // Remove all
+ assertTrue(driver.removeAll(clazz));
+ QueryResult<T> records = driver.get(clazz);
+ assertTrue(records.getRecords().isEmpty());
+
+ // Insert multiple
+ List<T> insertList = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ T newRecord = generateFakeRecord(clazz);
+ insertList.add(newRecord);
+ }
+
+ // Verify
+ assertTrue(driver.putAll(insertList, false, true));
+ records = driver.get(clazz);
+ assertEquals(records.getRecords().size(), 10);
+
+ // Remove Single
+ assertTrue(driver.remove(records.getRecords().get(0)));
+
+ // Verify
+ records = driver.get(clazz);
+ assertEquals(records.getRecords().size(), 9);
+
+ // Remove with filter
+ final T firstRecord = records.getRecords().get(0);
+ final Query<T> query0 = new Query<T>(firstRecord);
+ assertTrue(driver.remove(clazz, query0) > 0);
+
+ final T secondRecord = records.getRecords().get(1);
+ final Query<T> query1 = new Query<T>(secondRecord);
+ assertTrue(driver.remove(clazz, query1) > 0);
+
+ // Verify
+ records = driver.get(clazz);
+ assertEquals(records.getRecords().size(), 7);
+
+ // Remove all
+ assertTrue(driver.removeAll(clazz));
+
+ // Verify
+ records = driver.get(clazz);
+ assertTrue(records.getRecords().isEmpty());
+ }
+
+ public void testInsert(StateStoreDriver driver)
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ // TODO add records
+ }
+
+ public void testPut(StateStoreDriver driver)
+ throws IllegalArgumentException, ReflectiveOperationException,
+ IOException, SecurityException {
+ // TODO add records
+ }
+
+ public void testRemove(StateStoreDriver driver)
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ // TODO add records
+ }
+
+ public void testFetchErrors(StateStoreDriver driver)
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ // TODO add records
+ }
+
+ /**
+ * Sets the value of a field on the object.
+ *
+ * @param fieldName The string name of the field.
+ * @param data The data to pass to the field's setter.
+ *
+ * @return True if successful, fails if failed.
+ */
+ private static boolean setField(
+ BaseRecord record, String fieldName, Object data) {
+
+ Method m = locateSetter(record, fieldName);
+ if (m != null) {
+ try {
+ m.invoke(record, data);
+ } catch (Exception e) {
+ LOG.error("Cannot set field " + fieldName + " on object "
+ + record.getClass().getName() + " to data " + data + " of type "
+ + data.getClass(), e);
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Finds the appropriate setter for a field name.
+ *
+ * @param fieldName The legacy name of the field.
+ * @return The matching setter or null if not found.
+ */
+ private static Method locateSetter(BaseRecord record, String fieldName) {
+ for (Method m : record.getClass().getMethods()) {
+ if (m.getName().equalsIgnoreCase("set" + fieldName)) {
+ return m;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Returns all serializable fields in the object.
+ *
+ * @return Map with the fields.
+ */
+ private static Map<String, Class<?>> getFields(BaseRecord record) {
+ Map<String, Class<?>> getters = new HashMap<>();
+ for (Method m : record.getClass().getDeclaredMethods()) {
+ if (m.getName().startsWith("get")) {
+ try {
+ Class<?> type = m.getReturnType();
+ char[] c = m.getName().substring(3).toCharArray();
+ c[0] = Character.toLowerCase(c[0]);
+ String key = new String(c);
+ getters.put(key, type);
+ } catch (Exception e) {
+ LOG.error("Cannot execute getter " + m.getName()
+ + " on object " + record);
+ }
+ }
+ }
+ return getters;
+ }
+
+ /**
+ * Get the type of a field.
+ *
+ * @param fieldName
+ * @return Field type
+ */
+ private static Class<?> getFieldType(BaseRecord record, String fieldName) {
+ Method m = locateGetter(record, fieldName);
+ return m.getReturnType();
+ }
+
+ /**
+ * Fetches the value for a field name.
+ *
+ * @param fieldName the legacy name of the field.
+ * @return The field data or null if not found.
+ */
+ private static Object getField(BaseRecord record, String fieldName) {
+ Object result = null;
+ Method m = locateGetter(record, fieldName);
+ if (m != null) {
+ try {
+ result = m.invoke(record);
+ } catch (Exception e) {
+ LOG.error("Cannot get field " + fieldName + " on object " + record);
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Finds the appropriate getter for a field name.
+ *
+ * @param fieldName The legacy name of the field.
+ * @return The matching getter or null if not found.
+ */
+ private static Method locateGetter(BaseRecord record, String fieldName) {
+ for (Method m : record.getClass().getMethods()) {
+ if (m.getName().equalsIgnoreCase("get" + fieldName)) {
+ return m;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Expands a data object from the store into an record object. Default store
+ * data type is a String. Override if additional serialization is required.
+ *
+ * @param data Object containing the serialized data. Only string is
+ * supported.
+ * @param clazz Target object class to hold the deserialized data.
+ * @return An instance of the target data object initialized with the
+ * deserialized data.
+ */
+ @Deprecated
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ private static <T> T fromString(String data, Class<T> clazz) {
+
+ if (data.equals("null")) {
+ return null;
+ } else if (clazz == String.class) {
+ return (T) data;
+ } else if (clazz == Long.class || clazz == long.class) {
+ return (T) Long.valueOf(data);
+ } else if (clazz == Integer.class || clazz == int.class) {
+ return (T) Integer.valueOf(data);
+ } else if (clazz == Double.class || clazz == double.class) {
+ return (T) Double.valueOf(data);
+ } else if (clazz == Float.class || clazz == float.class) {
+ return (T) Float.valueOf(data);
+ } else if (clazz == Boolean.class || clazz == boolean.class) {
+ return (T) Boolean.valueOf(data);
+ } else if (clazz.isEnum()) {
+ return (T) Enum.valueOf((Class<Enum>) clazz, data);
+ }
+ return null;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3384811c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFile.java
new file mode 100644
index 0000000..920e280
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFile.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver;
+
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.getStateStoreConfiguration;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test the FileSystem (e.g., HDFS) implementation of the State Store driver.
+ */
+public class TestStateStoreFile extends TestStateStoreDriverBase {
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ Configuration conf = getStateStoreConfiguration(StateStoreFileImpl.class);
+ getStateStore(conf);
+ }
+
+ @Before
+ public void startup() throws IOException {
+ removeAll(getStateStoreDriver());
+ }
+
+ @Test
+ public void testInsert()
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testInsert(getStateStoreDriver());
+ }
+
+ @Test
+ public void testUpdate()
+ throws IllegalArgumentException, ReflectiveOperationException,
+ IOException, SecurityException {
+ testPut(getStateStoreDriver());
+ }
+
+ @Test
+ public void testDelete()
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testRemove(getStateStoreDriver());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3384811c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileSystem.java
new file mode 100644
index 0000000..da2e51d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileSystem.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils;
+import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileSystemImpl;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test the FileSystem (e.g., HDFS) implementation of the State Store driver.
+ */
+public class TestStateStoreFileSystem extends TestStateStoreDriverBase {
+
+ private static MiniDFSCluster dfsCluster;
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ Configuration conf = FederationStateStoreTestUtils
+ .getStateStoreConfiguration(StateStoreFileSystemImpl.class);
+ conf.set(StateStoreFileSystemImpl.FEDERATION_STORE_FS_PATH,
+ "/hdfs-federation/");
+
+ // Create HDFS cluster to back the state tore
+ MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
+ builder.numDataNodes(1);
+ dfsCluster = builder.build();
+ dfsCluster.waitClusterUp();
+ getStateStore(conf);
+ }
+
+ @AfterClass
+ public static void tearDownCluster() {
+ if (dfsCluster != null) {
+ dfsCluster.shutdown();
+ }
+ }
+
+ @Before
+ public void startup() throws IOException {
+ removeAll(getStateStoreDriver());
+ }
+
+ @Test
+ public void testInsert()
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testInsert(getStateStoreDriver());
+ }
+
+ @Test
+ public void testUpdate()
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testInsert(getStateStoreDriver());
+ }
+
+ @Test
+ public void testDelete()
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testInsert(getStateStoreDriver());
+ }
+
+ @Test
+ public void testFetchErrors()
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testFetchErrors(getStateStoreDriver());
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/40] hadoop git commit: Revert "Plan/ResourceAllocation data
structure enhancements required to support recurring reservations in
ReservationSystem."
Posted by in...@apache.org.
Revert "Plan/ResourceAllocation data structure enhancements required to support recurring reservations in ReservationSystem."
This reverts commit 7996eca7dcfaa1bdf970e32022274f2699bef8a1.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e3345e98
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e3345e98
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e3345e98
Branch: refs/heads/HDFS-10467
Commit: e3345e985bff93c6c74a76747e45376c6027f42c
Parents: 22de944
Author: Subru Krishnan <su...@apache.org>
Authored: Wed Sep 6 16:39:23 2017 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Wed Sep 6 16:39:23 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 6 -
.../yarn/conf/TestYarnConfigurationFields.java | 4 +-
.../reservation/AbstractReservationSystem.java | 90 ++--
.../AbstractSchedulerPlanFollower.java | 183 ++++----
.../reservation/InMemoryPlan.java | 400 +++++------------
.../InMemoryReservationAllocation.java | 36 +-
.../reservation/NoOverCommitPolicy.java | 2 +-
.../PeriodicRLESparseResourceAllocation.java | 130 ++----
.../resourcemanager/reservation/PlanEdit.java | 24 +-
.../resourcemanager/reservation/PlanView.java | 94 ++--
.../RLESparseResourceAllocation.java | 115 +++--
.../reservation/ReservationAllocation.java | 60 +--
.../reservation/ReservationInputValidator.java | 134 +++---
.../reservation/ReservationSystem.java | 6 +-
.../reservation/SharingPolicy.java | 13 +-
.../reservation/planning/Planner.java | 2 +-
.../reservation/planning/PlanningAlgorithm.java | 2 +-
.../reservation/planning/StageAllocator.java | 2 +-
.../planning/StageAllocatorGreedy.java | 2 +-
.../planning/StageAllocatorGreedyRLE.java | 5 +-
.../planning/StageAllocatorLowCostAligned.java | 4 +-
.../reservation/ReservationSystemTestUtil.java | 135 +++---
.../reservation/TestInMemoryPlan.java | 431 +++++++------------
...TestPeriodicRLESparseResourceAllocation.java | 109 ++---
.../TestRLESparseResourceAllocation.java | 122 +++---
.../planning/TestSimpleCapacityReplanner.java | 8 +-
26 files changed, 777 insertions(+), 1342 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3345e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 27ca957..4944821 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -262,12 +262,6 @@ public class YarnConfiguration extends Configuration {
public static final long DEFAULT_RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP =
1000L;
- /** The maximum periodicity for the Reservation System. */
- public static final String RM_RESERVATION_SYSTEM_MAX_PERIODICITY =
- RM_PREFIX + "reservation-system.max-periodicity";
- public static final long DEFAULT_RM_RESERVATION_SYSTEM_MAX_PERIODICITY =
- 86400000L;
-
/**
* Enable periodic monitor threads.
* @see #RM_SCHEDULER_MONITOR_POLICIES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3345e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index 1d3111c..bd7bf93 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.conf.TestConfigurationFieldsBase;
*/
public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
- @SuppressWarnings({"deprecation", "methodlength"})
+ @SuppressWarnings("deprecation")
@Override
public void initializeMemberVariables() {
xmlFilename = new String("yarn-default.xml");
@@ -69,8 +69,6 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
configurationPropsToSkipCompare.add(YarnConfiguration
.YARN_SECURITY_SERVICE_AUTHORIZATION_COLLECTOR_NODEMANAGER_PROTOCOL);
configurationPropsToSkipCompare.add(YarnConfiguration.CURATOR_LEADER_ELECTOR);
- configurationPropsToSkipCompare
- .add(YarnConfiguration.RM_RESERVATION_SYSTEM_MAX_PERIODICITY);
// Federation default configs to be ignored
configurationPropsToSkipCompare
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3345e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
index 5b8772c..5ef4912 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
@@ -18,17 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.reservation;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
@@ -57,6 +46,17 @@ import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
/**
* This is the implementation of {@link ReservationSystem} based on the
* {@link ResourceScheduler}
@@ -66,8 +66,8 @@ import org.slf4j.LoggerFactory;
public abstract class AbstractReservationSystem extends AbstractService
implements ReservationSystem {
- private static final Logger LOG =
- LoggerFactory.getLogger(AbstractReservationSystem.class);
+ private static final Logger LOG = LoggerFactory
+ .getLogger(AbstractReservationSystem.class);
// private static final String DEFAULT_CAPACITY_SCHEDULER_PLAN
@@ -103,8 +103,6 @@ public abstract class AbstractReservationSystem extends AbstractService
private boolean isRecoveryEnabled = false;
- private long maxPeriodicity;
-
/**
* Construct the service.
*
@@ -145,41 +143,36 @@ public abstract class AbstractReservationSystem extends AbstractService
this.conf = conf;
scheduler = rmContext.getScheduler();
// Get the plan step size
- planStepSize = conf.getTimeDuration(
- YarnConfiguration.RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP,
- YarnConfiguration.DEFAULT_RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP,
- TimeUnit.MILLISECONDS);
+ planStepSize =
+ conf.getTimeDuration(
+ YarnConfiguration.RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP,
+ YarnConfiguration.DEFAULT_RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP,
+ TimeUnit.MILLISECONDS);
if (planStepSize < 0) {
planStepSize =
YarnConfiguration.DEFAULT_RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP;
}
- maxPeriodicity =
- conf.getLong(YarnConfiguration.RM_RESERVATION_SYSTEM_MAX_PERIODICITY,
- YarnConfiguration.DEFAULT_RM_RESERVATION_SYSTEM_MAX_PERIODICITY);
- if (maxPeriodicity <= 0) {
- maxPeriodicity =
- YarnConfiguration.DEFAULT_RM_RESERVATION_SYSTEM_MAX_PERIODICITY;
- }
// Create a plan corresponding to every reservable queue
Set<String> planQueueNames = scheduler.getPlanQueues();
for (String planQueueName : planQueueNames) {
Plan plan = initializePlan(planQueueName);
plans.put(planQueueName, plan);
}
- isRecoveryEnabled = conf.getBoolean(YarnConfiguration.RECOVERY_ENABLED,
+ isRecoveryEnabled = conf.getBoolean(
+ YarnConfiguration.RECOVERY_ENABLED,
YarnConfiguration.DEFAULT_RM_RECOVERY_ENABLED);
if (conf.getBoolean(YarnConfiguration.YARN_RESERVATION_ACL_ENABLE,
- YarnConfiguration.DEFAULT_YARN_RESERVATION_ACL_ENABLE)
- && conf.getBoolean(YarnConfiguration.YARN_ACL_ENABLE,
- YarnConfiguration.DEFAULT_YARN_ACL_ENABLE)) {
+ YarnConfiguration.DEFAULT_YARN_RESERVATION_ACL_ENABLE) &&
+ conf.getBoolean(YarnConfiguration.YARN_ACL_ENABLE,
+ YarnConfiguration.DEFAULT_YARN_ACL_ENABLE)) {
reservationsACLsManager = new ReservationsACLsManager(scheduler, conf);
}
}
private void loadPlan(String planName,
Map<ReservationId, ReservationAllocationStateProto> reservations)
- throws PlanningException {
+ throws PlanningException {
Plan plan = plans.get(planName);
Resource minAllocation = getMinAllocation();
ResourceCalculator rescCalculator = getResourceCalculator();
@@ -255,8 +248,8 @@ public abstract class AbstractReservationSystem extends AbstractService
Class<?> planFollowerPolicyClazz =
conf.getClassByName(planFollowerPolicyClassName);
if (PlanFollower.class.isAssignableFrom(planFollowerPolicyClazz)) {
- return (PlanFollower) ReflectionUtils
- .newInstance(planFollowerPolicyClazz, conf);
+ return (PlanFollower) ReflectionUtils.newInstance(
+ planFollowerPolicyClazz, conf);
} else {
throw new YarnRuntimeException("Class: " + planFollowerPolicyClassName
+ " not instance of " + PlanFollower.class.getCanonicalName());
@@ -264,8 +257,7 @@ public abstract class AbstractReservationSystem extends AbstractService
} catch (ClassNotFoundException e) {
throw new YarnRuntimeException(
"Could not instantiate PlanFollowerPolicy: "
- + planFollowerPolicyClassName,
- e);
+ + planFollowerPolicyClassName, e);
}
}
@@ -379,8 +371,9 @@ public abstract class AbstractReservationSystem extends AbstractService
public ReservationId getNewReservationId() {
writeLock.lock();
try {
- ReservationId resId = ReservationId.newInstance(
- ResourceManager.getClusterTimeStamp(), resCounter.incrementAndGet());
+ ReservationId resId =
+ ReservationId.newInstance(ResourceManager.getClusterTimeStamp(),
+ resCounter.incrementAndGet());
LOG.info("Allocated new reservationId: " + resId);
return resId;
} finally {
@@ -397,11 +390,8 @@ public abstract class AbstractReservationSystem extends AbstractService
* Get the default reservation system corresponding to the scheduler
*
* @param scheduler the scheduler for which the reservation system is required
- *
- * @return the {@link ReservationSystem} based on the configured scheduler
*/
- public static String getDefaultReservationSystem(
- ResourceScheduler scheduler) {
+ public static String getDefaultReservationSystem(ResourceScheduler scheduler) {
if (scheduler instanceof CapacityScheduler) {
return CapacityReservationSystem.class.getName();
} else if (scheduler instanceof FairScheduler) {
@@ -419,11 +409,12 @@ public abstract class AbstractReservationSystem extends AbstractService
Resource maxAllocation = getMaxAllocation();
ResourceCalculator rescCalc = getResourceCalculator();
Resource totCap = getPlanQueueCapacity(planQueueName);
- Plan plan = new InMemoryPlan(getRootQueueMetrics(), adPolicy,
- getAgent(planQueuePath), totCap, planStepSize, rescCalc, minAllocation,
- maxAllocation, planQueueName, getReplanner(planQueuePath),
- getReservationSchedulerConfiguration().getMoveOnExpiry(planQueuePath),
- maxPeriodicity, rmContext);
+ Plan plan =
+ new InMemoryPlan(getRootQueueMetrics(), adPolicy,
+ getAgent(planQueuePath), totCap, planStepSize, rescCalc,
+ minAllocation, maxAllocation, planQueueName,
+ getReplanner(planQueuePath), getReservationSchedulerConfiguration()
+ .getMoveOnExpiry(planQueuePath), rmContext);
LOG.info("Initialized plan {} based on reservable queue {}",
plan.toString(), planQueueName);
return plan;
@@ -486,8 +477,8 @@ public abstract class AbstractReservationSystem extends AbstractService
Class<?> admissionPolicyClazz =
conf.getClassByName(admissionPolicyClassName);
if (SharingPolicy.class.isAssignableFrom(admissionPolicyClazz)) {
- return (SharingPolicy) ReflectionUtils.newInstance(admissionPolicyClazz,
- conf);
+ return (SharingPolicy) ReflectionUtils.newInstance(
+ admissionPolicyClazz, conf);
} else {
throw new YarnRuntimeException("Class: " + admissionPolicyClassName
+ " not instance of " + SharingPolicy.class.getCanonicalName());
@@ -502,7 +493,8 @@ public abstract class AbstractReservationSystem extends AbstractService
return this.reservationsACLsManager;
}
- protected abstract ReservationSchedulerConfiguration getReservationSchedulerConfiguration();
+ protected abstract ReservationSchedulerConfiguration
+ getReservationSchedulerConfiguration();
protected abstract String getPlanQueuePath(String planQueueName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3345e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java
index 9b6a0b0..90357e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java
@@ -18,14 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.reservation;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -41,17 +33,24 @@ import org.apache.hadoop.yarn.util.resource.Resources;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
- private static final Logger LOG =
- LoggerFactory.getLogger(AbstractSchedulerPlanFollower.class);
+ private static final Logger LOG = LoggerFactory
+ .getLogger(AbstractSchedulerPlanFollower.class);
protected Collection<Plan> plans = new ArrayList<Plan>();
protected YarnScheduler scheduler;
protected Clock clock;
@Override
- public void init(Clock clock, ResourceScheduler sched,
- Collection<Plan> plans) {
+ public void init(Clock clock, ResourceScheduler sched, Collection<Plan> plans) {
this.clock = clock;
this.scheduler = sched;
this.plans.addAll(plans);
@@ -72,7 +71,7 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
@Override
public synchronized void synchronizePlan(Plan plan, boolean shouldReplan) {
- String planQueueName = plan.getQueueName();
+ String planQueueName = plan.getQueueName();
if (LOG.isDebugEnabled()) {
LOG.debug("Running plan follower edit policy for plan: " + planQueueName);
}
@@ -83,14 +82,12 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
now += step - (now % step);
}
Queue planQueue = getPlanQueue(planQueueName);
- if (planQueue == null) {
- return;
- }
+ if (planQueue == null) return;
// first we publish to the plan the current availability of resources
Resource clusterResources = scheduler.getClusterResource();
- Resource planResources =
- getPlanResources(plan, planQueue, clusterResources);
+ Resource planResources = getPlanResources(plan, planQueue,
+ clusterResources);
Set<ReservationAllocation> currentReservations =
plan.getReservationsAtTime(now);
Set<String> curReservationNames = new HashSet<String>();
@@ -98,11 +95,12 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
int numRes = getReservedResources(now, currentReservations,
curReservationNames, reservedResources);
// create the default reservation queue if it doesnt exist
- String defReservationId = getReservationIdFromQueueName(planQueueName)
- + ReservationConstants.DEFAULT_QUEUE_SUFFIX;
- String defReservationQueue =
- getReservationQueueName(planQueueName, defReservationId);
- createDefaultReservationQueue(planQueueName, planQueue, defReservationId);
+ String defReservationId = getReservationIdFromQueueName(planQueueName) +
+ ReservationConstants.DEFAULT_QUEUE_SUFFIX;
+ String defReservationQueue = getReservationQueueName(planQueueName,
+ defReservationId);
+ createDefaultReservationQueue(planQueueName, planQueue,
+ defReservationId);
curReservationNames.add(defReservationId);
// if the resources dedicated to this plan has shrunk invoke replanner
boolean shouldResize = false;
@@ -151,8 +149,10 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
// sort allocations from the one giving up the most resources, to the
// one asking for the most avoid order-of-operation errors that
// temporarily violate 100% capacity bound
- List<ReservationAllocation> sortedAllocations = sortByDelta(
- new ArrayList<ReservationAllocation>(currentReservations), now, plan);
+ List<ReservationAllocation> sortedAllocations =
+ sortByDelta(
+ new ArrayList<ReservationAllocation>(currentReservations), now,
+ plan);
for (ReservationAllocation res : sortedAllocations) {
String currResId = res.getReservationId().toString();
if (curReservationNames.contains(currResId)) {
@@ -163,9 +163,10 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
if (planResources.getMemorySize() > 0
&& planResources.getVirtualCores() > 0) {
if (shouldResize) {
- capToAssign = calculateReservationToPlanProportion(
- plan.getResourceCalculator(), planResources, reservedResources,
- capToAssign);
+ capToAssign =
+ calculateReservationToPlanProportion(
+ plan.getResourceCalculator(), planResources,
+ reservedResources, capToAssign);
}
targetCapacity =
calculateReservationToPlanRatio(plan.getResourceCalculator(),
@@ -184,8 +185,7 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
maxCapacity = targetCapacity;
}
try {
- setQueueEntitlement(planQueueName, currResId, targetCapacity,
- maxCapacity);
+ setQueueEntitlement(planQueueName, currResId, targetCapacity, maxCapacity);
} catch (YarnException e) {
LOG.warn("Exception while trying to size reservation for plan: {}",
currResId, planQueueName, e);
@@ -196,10 +196,9 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
// compute the default queue capacity
float defQCap = 1.0f - totalAssignedCapacity;
if (LOG.isDebugEnabled()) {
- LOG.debug(
- "PlanFollowerEditPolicyTask: total Plan Capacity: {} "
- + "currReservation: {} default-queue capacity: {}",
- planResources, numRes, defQCap);
+ LOG.debug("PlanFollowerEditPolicyTask: total Plan Capacity: {} "
+ + "currReservation: {} default-queue capacity: {}", planResources,
+ numRes, defQCap);
}
// set the default queue to eat-up all remaining capacity
try {
@@ -226,11 +225,12 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
}
protected void setQueueEntitlement(String planQueueName, String currResId,
- float targetCapacity, float maxCapacity) throws YarnException {
- String reservationQueueName =
- getReservationQueueName(planQueueName, currResId);
- scheduler.setEntitlement(reservationQueueName,
- new QueueEntitlement(targetCapacity, maxCapacity));
+ float targetCapacity,
+ float maxCapacity) throws YarnException {
+ String reservationQueueName = getReservationQueueName(planQueueName,
+ currResId);
+ scheduler.setEntitlement(reservationQueueName, new QueueEntitlement(
+ targetCapacity, maxCapacity));
}
// Schedulers have different ways of naming queues. See YARN-2773
@@ -244,21 +244,14 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
* Then move all apps in the set of queues to the parent plan queue's default
* reservation queue if move is enabled. Finally cleanups the queue by killing
* any apps (if move is disabled or move failed) and removing the queue
- *
- * @param planQueueName the name of {@code PlanQueue}
- * @param shouldMove flag to indicate if any running apps should be moved or
- * killed
- * @param toRemove the remnant apps to clean up
- * @param defReservationQueue the default {@code ReservationQueue} of the
- * {@link Plan}
*/
- protected void cleanupExpiredQueues(String planQueueName, boolean shouldMove,
- Set<String> toRemove, String defReservationQueue) {
+ protected void cleanupExpiredQueues(String planQueueName,
+ boolean shouldMove, Set<String> toRemove, String defReservationQueue) {
for (String expiredReservationId : toRemove) {
try {
// reduce entitlement to 0
- String expiredReservation =
- getReservationQueueName(planQueueName, expiredReservationId);
+ String expiredReservation = getReservationQueueName(planQueueName,
+ expiredReservationId);
setQueueEntitlement(planQueueName, expiredReservation, 0.0f, 0.0f);
if (shouldMove) {
moveAppsInQueueSync(expiredReservation, defReservationQueue);
@@ -282,7 +275,7 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
* reservation queue in a synchronous fashion
*/
private void moveAppsInQueueSync(String expiredReservation,
- String defReservationQueue) {
+ String defReservationQueue) {
List<ApplicationAttemptId> activeApps =
scheduler.getAppsInQueue(expiredReservation);
if (activeApps.isEmpty()) {
@@ -294,16 +287,16 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
scheduler.moveApplication(app.getApplicationId(), defReservationQueue);
} catch (YarnException e) {
LOG.warn(
- "Encountered unexpected error during migration of application: {}"
- + " from reservation: {}",
+ "Encountered unexpected error during migration of application: {}" +
+ " from reservation: {}",
app, expiredReservation, e);
}
}
}
- protected int getReservedResources(long now,
- Set<ReservationAllocation> currentReservations,
- Set<String> curReservationNames, Resource reservedResources) {
+ protected int getReservedResources(long now, Set<ReservationAllocation>
+ currentReservations, Set<String> curReservationNames,
+ Resource reservedResources) {
int numRes = 0;
if (currentReservations != null) {
numRes = currentReservations.size();
@@ -319,30 +312,23 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
* Sort in the order from the least new amount of resources asked (likely
* negative) to the highest. This prevents "order-of-operation" errors related
* to exceeding 100% capacity temporarily.
- *
- * @param currentReservations the currently active reservations
- * @param now the current time
- * @param plan the {@link Plan} that is being considered
- *
- * @return the sorted list of {@link ReservationAllocation}s
*/
protected List<ReservationAllocation> sortByDelta(
List<ReservationAllocation> currentReservations, long now, Plan plan) {
- Collections.sort(currentReservations,
- new ReservationAllocationComparator(now, this, plan));
+ Collections.sort(currentReservations, new ReservationAllocationComparator(
+ now, this, plan));
return currentReservations;
}
/**
- * Get queue associated with reservable queue named.
- *
- * @param planQueueName name of the reservable queue
+ * Get queue associated with reservable queue named
+ * @param planQueueName Name of the reservable queue
* @return queue associated with the reservable queue
*/
protected abstract Queue getPlanQueue(String planQueueName);
/**
- * Resizes reservations based on currently available resources.
+ * Resizes reservations based on currently available resources
*/
private Resource calculateReservationToPlanProportion(
ResourceCalculator rescCalculator, Resource availablePlanResources,
@@ -352,7 +338,7 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
}
/**
- * Calculates ratio of reservationResources to planResources.
+ * Calculates ratio of reservationResources to planResources
*/
private float calculateReservationToPlanRatio(
ResourceCalculator rescCalculator, Resource clusterResources,
@@ -362,7 +348,7 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
}
/**
- * Check if plan resources are less than expected reservation resources.
+ * Check if plan resources are less than expected reservation resources
*/
private boolean arePlanResourcesLessThanReservations(
ResourceCalculator rescCalculator, Resource clusterResources,
@@ -372,56 +358,38 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
}
/**
- * Get a list of reservation queues for this planQueue.
- *
- * @param planQueue the queue for the current {@link Plan}
- *
- * @return the queues corresponding to the reservations
+ * Get a list of reservation queues for this planQueue
*/
protected abstract List<? extends Queue> getChildReservationQueues(
Queue planQueue);
/**
- * Add a new reservation queue for reservation currResId for this planQueue.
+ * Add a new reservation queue for reservation currResId for this planQueue
*/
- protected abstract void addReservationQueue(String planQueueName, Queue queue,
- String currResId);
+ protected abstract void addReservationQueue(
+ String planQueueName, Queue queue, String currResId);
/**
- * Creates the default reservation queue for use when no reservation is used
- * for applications submitted to this planQueue.
- *
- * @param planQueueName name of the reservable queue
- * @param queue the queue for the current {@link Plan}
- * @param defReservationQueue name of the default {@code ReservationQueue}
+ * Creates the default reservation queue for use when no reservation is
+ * used for applications submitted to this planQueue
*/
- protected abstract void createDefaultReservationQueue(String planQueueName,
- Queue queue, String defReservationQueue);
+ protected abstract void createDefaultReservationQueue(
+ String planQueueName, Queue queue, String defReservationQueue);
/**
- * Get plan resources for this planQueue.
- *
- * @param plan the current {@link Plan} being considered
- * @param clusterResources the resources available in the cluster
- *
- * @return the resources allocated to the specified {@link Plan}
+ * Get plan resources for this planQueue
*/
- protected abstract Resource getPlanResources(Plan plan, Queue queue,
- Resource clusterResources);
+ protected abstract Resource getPlanResources(
+ Plan plan, Queue queue, Resource clusterResources);
/**
* Get reservation queue resources if it exists otherwise return null.
- *
- * @param plan the current {@link Plan} being considered
- * @param reservationId the identifier of the reservation
- *
- * @return the resources allocated to the specified reservation
*/
protected abstract Resource getReservationQueueResourceIfExists(Plan plan,
ReservationId reservationId);
- private static class ReservationAllocationComparator
- implements Comparator<ReservationAllocation> {
+ private static class ReservationAllocationComparator implements
+ Comparator<ReservationAllocation> {
AbstractSchedulerPlanFollower planFollower;
long now;
Plan plan;
@@ -436,12 +404,14 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
private Resource getUnallocatedReservedResources(
ReservationAllocation reservation) {
Resource resResource;
- Resource reservationResource =
- planFollower.getReservationQueueResourceIfExists(plan,
- reservation.getReservationId());
+ Resource reservationResource = planFollower
+ .getReservationQueueResourceIfExists
+ (plan, reservation.getReservationId());
if (reservationResource != null) {
- resResource = Resources.subtract(reservation.getResourcesAtTime(now),
- reservationResource);
+ resResource =
+ Resources.subtract(
+ reservation.getResourcesAtTime(now),
+ reservationResource);
} else {
resResource = reservation.getResourcesAtTime(now);
}
@@ -458,3 +428,4 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
}
}
}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3345e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java
index 9eb1820..783fd09 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -33,10 +33,9 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation.RLEOperator;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
-import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation.RLEOperator;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.planning.Planner;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.planning.ReservationAgent;
@@ -65,14 +64,9 @@ public class InMemoryPlan implements Plan {
private RLESparseResourceAllocation rleSparseVector;
- private PeriodicRLESparseResourceAllocation periodicRle;
-
private Map<String, RLESparseResourceAllocation> userResourceAlloc =
new HashMap<String, RLESparseResourceAllocation>();
- private Map<String, RLESparseResourceAllocation> userPeriodicResourceAlloc =
- new HashMap<String, RLESparseResourceAllocation>();
-
private Map<String, RLESparseResourceAllocation> userActiveReservationCount =
new HashMap<String, RLESparseResourceAllocation>();
@@ -102,27 +96,15 @@ public class InMemoryPlan implements Plan {
String queueName, Planner replanner, boolean getMoveOnExpiry,
RMContext rmContext) {
this(queueMetrics, policy, agent, totalCapacity, step, resCalc, minAlloc,
- maxAlloc, queueName, replanner, getMoveOnExpiry,
- YarnConfiguration.DEFAULT_RM_RESERVATION_SYSTEM_MAX_PERIODICITY,
- rmContext);
+ maxAlloc, queueName, replanner, getMoveOnExpiry, rmContext,
+ new UTCClock());
}
public InMemoryPlan(QueueMetrics queueMetrics, SharingPolicy policy,
ReservationAgent agent, Resource totalCapacity, long step,
ResourceCalculator resCalc, Resource minAlloc, Resource maxAlloc,
String queueName, Planner replanner, boolean getMoveOnExpiry,
- long maxPeriodicty, RMContext rmContext) {
- this(queueMetrics, policy, agent, totalCapacity, step, resCalc, minAlloc,
- maxAlloc, queueName, replanner, getMoveOnExpiry, maxPeriodicty,
- rmContext, new UTCClock());
- }
-
- @SuppressWarnings("checkstyle:parameternumber")
- public InMemoryPlan(QueueMetrics queueMetrics, SharingPolicy policy,
- ReservationAgent agent, Resource totalCapacity, long step,
- ResourceCalculator resCalc, Resource minAlloc, Resource maxAlloc,
- String queueName, Planner replanner, boolean getMoveOnExpiry,
- long maxPeriodicty, RMContext rmContext, Clock clock) {
+ RMContext rmContext, Clock clock) {
this.queueMetrics = queueMetrics;
this.policy = policy;
this.agent = agent;
@@ -132,8 +114,6 @@ public class InMemoryPlan implements Plan {
this.minAlloc = minAlloc;
this.maxAlloc = maxAlloc;
this.rleSparseVector = new RLESparseResourceAllocation(resCalc);
- this.periodicRle =
- new PeriodicRLESparseResourceAllocation(resCalc, maxPeriodicty);
this.queueName = queueName;
this.replanner = replanner;
this.getMoveOnExpiry = getMoveOnExpiry;
@@ -146,39 +126,6 @@ public class InMemoryPlan implements Plan {
return queueMetrics;
}
- private RLESparseResourceAllocation getUserRLEResourceAllocation(String user,
- long period) {
- RLESparseResourceAllocation resAlloc = null;
- if (period > 0) {
- if (userPeriodicResourceAlloc.containsKey(user)) {
- resAlloc = userPeriodicResourceAlloc.get(user);
- } else {
- resAlloc = new PeriodicRLESparseResourceAllocation(resCalc,
- periodicRle.getTimePeriod());
- userPeriodicResourceAlloc.put(user, resAlloc);
- }
- } else {
- if (userResourceAlloc.containsKey(user)) {
- resAlloc = userResourceAlloc.get(user);
- } else {
- resAlloc = new RLESparseResourceAllocation(resCalc);
- userResourceAlloc.put(user, resAlloc);
- }
- }
- return resAlloc;
- }
-
- private void gcUserRLEResourceAllocation(String user, long period) {
- if (period > 0) {
- if (userPeriodicResourceAlloc.get(user).isEmpty()) {
- userPeriodicResourceAlloc.remove(user);
- }
- } else {
- if (userResourceAlloc.get(user).isEmpty()) {
- userResourceAlloc.remove(user);
- }
- }
- }
private void incrementAllocation(ReservationAllocation reservation) {
assert (readWriteLock.isWriteLockedByCurrentThread());
@@ -186,10 +133,11 @@ public class InMemoryPlan implements Plan {
reservation.getAllocationRequests();
// check if we have encountered the user earlier and if not add an entry
String user = reservation.getUser();
- long period = reservation.getPeriodicity();
- RLESparseResourceAllocation resAlloc =
- getUserRLEResourceAllocation(user, period);
-
+ RLESparseResourceAllocation resAlloc = userResourceAlloc.get(user);
+ if (resAlloc == null) {
+ resAlloc = new RLESparseResourceAllocation(resCalc);
+ userResourceAlloc.put(user, resAlloc);
+ }
RLESparseResourceAllocation resCount = userActiveReservationCount.get(user);
if (resCount == null) {
resCount = new RLESparseResourceAllocation(resCalc);
@@ -201,43 +149,14 @@ public class InMemoryPlan implements Plan {
for (Map.Entry<ReservationInterval, Resource> r : allocationRequests
.entrySet()) {
-
- if (period > 0L) {
- for (int i = 0; i < periodicRle.getTimePeriod() / period; i++) {
-
- long rStart = r.getKey().getStartTime() + i * period;
- long rEnd = r.getKey().getEndTime() + i * period;
-
- // handle wrap-around
- if (rEnd > periodicRle.getTimePeriod()) {
- long diff = rEnd - periodicRle.getTimePeriod();
- rEnd = periodicRle.getTimePeriod();
- ReservationInterval newInterval = new ReservationInterval(0, diff);
- periodicRle.addInterval(newInterval, r.getValue());
- resAlloc.addInterval(newInterval, r.getValue());
- }
-
- ReservationInterval newInterval =
- new ReservationInterval(rStart, rEnd);
- periodicRle.addInterval(newInterval, r.getValue());
- resAlloc.addInterval(newInterval, r.getValue());
- }
-
- } else {
- rleSparseVector.addInterval(r.getKey(), r.getValue());
- resAlloc.addInterval(r.getKey(), r.getValue());
- if (Resources.greaterThan(resCalc, totalCapacity, r.getValue(),
- ZERO_RESOURCE)) {
- earliestActive = Math.min(earliestActive, r.getKey().getStartTime());
- latestActive = Math.max(latestActive, r.getKey().getEndTime());
- }
+ resAlloc.addInterval(r.getKey(), r.getValue());
+ rleSparseVector.addInterval(r.getKey(), r.getValue());
+ if (Resources.greaterThan(resCalc, totalCapacity, r.getValue(),
+ ZERO_RESOURCE)) {
+ earliestActive = Math.min(earliestActive, r.getKey().getStartTime());
+ latestActive = Math.max(latestActive, r.getKey().getEndTime());
}
}
- // periodic reservations are active from start time and good till cancelled
- if (period > 0L) {
- earliestActive = reservation.getStartTime();
- latestActive = Long.MAX_VALUE;
- }
resCount.addInterval(new ReservationInterval(earliestActive, latestActive),
Resource.newInstance(1, 1));
}
@@ -247,55 +166,27 @@ public class InMemoryPlan implements Plan {
Map<ReservationInterval, Resource> allocationRequests =
reservation.getAllocationRequests();
String user = reservation.getUser();
- long period = reservation.getPeriodicity();
- RLESparseResourceAllocation resAlloc =
- getUserRLEResourceAllocation(user, period);
+ RLESparseResourceAllocation resAlloc = userResourceAlloc.get(user);
long earliestActive = Long.MAX_VALUE;
long latestActive = Long.MIN_VALUE;
for (Map.Entry<ReservationInterval, Resource> r : allocationRequests
.entrySet()) {
- if (period > 0L) {
- for (int i = 0; i < periodicRle.getTimePeriod() / period; i++) {
-
- long rStart = r.getKey().getStartTime() + i * period;
- long rEnd = r.getKey().getEndTime() + i * period;
-
- // handle wrap-around
- if (rEnd > periodicRle.getTimePeriod()) {
- long diff = rEnd - periodicRle.getTimePeriod();
- rEnd = periodicRle.getTimePeriod();
- ReservationInterval newInterval = new ReservationInterval(0, diff);
- periodicRle.removeInterval(newInterval, r.getValue());
- resAlloc.removeInterval(newInterval, r.getValue());
- }
-
- ReservationInterval newInterval =
- new ReservationInterval(rStart, rEnd);
- periodicRle.removeInterval(newInterval, r.getValue());
- resAlloc.removeInterval(newInterval, r.getValue());
- }
- } else {
- rleSparseVector.removeInterval(r.getKey(), r.getValue());
- resAlloc.removeInterval(r.getKey(), r.getValue());
- if (Resources.greaterThan(resCalc, totalCapacity, r.getValue(),
- ZERO_RESOURCE)) {
- earliestActive = Math.min(earliestActive, r.getKey().getStartTime());
- latestActive = Math.max(latestActive, r.getKey().getEndTime());
- }
+ resAlloc.removeInterval(r.getKey(), r.getValue());
+ rleSparseVector.removeInterval(r.getKey(), r.getValue());
+ if (Resources.greaterThan(resCalc, totalCapacity, r.getValue(),
+ ZERO_RESOURCE)) {
+ earliestActive = Math.min(earliestActive, r.getKey().getStartTime());
+ latestActive = Math.max(latestActive, r.getKey().getEndTime());
}
}
- gcUserRLEResourceAllocation(user, period);
+ if (resAlloc.isEmpty()) {
+ userResourceAlloc.remove(user);
+ }
RLESparseResourceAllocation resCount = userActiveReservationCount.get(user);
- // periodic reservations are active from start time and good till cancelled
- if (period > 0L) {
- earliestActive = reservation.getStartTime();
- latestActive = Long.MAX_VALUE;
- }
- resCount.removeInterval(
- new ReservationInterval(earliestActive, latestActive),
- Resource.newInstance(1, 1));
+ resCount.removeInterval(new ReservationInterval(earliestActive,
+ latestActive), Resource.newInstance(1, 1));
if (resCount.isEmpty()) {
userActiveReservationCount.remove(user);
}
@@ -307,9 +198,9 @@ public class InMemoryPlan implements Plan {
if (currentReservations != null) {
Set<ReservationAllocation> flattenedReservations =
new TreeSet<ReservationAllocation>();
- for (Set<InMemoryReservationAllocation> res : currentReservations
- .values()) {
- flattenedReservations.addAll(res);
+ for (Set<InMemoryReservationAllocation> reservationEntries :
+ currentReservations.values()) {
+ flattenedReservations.addAll(reservationEntries);
}
return flattenedReservations;
} else {
@@ -327,16 +218,19 @@ public class InMemoryPlan implements Plan {
InMemoryReservationAllocation inMemReservation =
(InMemoryReservationAllocation) reservation;
if (inMemReservation.getUser() == null) {
- String errMsg = "The specified Reservation with ID "
- + inMemReservation.getReservationId() + " is not mapped to any user";
+ String errMsg =
+ "The specified Reservation with ID "
+ + inMemReservation.getReservationId()
+ + " is not mapped to any user";
LOG.error(errMsg);
throw new IllegalArgumentException(errMsg);
}
writeLock.lock();
try {
if (reservationTable.containsKey(inMemReservation.getReservationId())) {
- String errMsg = "The specified Reservation with ID "
- + inMemReservation.getReservationId() + " already exists";
+ String errMsg =
+ "The specified Reservation with ID "
+ + inMemReservation.getReservationId() + " already exists";
LOG.error(errMsg);
throw new IllegalArgumentException(errMsg);
}
@@ -352,8 +246,9 @@ public class InMemoryPlan implements Plan {
getQueueName(), inMemReservation.getReservationId().toString());
}
}
- ReservationInterval searchInterval = new ReservationInterval(
- inMemReservation.getStartTime(), inMemReservation.getEndTime());
+ ReservationInterval searchInterval =
+ new ReservationInterval(inMemReservation.getStartTime(),
+ inMemReservation.getEndTime());
Set<InMemoryReservationAllocation> reservations =
currentReservations.get(searchInterval);
if (reservations == null) {
@@ -385,8 +280,9 @@ public class InMemoryPlan implements Plan {
ReservationId resId = reservation.getReservationId();
ReservationAllocation currReservation = getReservationById(resId);
if (currReservation == null) {
- String errMsg = "The specified Reservation with ID " + resId
- + " does not exist in the plan";
+ String errMsg =
+ "The specified Reservation with ID " + resId
+ + " does not exist in the plan";
LOG.error(errMsg);
throw new IllegalArgumentException(errMsg);
}
@@ -422,8 +318,9 @@ public class InMemoryPlan implements Plan {
private boolean removeReservation(ReservationAllocation reservation) {
assert (readWriteLock.isWriteLockedByCurrentThread());
- ReservationInterval searchInterval = new ReservationInterval(
- reservation.getStartTime(), reservation.getEndTime());
+ ReservationInterval searchInterval =
+ new ReservationInterval(reservation.getStartTime(),
+ reservation.getEndTime());
Set<InMemoryReservationAllocation> reservations =
currentReservations.get(searchInterval);
if (reservations != null) {
@@ -440,15 +337,16 @@ public class InMemoryPlan implements Plan {
currentReservations.remove(searchInterval);
}
} else {
- String errMsg = "The specified Reservation with ID "
- + reservation.getReservationId() + " does not exist in the plan";
+ String errMsg =
+ "The specified Reservation with ID " + reservation.getReservationId()
+ + " does not exist in the plan";
LOG.error(errMsg);
throw new IllegalArgumentException(errMsg);
}
reservationTable.remove(reservation.getReservationId());
decrementAllocation(reservation);
LOG.info("Sucessfully deleted reservation: {} in plan.",
- reservation.getReservationId());
+ reservation.getReservationId());
return true;
}
@@ -458,8 +356,9 @@ public class InMemoryPlan implements Plan {
try {
ReservationAllocation reservation = getReservationById(reservationID);
if (reservation == null) {
- String errMsg = "The specified Reservation with ID " + reservationID
- + " does not exist in the plan";
+ String errMsg =
+ "The specified Reservation with ID " + reservationID
+ + " does not exist in the plan";
LOG.error(errMsg);
throw new IllegalArgumentException(errMsg);
}
@@ -554,90 +453,66 @@ public class InMemoryPlan implements Plan {
long start, long end) {
readLock.lock();
try {
- // merge periodic and non-periodic allocations
RLESparseResourceAllocation userResAlloc = userResourceAlloc.get(user);
- RLESparseResourceAllocation userPeriodicResAlloc =
- userPeriodicResourceAlloc.get(user);
- if (userResAlloc != null && userPeriodicResAlloc != null) {
- return RLESparseResourceAllocation.merge(resCalc, totalCapacity,
- userResAlloc, userPeriodicResAlloc, RLEOperator.add, start, end);
- }
if (userResAlloc != null) {
return userResAlloc.getRangeOverlapping(start, end);
+ } else {
+ return new RLESparseResourceAllocation(resCalc);
}
- if (userPeriodicResAlloc != null) {
- return userPeriodicResAlloc.getRangeOverlapping(start, end);
- }
- } catch (PlanningException e) {
- LOG.warn("Exception while trying to merge periodic"
- + " and non-periodic user allocations: {}", e.getMessage(), e);
} finally {
readLock.unlock();
}
- return new RLESparseResourceAllocation(resCalc);
}
@Override
public Resource getTotalCommittedResources(long t) {
readLock.lock();
try {
- return Resources.add(rleSparseVector.getCapacityAtTime(t),
- periodicRle.getCapacityAtTime(t));
+ return rleSparseVector.getCapacityAtTime(t);
} finally {
readLock.unlock();
}
}
@Override
- public Set<ReservationAllocation> getReservations(ReservationId reservationID,
- ReservationInterval interval) {
+ public Set<ReservationAllocation> getReservations(ReservationId
+ reservationID, ReservationInterval interval) {
return getReservations(reservationID, interval, null);
}
@Override
- public Set<ReservationAllocation> getReservations(ReservationId reservationID,
- ReservationInterval interval, String user) {
+ public Set<ReservationAllocation> getReservations(ReservationId
+ reservationID, ReservationInterval interval, String user) {
if (reservationID != null) {
ReservationAllocation allocation = getReservationById(reservationID);
- if (allocation == null) {
+ if (allocation == null){
return Collections.emptySet();
}
return Collections.singleton(allocation);
}
- long startTime = interval == null ? 0 : interval.getStartTime();
- long endTime = interval == null ? Long.MAX_VALUE : interval.getEndTime();
+ long startTime = interval == null? 0 : interval.getStartTime();
+ long endTime = interval == null? Long.MAX_VALUE : interval.getEndTime();
ReservationInterval searchInterval =
- new ReservationInterval(endTime, Long.MAX_VALUE);
+ new ReservationInterval(endTime, Long.MAX_VALUE);
readLock.lock();
try {
- SortedMap<ReservationInterval, Set<InMemoryReservationAllocation>> res =
- currentReservations.headMap(searchInterval, true);
- if (!res.isEmpty()) {
- Set<ReservationAllocation> flattenedReservations = new HashSet<>();
- for (Set<InMemoryReservationAllocation> resEntries : res.values()) {
- for (InMemoryReservationAllocation reservation : resEntries) {
- // validate user
- if (user != null && !user.isEmpty()
- && !reservation.getUser().equals(user)) {
- continue;
- }
- // handle periodic reservations
- long period = reservation.getPeriodicity();
- if (period > 0) {
- long t = endTime % period;
- // check for both contained and wrap-around reservations
- if ((t - startTime) * (t - endTime)
- * (startTime - endTime) >= 0) {
- flattenedReservations.add(reservation);
- }
- } else {
- // check for non-periodic reservations
- if (reservation.getEndTime() > startTime) {
- flattenedReservations.add(reservation);
+ SortedMap<ReservationInterval, Set<InMemoryReservationAllocation>>
+ reservations = currentReservations.headMap(searchInterval, true);
+ if (!reservations.isEmpty()) {
+ Set<ReservationAllocation> flattenedReservations =
+ new HashSet<>();
+ for (Set<InMemoryReservationAllocation> reservationEntries :
+ reservations.values()) {
+ for (InMemoryReservationAllocation res : reservationEntries) {
+ if (res.getEndTime() > startTime) {
+ if (user != null && !user.isEmpty()
+ && !res.getUser().equals(user)) {
+ continue;
}
+ flattenedReservations.add(res);
}
}
}
@@ -675,82 +550,36 @@ public class InMemoryPlan implements Plan {
@Override
public RLESparseResourceAllocation getAvailableResourceOverTime(String user,
- ReservationId oldId, long start, long end, long period)
- throws PlanningException {
+ ReservationId oldId, long start, long end) throws PlanningException {
readLock.lock();
try {
-
- // for non-periodic return simple available resources
- if (period == 0) {
-
- // create RLE of totCapacity
- TreeMap<Long, Resource> totAvailable = new TreeMap<Long, Resource>();
- totAvailable.put(start, Resources.clone(totalCapacity));
- RLESparseResourceAllocation totRLEAvail =
- new RLESparseResourceAllocation(totAvailable, resCalc);
-
- // subtract used from available
- RLESparseResourceAllocation netAvailable;
-
- netAvailable = RLESparseResourceAllocation.merge(resCalc,
- Resources.clone(totalCapacity), totRLEAvail, rleSparseVector,
- RLEOperator.subtractTestNonNegative, start, end);
-
- // remove periodic component
- netAvailable = RLESparseResourceAllocation.merge(resCalc,
- Resources.clone(totalCapacity), netAvailable, periodicRle,
- RLEOperator.subtractTestNonNegative, start, end);
-
- // add back in old reservation used resources if any
- ReservationAllocation old = reservationTable.get(oldId);
- if (old != null) {
-
- RLESparseResourceAllocation addBackPrevious =
- old.getResourcesOverTime(start, end);
- netAvailable = RLESparseResourceAllocation.merge(resCalc,
- Resources.clone(totalCapacity), netAvailable, addBackPrevious,
- RLEOperator.add, start, end);
- }
- // lower it if this is needed by the sharing policy
- netAvailable = getSharingPolicy().availableResources(netAvailable, this,
- user, oldId, start, end);
- return netAvailable;
- } else {
-
- if (periodicRle.getTimePeriod() % period != 0) {
- throw new PlanningException("The reservation periodicity (" + period
- + ") must be" + "an exact divider of the system maxPeriod ("
- + periodicRle.getTimePeriod() + ")");
- }
-
- // find the minimum resources available among all the instances that fit
- // in the LCM
- long numInstInLCM = periodicRle.getTimePeriod() / period;
-
- RLESparseResourceAllocation minOverLCM =
- getAvailableResourceOverTime(user, oldId, start, end, 0);
- for (int i = 1; i < numInstInLCM; i++) {
-
- long rStart = start + i * period;
- long rEnd = end + i * period;
-
- // recursive invocation of non-periodic range (to pick raw-info)
- RLESparseResourceAllocation snapShot =
- getAvailableResourceOverTime(user, oldId, rStart, rEnd, 0);
-
- // time-align on start
- snapShot.shift(-(i * period));
-
- // pick the minimum amount of resources in each time interval
- minOverLCM =
- RLESparseResourceAllocation.merge(resCalc, getTotalCapacity(),
- minOverLCM, snapShot, RLEOperator.min, start, end);
-
- }
-
- return minOverLCM;
-
+ // create RLE of totCapacity
+ TreeMap<Long, Resource> totAvailable = new TreeMap<Long, Resource>();
+ totAvailable.put(start, Resources.clone(totalCapacity));
+ RLESparseResourceAllocation totRLEAvail =
+ new RLESparseResourceAllocation(totAvailable, resCalc);
+
+ // subtract used from available
+ RLESparseResourceAllocation netAvailable;
+
+ netAvailable =
+ RLESparseResourceAllocation.merge(resCalc,
+ Resources.clone(totalCapacity), totRLEAvail, rleSparseVector,
+ RLEOperator.subtractTestNonNegative, start, end);
+
+ // add back in old reservation used resources if any
+ ReservationAllocation old = reservationTable.get(oldId);
+ if (old != null) {
+ netAvailable =
+ RLESparseResourceAllocation.merge(resCalc,
+ Resources.clone(totalCapacity), netAvailable,
+ old.getResourcesOverTime(), RLEOperator.add, start, end);
}
+ // lower it if this is needed by the sharing policy
+ netAvailable =
+ getSharingPolicy().availableResources(netAvailable, this, user,
+ oldId, start, end);
+ return netAvailable;
} finally {
readLock.unlock();
}
@@ -808,7 +637,7 @@ public class InMemoryPlan implements Plan {
public String toCumulativeString() {
readLock.lock();
try {
- return rleSparseVector.toString() + "\n" + periodicRle.toString();
+ return rleSparseVector.toString();
} finally {
readLock.unlock();
}
@@ -860,18 +689,11 @@ public class InMemoryPlan implements Plan {
}
@Override
- public RLESparseResourceAllocation getCumulativeLoadOverTime(long start,
- long end) throws PlanningException {
+ public RLESparseResourceAllocation getCumulativeLoadOverTime(
+ long start, long end) {
readLock.lock();
try {
-
- RLESparseResourceAllocation ret =
- rleSparseVector.getRangeOverlapping(start, end);
- ret = RLESparseResourceAllocation.merge(resCalc, totalCapacity, ret,
- periodicRle.getRangeOverlapping(start, end), RLEOperator.add, start,
- end);
-
- return ret;
+ return rleSparseVector.getRangeOverlapping(start, end);
} finally {
readLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3345e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryReservationAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryReservationAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryReservationAllocation.java
index 00c8e44..69fd43f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryReservationAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryReservationAllocation.java
@@ -42,7 +42,6 @@ public class InMemoryReservationAllocation implements ReservationAllocation {
private final Map<ReservationInterval, Resource> allocationRequests;
private boolean hasGang = false;
private long acceptedAt = -1;
- private long periodicity = 0;
private RLESparseResourceAllocation resourcesOverTime;
@@ -68,16 +67,9 @@ public class InMemoryReservationAllocation implements ReservationAllocation {
this.allocationRequests = allocations;
this.planName = planName;
this.hasGang = hasGang;
- if (contract != null && contract.getRecurrenceExpression() != null) {
- this.periodicity = Long.parseLong(contract.getRecurrenceExpression());
- }
- if (periodicity > 0) {
- resourcesOverTime =
- new PeriodicRLESparseResourceAllocation(calculator, periodicity);
- } else {
- resourcesOverTime = new RLESparseResourceAllocation(calculator);
- }
- for (Map.Entry<ReservationInterval, Resource> r : allocations.entrySet()) {
+ resourcesOverTime = new RLESparseResourceAllocation(calculator);
+ for (Map.Entry<ReservationInterval, Resource> r : allocations
+ .entrySet()) {
resourcesOverTime.addInterval(r.getKey(), r.getValue());
}
}
@@ -141,33 +133,17 @@ public class InMemoryReservationAllocation implements ReservationAllocation {
}
@Override
- public RLESparseResourceAllocation getResourcesOverTime() {
+ public RLESparseResourceAllocation getResourcesOverTime(){
return resourcesOverTime;
}
@Override
- public RLESparseResourceAllocation getResourcesOverTime(long start,
- long end) {
- return resourcesOverTime.getRangeOverlapping(start, end);
- }
-
- @Override
- public long getPeriodicity() {
- return periodicity;
- }
-
- @Override
- public void setPeriodicity(long period) {
- periodicity = period;
- }
-
- @Override
public String toString() {
StringBuilder sBuf = new StringBuilder();
sBuf.append(getReservationId()).append(" user:").append(getUser())
.append(" startTime: ").append(getStartTime()).append(" endTime: ")
- .append(getEndTime()).append(" Periodiciy: ").append(periodicity)
- .append(" alloc:\n[").append(resourcesOverTime.toString()).append("] ");
+ .append(getEndTime()).append(" alloc:\n[")
+ .append(resourcesOverTime.toString()).append("] ");
return sBuf.toString();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3345e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java
index 49d4702..55f1d00 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java
@@ -40,7 +40,7 @@ public class NoOverCommitPolicy implements SharingPolicy {
RLESparseResourceAllocation available = plan.getAvailableResourceOverTime(
reservation.getUser(), reservation.getReservationId(),
- reservation.getStartTime(), reservation.getEndTime(), 0);
+ reservation.getStartTime(), reservation.getEndTime());
// test the reservation does not exceed what is available
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3345e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PeriodicRLESparseResourceAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PeriodicRLESparseResourceAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PeriodicRLESparseResourceAllocation.java
index 7bc44f5..8e3be8b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PeriodicRLESparseResourceAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PeriodicRLESparseResourceAllocation.java
@@ -18,94 +18,47 @@
package org.apache.hadoop.yarn.server.resourcemanager.reservation;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.TreeMap;
-
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.annotations.VisibleForTesting;
-
/**
- * This data structure stores a periodic {@link RLESparseResourceAllocation}.
+ * This data structure stores a periodic RLESparseResourceAllocation.
* Default period is 1 day (86400000ms).
*/
-public class PeriodicRLESparseResourceAllocation
- extends RLESparseResourceAllocation {
+public class PeriodicRLESparseResourceAllocation extends
+ RLESparseResourceAllocation {
// Log
- private static final Logger LOG =
- LoggerFactory.getLogger(PeriodicRLESparseResourceAllocation.class);
+ private static final Logger LOG = LoggerFactory
+ .getLogger(PeriodicRLESparseResourceAllocation.class);
private long timePeriod;
/**
* Constructor.
*
- * @param resourceCalculator {@link ResourceCalculator} the resource
- * calculator to use.
+ * @param rleVector {@link RLESparseResourceAllocation} with the run-length
+ encoded data.
* @param timePeriod Time period in milliseconds.
*/
public PeriodicRLESparseResourceAllocation(
- ResourceCalculator resourceCalculator, Long timePeriod) {
- super(resourceCalculator);
+ RLESparseResourceAllocation rleVector, Long timePeriod) {
+ super(rleVector.getCumulative(), rleVector.getResourceCalculator());
this.timePeriod = timePeriod;
}
/**
* Constructor. Default time period set to 1 day.
*
- * @param resourceCalculator {@link ResourceCalculator} the resource
- * calculator to use..
- */
- public PeriodicRLESparseResourceAllocation(
- ResourceCalculator resourceCalculator) {
- this(resourceCalculator,
- YarnConfiguration.DEFAULT_RM_RESERVATION_SYSTEM_MAX_PERIODICITY);
- }
-
- /**
- * Constructor.
- *
* @param rleVector {@link RLESparseResourceAllocation} with the run-length
- * encoded data.
- * @param timePeriod Time period in milliseconds.
+ encoded data.
*/
- @VisibleForTesting
public PeriodicRLESparseResourceAllocation(
- RLESparseResourceAllocation rleVector, Long timePeriod) {
- super(rleVector.getCumulative(), rleVector.getResourceCalculator());
- this.timePeriod = timePeriod;
-
- // make sure the PeriodicRLE is zero-based, and handles wrap-around
- long delta = (getEarliestStartTime() % timePeriod - getEarliestStartTime());
- shift(delta);
-
- List<Long> toRemove = new ArrayList<>();
- Map<Long, Resource> toAdd = new TreeMap<>();
-
- for (Map.Entry<Long, Resource> entry : cumulativeCapacity.entrySet()) {
- if (entry.getKey() > timePeriod) {
- toRemove.add(entry.getKey());
- if (entry.getValue() != null) {
- toAdd.put(timePeriod, entry.getValue());
- long prev = entry.getKey() % timePeriod;
- toAdd.put(prev, this.getCapacityAtTime(prev));
- toAdd.put(0L, entry.getValue());
- }
- }
- }
- for (Long l : toRemove) {
- cumulativeCapacity.remove(l);
- }
- cumulativeCapacity.putAll(toAdd);
+ RLESparseResourceAllocation rleVector) {
+ this(rleVector, 86400000L);
}
/**
@@ -125,25 +78,24 @@ public class PeriodicRLESparseResourceAllocation
* The interval may include 0, but the end time must be strictly less than
* timePeriod.
*
- * @param interval {@link ReservationInterval} to which the specified resource
- * is to be added.
+ * @param interval {@link ReservationInterval} to which the specified
+ * resource is to be added.
* @param resource {@link Resource} to be added to the interval specified.
* @return true if addition is successful, false otherwise
*/
- public boolean addInterval(ReservationInterval interval, Resource resource) {
+ public boolean addInterval(ReservationInterval interval,
+ Resource resource) {
long startTime = interval.getStartTime();
long endTime = interval.getEndTime();
-
if (startTime >= 0 && endTime > startTime && endTime <= timePeriod) {
return super.addInterval(interval, resource);
} else {
- LOG.info("Cannot set capacity beyond end time: " + timePeriod + " was ("
- + interval.toString() + ")");
+ LOG.info("Cannot set capacity beyond end time: " + timePeriod);
return false;
}
}
- /**
+ /**
* Removes a resource for the specified interval.
*
* @param interval the {@link ReservationInterval} for which the resource is
@@ -151,15 +103,14 @@ public class PeriodicRLESparseResourceAllocation
* @param resource the {@link Resource} to be removed.
* @return true if removal is successful, false otherwise
*/
- public boolean removeInterval(ReservationInterval interval,
- Resource resource) {
+ public boolean removeInterval(
+ ReservationInterval interval, Resource resource) {
long startTime = interval.getStartTime();
long endTime = interval.getEndTime();
// If the resource to be subtracted is less than the minimum resource in
// the range, abort removal to avoid negative capacity.
- // TODO revesit decrementing endTime
- if (!Resources.fitsIn(resource, getMinimumCapacityInInterval(
- new ReservationInterval(startTime, endTime - 1)))) {
+ if (!Resources.fitsIn(
+ resource, super.getMinimumCapacityInInterval(interval))) {
LOG.info("Request to remove more resources than what is available");
return false;
}
@@ -174,16 +125,17 @@ public class PeriodicRLESparseResourceAllocation
/**
* Get maximum capacity at periodic offsets from the specified time.
*
- * @param tick UTC time base from which offsets are specified for finding the
- * maximum capacity.
- * @param period periodic offset at which capacities are evaluated.
+ * @param tick UTC time base from which offsets are specified for finding
+ * the maximum capacity.
+ * @param period periodic offset at which capacities are evaluted.
* @return the maximum {@link Resource} across the specified time instants.
* @return true if removal is successful, false otherwise
*/
public Resource getMaximumPeriodicCapacity(long tick, long period) {
Resource maxResource;
if (period < timePeriod) {
- maxResource = super.getMaximumPeriodicCapacity(tick % timePeriod, period);
+ maxResource =
+ super.getMaximumPeriodicCapacity(tick % timePeriod, period);
} else {
// if period is greater than the length of PeriodicRLESparseAllocation,
// only a single value exists in this interval.
@@ -212,30 +164,4 @@ public class PeriodicRLESparseResourceAllocation
return ret.toString();
}
- @Override
- public RLESparseResourceAllocation getRangeOverlapping(long start, long end) {
- NavigableMap<Long, Resource> unrolledMap = new TreeMap<>();
- readLock.lock();
- try {
- long relativeStart = (start >= 0) ? start % timePeriod : 0;
- NavigableMap<Long, Resource> cumulativeMap = this.getCumulative();
- Long previous = cumulativeMap.floorKey(relativeStart);
- previous = (previous != null) ? previous : 0;
- for (long i = 0; i <= (end - start) / timePeriod; i++) {
- for (Map.Entry<Long, Resource> e : cumulativeMap.entrySet()) {
- long curKey = e.getKey() + (i * timePeriod);
- if (curKey >= previous && (start + curKey - relativeStart) <= end) {
- unrolledMap.put(curKey, e.getValue());
- }
- }
- }
- RLESparseResourceAllocation rle =
- new RLESparseResourceAllocation(unrolledMap, getResourceCalculator());
- rle.shift(start - relativeStart);
- return rle;
- } finally {
- readLock.unlock();
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3345e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanEdit.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanEdit.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanEdit.java
index 9afa324..504a250 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanEdit.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanEdit.java
@@ -28,58 +28,54 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.Plan
public interface PlanEdit extends PlanContext, PlanView {
/**
- * Add a new {@link ReservationAllocation} to the plan.
+ * Add a new {@link ReservationAllocation} to the plan
*
* @param reservation the {@link ReservationAllocation} to be added to the
* plan
* @param isRecovering flag to indicate if reservation is being added as part
* of failover or not
* @return true if addition is successful, false otherwise
- * @throws PlanningException if addition is unsuccessful
*/
- boolean addReservation(ReservationAllocation reservation,
+ public boolean addReservation(ReservationAllocation reservation,
boolean isRecovering) throws PlanningException;
/**
* Updates an existing {@link ReservationAllocation} in the plan. This is
- * required for re-negotiation.
+ * required for re-negotiation
*
* @param reservation the {@link ReservationAllocation} to be updated the plan
* @return true if update is successful, false otherwise
- * @throws PlanningException if update is unsuccessful
*/
- boolean updateReservation(ReservationAllocation reservation)
+ public boolean updateReservation(ReservationAllocation reservation)
throws PlanningException;
/**
* Delete an existing {@link ReservationAllocation} from the plan identified
* uniquely by its {@link ReservationId}. This will generally be used for
- * garbage collection.
+ * garbage collection
*
* @param reservationID the {@link ReservationAllocation} to be deleted from
* the plan identified uniquely by its {@link ReservationId}
* @return true if delete is successful, false otherwise
- * @throws PlanningException if deletion is unsuccessful
*/
- boolean deleteReservation(ReservationId reservationID)
+ public boolean deleteReservation(ReservationId reservationID)
throws PlanningException;
/**
* Method invoked to garbage collect old reservations. It cleans up expired
- * reservations that have fallen out of the sliding archival window.
+ * reservations that have fallen out of the sliding archival window
*
* @param tick the current time from which the archival window is computed
- * @throws PlanningException if archival is unsuccessful
*/
- void archiveCompletedReservations(long tick) throws PlanningException;
+ public void archiveCompletedReservations(long tick) throws PlanningException;
/**
* Sets the overall capacity in terms of {@link Resource} assigned to this
- * plan.
+ * plan
*
* @param capacity the overall capacity in terms of {@link Resource} assigned
* to this plan
*/
- void setTotalCapacity(Resource capacity);
+ public void setTotalCapacity(Resource capacity);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3345e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanView.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanView.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanView.java
index 4035f68..2767993 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanView.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanView.java
@@ -17,50 +17,50 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.reservation;
-import java.util.Set;
-
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
+import java.util.Set;
+
/**
* This interface provides a read-only view on the allocations made in this
* plan. This methods are used for example by {@code ReservationAgent}s to
* determine the free resources in a certain point in time, and by
* PlanFollowerPolicy to publish this plan to the scheduler.
*/
-interface PlanView extends PlanContext {
+public interface PlanView extends PlanContext {
/**
* Return a set of {@link ReservationAllocation} identified by the user who
* made the reservation.
*
* @param reservationID the unqiue id to identify the
- * {@link ReservationAllocation}
+ * {@link ReservationAllocation}
* @param interval the time interval used to retrieve the reservation
- * allocations from. Only reservations with start time no greater
- * than the interval end time, and end time no less than the interval
- * start time will be selected.
+ * allocations from. Only reservations with start time no
+ * greater than the interval end time, and end time no less
+ * than the interval start time will be selected.
* @param user the user to retrieve the reservation allocation from.
* @return a set of {@link ReservationAllocation} identified by the user who
- * made the reservation
+ * made the reservation
*/
- Set<ReservationAllocation> getReservations(ReservationId reservationID,
- ReservationInterval interval, String user);
+ Set<ReservationAllocation> getReservations(ReservationId
+ reservationID, ReservationInterval interval, String user);
/**
* Return a set of {@link ReservationAllocation} identified by any user.
*
* @param reservationID the unqiue id to identify the
- * {@link ReservationAllocation}
+ * {@link ReservationAllocation}
* @param interval the time interval used to retrieve the reservation
- * allocations from. Only reservations with start time no greater
- * than the interval end time, and end time no less than the interval
- * start time will be selected.
+ * allocations from. Only reservations with start time no
+ * greater than the interval end time, and end time no less
+ * than the interval start time will be selected.
* @return a set of {@link ReservationAllocation} identified by any user
*/
Set<ReservationAllocation> getReservations(ReservationId reservationID,
- ReservationInterval interval);
+ ReservationInterval interval);
/**
* Return a {@link ReservationAllocation} identified by its
@@ -70,7 +70,7 @@ interface PlanView extends PlanContext {
* {@link ReservationAllocation}
* @return {@link ReservationAllocation} identified by the specified id
*/
- ReservationAllocation getReservationById(ReservationId reservationID);
+ public ReservationAllocation getReservationById(ReservationId reservationID);
/**
* Return a set of {@link ReservationAllocation} that belongs to a certain
@@ -78,10 +78,11 @@ interface PlanView extends PlanContext {
*
* @param user the user being considered
* @param t the instant in time being considered
- * @return set of active {@link ReservationAllocation}s for this user at this
- * time
+ * @return set of active {@link ReservationAllocation}s for this
+ * user at this time
*/
- Set<ReservationAllocation> getReservationByUserAtTime(String user, long t);
+ public Set<ReservationAllocation> getReservationByUserAtTime(String user,
+ long t);
/**
* Gets all the active reservations at the specified point of time
@@ -90,14 +91,14 @@ interface PlanView extends PlanContext {
* requested
* @return set of active reservations at the specified time
*/
- Set<ReservationAllocation> getReservationsAtTime(long tick);
+ public Set<ReservationAllocation> getReservationsAtTime(long tick);
/**
* Gets all the reservations in the plan
*
* @return set of all reservations handled by this Plan
*/
- Set<ReservationAllocation> getAllReservations();
+ public Set<ReservationAllocation> getAllReservations();
/**
* Returns the total {@link Resource} reserved for all users at the specified
@@ -125,68 +126,61 @@ interface PlanView extends PlanContext {
*
* @return the time (UTC in ms) at which the first reservation starts
*/
- long getEarliestStartTime();
+ public long getEarliestStartTime();
/**
* Returns the time (UTC in ms) at which the last reservation terminates
*
* @return the time (UTC in ms) at which the last reservation terminates
*/
- long getLastEndTime();
+ public long getLastEndTime();
/**
* This method returns the amount of resources available to a given user
* (optionally if removing a certain reservation) over the start-end time
- * range. If the request is periodic (period is non-zero) we return the
- * minimum amount of resources available to periodic reservations (in all
- * "period" windows within the system maxPeriod / LCM).
+ * range.
*
- * @param user the user being considered
- * @param oldId the identifier of the existing reservation
- * @param start start of the time interval.
- * @param end end of the time interval.
- * @param period the ms periodicty for this request (loop and pick min till
- * maxPeriodicity)
+ * @param user
+ * @param oldId
+ * @param start
+ * @param end
* @return a view of the plan as it is available to this user
- * @throws PlanningException if operation is unsuccessful
+ * @throws PlanningException
*/
- RLESparseResourceAllocation getAvailableResourceOverTime(String user,
- ReservationId oldId, long start, long end, long period)
- throws PlanningException;
+ public RLESparseResourceAllocation getAvailableResourceOverTime(String user,
+ ReservationId oldId, long start, long end) throws PlanningException;
/**
* This method returns a RLE encoded view of the user reservation count
* utilization between start and end time.
*
- * @param user the user being considered
- * @param start start of the time interval.
- * @param end end of the time interval.
+ * @param user
+ * @param start
+ * @param end
* @return RLE encoded view of reservation used over time
*/
- RLESparseResourceAllocation getReservationCountForUserOverTime(String user,
- long start, long end);
+ public RLESparseResourceAllocation getReservationCountForUserOverTime(
+ String user, long start, long end);
/**
* This method returns a RLE encoded view of the user reservation utilization
* between start and end time.
*
- * @param user the user being considered
- * @param start start of the time interval.
- * @param end end of the time interval.
+ * @param user
+ * @param start
+ * @param end
* @return RLE encoded view of resources used over time
*/
- RLESparseResourceAllocation getConsumptionForUserOverTime(String user,
+ public RLESparseResourceAllocation getConsumptionForUserOverTime(String user,
long start, long end);
/**
* Get the cumulative load over a time interval.
*
- * @param start start of the time interval.
- * @param end end of the time interval.
+ * @param start Start of the time interval.
+ * @param end End of the time interval.
* @return RLE sparse allocation.
- * @throws PlanningException if operation is unsuccessful
*/
- RLESparseResourceAllocation getCumulativeLoadOverTime(long start, long end)
- throws PlanningException;
+ RLESparseResourceAllocation getCumulativeLoadOverTime(long start, long end);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/40] hadoop git commit: HDFS-10687. Federation Membership State
Store internal API. Contributed by Jason Kace and Inigo Goiri.
Posted by in...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateNamenodeRegistrationResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateNamenodeRegistrationResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateNamenodeRegistrationResponse.java
new file mode 100644
index 0000000..1f0d556
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateNamenodeRegistrationResponse.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API response for overriding an existing namenode registration in the state
+ * store.
+ */
+public abstract class UpdateNamenodeRegistrationResponse {
+
+ public static UpdateNamenodeRegistrationResponse newInstance() {
+ return StateStoreSerializer.newRecord(
+ UpdateNamenodeRegistrationResponse.class);
+ }
+
+ public static UpdateNamenodeRegistrationResponse newInstance(boolean status)
+ throws IOException {
+ UpdateNamenodeRegistrationResponse response = newInstance();
+ response.setResult(status);
+ return response;
+ }
+
+ @Private
+ @Unstable
+ public abstract boolean getResult();
+
+ @Private
+ @Unstable
+ public abstract void setResult(boolean result);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/FederationProtocolPBTranslator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/FederationProtocolPBTranslator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/FederationProtocolPBTranslator.java
new file mode 100644
index 0000000..baad113
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/FederationProtocolPBTranslator.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+
+import org.apache.commons.codec.binary.Base64;
+
+import com.google.protobuf.GeneratedMessage;
+import com.google.protobuf.Message;
+import com.google.protobuf.Message.Builder;
+import com.google.protobuf.MessageOrBuilder;
+
+/**
+ * Helper class for setting/getting data elements in an object backed by a
+ * protobuf implementation.
+ */
+public class FederationProtocolPBTranslator<P extends GeneratedMessage,
+ B extends Builder, T extends MessageOrBuilder> {
+
+ /** Optional proto byte stream used to create this object. */
+ private P proto;
+ /** The class of the proto handler for this translator. */
+ private Class<P> protoClass;
+ /** Internal builder, used to store data that has been set. */
+ private B builder;
+
+ public FederationProtocolPBTranslator(Class<P> protoType) {
+ this.protoClass = protoType;
+ }
+
+ /**
+ * Called if this translator is to be created from an existing protobuf byte
+ * stream.
+ *
+ * @param p The existing proto object to use to initialize the translator.
+ * @throws IllegalArgumentException
+ */
+ @SuppressWarnings("unchecked")
+ public void setProto(Message p) {
+ if (protoClass.isInstance(p)) {
+ if (this.builder != null) {
+ // Merge with builder
+ this.builder.mergeFrom((P) p);
+ } else {
+ // Store proto
+ this.proto = (P) p;
+ }
+ } else {
+ throw new IllegalArgumentException(
+ "Cannot decode proto type " + p.getClass().getName());
+ }
+ }
+
+ /**
+ * Create or return the cached protobuf builder for this translator.
+ *
+ * @return cached Builder instance
+ */
+ @SuppressWarnings("unchecked")
+ public B getBuilder() {
+ if (this.builder == null) {
+ try {
+ Method method = protoClass.getMethod("newBuilder");
+ this.builder = (B) method.invoke(null);
+ if (this.proto != null) {
+ // Merge in existing immutable proto
+ this.builder.mergeFrom(this.proto);
+ }
+ } catch (ReflectiveOperationException e) {
+ this.builder = null;
+ }
+ }
+ return this.builder;
+ }
+
+ /**
+ * Get the serialized proto object. If the translator was created from a byte
+ * stream, returns the intitial byte stream. Otherwise creates a new byte
+ * stream from the cached builder.
+ *
+ * @return Protobuf message object
+ */
+ @SuppressWarnings("unchecked")
+ public P build() {
+ if (this.builder != null) {
+ // serialize from builder (mutable) first
+ Message m = this.builder.build();
+ return (P) m;
+ } else if (this.proto != null) {
+ // Use immutable message source, message is unchanged
+ return this.proto;
+ }
+ return null;
+ }
+
+ /**
+ * Returns an interface to access data stored within this object. The object
+ * may have been initialized either via a builder or by an existing protobuf
+ * byte stream.
+ *
+ * @return MessageOrBuilder protobuf interface for the requested class.
+ */
+ @SuppressWarnings("unchecked")
+ public T getProtoOrBuilder() {
+ if (this.builder != null) {
+ // Use mutable builder if it exists
+ return (T) this.builder;
+ } else if (this.proto != null) {
+ // Use immutable message source
+ return (T) this.proto;
+ } else {
+ // Construct empty builder
+ return (T) this.getBuilder();
+ }
+ }
+
+ /**
+ * Read instance from base64 data.
+ * @param base64String
+ * @throws IOException
+ */
+ @SuppressWarnings("unchecked")
+ public void readInstance(String base64String) throws IOException {
+ byte[] bytes = Base64.decodeBase64(base64String);
+ Message msg = getBuilder().mergeFrom(bytes).build();
+ this.proto = (P) msg;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamenodeRegistrationsRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamenodeRegistrationsRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamenodeRegistrationsRequestPBImpl.java
new file mode 100644
index 0000000..4f7fee1
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamenodeRegistrationsRequestPBImpl.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetNamenodeRegistrationsRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetNamenodeRegistrationsRequestProtoOrBuilder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipRecordProto;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.MembershipStatePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * GetNamenodeRegistrationsRequest.
+ */
+public class GetNamenodeRegistrationsRequestPBImpl
+ extends GetNamenodeRegistrationsRequest implements PBRecord {
+
+ private FederationProtocolPBTranslator<GetNamenodeRegistrationsRequestProto,
+ GetNamenodeRegistrationsRequestProto.Builder,
+ GetNamenodeRegistrationsRequestProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<
+ GetNamenodeRegistrationsRequestProto,
+ GetNamenodeRegistrationsRequestProto.Builder,
+ GetNamenodeRegistrationsRequestProtoOrBuilder>(
+ GetNamenodeRegistrationsRequestProto.class);
+
+ public GetNamenodeRegistrationsRequestPBImpl() {
+ }
+
+ public GetNamenodeRegistrationsRequestPBImpl(
+ GetNamenodeRegistrationsRequestProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public GetNamenodeRegistrationsRequestProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public MembershipState getPartialMembership() {
+ GetNamenodeRegistrationsRequestProtoOrBuilder proto =
+ this.translator.getProtoOrBuilder();
+ if (!proto.hasMembership()){
+ return null;
+ }
+ NamenodeMembershipRecordProto memberProto = proto.getMembership();
+ return new MembershipStatePBImpl(memberProto);
+ }
+
+ @Override
+ public void setPartialMembership(MembershipState member) {
+ MembershipStatePBImpl memberPB = (MembershipStatePBImpl)member;
+ this.translator.getBuilder().setMembership(memberPB.getProto());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamenodeRegistrationsResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamenodeRegistrationsResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamenodeRegistrationsResponsePBImpl.java
new file mode 100644
index 0000000..f6be11d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamenodeRegistrationsResponsePBImpl.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetNamenodeRegistrationsResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetNamenodeRegistrationsResponseProtoOrBuilder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipRecordProto;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.MembershipStatePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * GetNamenodeRegistrationsResponse.
+ */
+public class GetNamenodeRegistrationsResponsePBImpl
+ extends GetNamenodeRegistrationsResponse implements PBRecord {
+
+ private FederationProtocolPBTranslator<GetNamenodeRegistrationsResponseProto,
+ GetNamenodeRegistrationsResponseProto.Builder,
+ GetNamenodeRegistrationsResponseProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<
+ GetNamenodeRegistrationsResponseProto,
+ GetNamenodeRegistrationsResponseProto.Builder,
+ GetNamenodeRegistrationsResponseProtoOrBuilder>(
+ GetNamenodeRegistrationsResponseProto.class);
+
+ public GetNamenodeRegistrationsResponsePBImpl() {
+ }
+
+ public GetNamenodeRegistrationsResponsePBImpl(
+ GetNamenodeRegistrationsResponseProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public GetNamenodeRegistrationsResponseProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public List<MembershipState> getNamenodeMemberships()
+ throws IOException {
+
+ List<MembershipState> ret = new ArrayList<MembershipState>();
+ List<NamenodeMembershipRecordProto> memberships =
+ this.translator.getProtoOrBuilder().getNamenodeMembershipsList();
+ for (NamenodeMembershipRecordProto memberProto : memberships) {
+ MembershipState membership = new MembershipStatePBImpl(memberProto);
+ ret.add(membership);
+ }
+
+ return ret;
+ }
+
+ @Override
+ public void setNamenodeMemberships(List<MembershipState> records)
+ throws IOException {
+ for (MembershipState member : records) {
+ if (member instanceof MembershipStatePBImpl) {
+ MembershipStatePBImpl memberPB = (MembershipStatePBImpl)member;
+ this.translator.getBuilder().addNamenodeMemberships(
+ memberPB.getProto());
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamespaceInfoRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamespaceInfoRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamespaceInfoRequestPBImpl.java
new file mode 100644
index 0000000..5f3e186
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamespaceInfoRequestPBImpl.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetNamespaceInfoRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetNamespaceInfoRequestProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetNamespaceInfoRequestProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * GetNamespaceInfoRequest.
+ */
+public class GetNamespaceInfoRequestPBImpl extends GetNamespaceInfoRequest
+ implements PBRecord {
+
+ private FederationProtocolPBTranslator<GetNamespaceInfoRequestProto,
+ Builder, GetNamespaceInfoRequestProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<GetNamespaceInfoRequestProto,
+ Builder, GetNamespaceInfoRequestProtoOrBuilder>(
+ GetNamespaceInfoRequestProto.class);
+
+ public GetNamespaceInfoRequestPBImpl() {
+ }
+
+ @Override
+ public GetNamespaceInfoRequestProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message protocol) {
+ this.translator.setProto(protocol);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamespaceInfoResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamespaceInfoResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamespaceInfoResponsePBImpl.java
new file mode 100644
index 0000000..be1b184
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamespaceInfoResponsePBImpl.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.FederationNamespaceInfoProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetNamespaceInfoResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetNamespaceInfoResponseProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * GetNamespaceInfoResponse.
+ */
+public class GetNamespaceInfoResponsePBImpl
+ extends GetNamespaceInfoResponse implements PBRecord {
+
+ private FederationProtocolPBTranslator<GetNamespaceInfoResponseProto,
+ GetNamespaceInfoResponseProto.Builder,
+ GetNamespaceInfoResponseProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<GetNamespaceInfoResponseProto,
+ GetNamespaceInfoResponseProto.Builder,
+ GetNamespaceInfoResponseProtoOrBuilder>(
+ GetNamespaceInfoResponseProto.class);
+
+ public GetNamespaceInfoResponsePBImpl() {
+ }
+
+ @Override
+ public GetNamespaceInfoResponseProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message protocol) {
+ this.translator.setProto(protocol);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public Set<FederationNamespaceInfo> getNamespaceInfo() {
+
+ Set<FederationNamespaceInfo> ret = new HashSet<FederationNamespaceInfo>();
+ List<FederationNamespaceInfoProto> namespaceList =
+ this.translator.getProtoOrBuilder().getNamespaceInfosList();
+ for (FederationNamespaceInfoProto ns : namespaceList) {
+ FederationNamespaceInfo info = new FederationNamespaceInfo(
+ ns.getBlockPoolId(), ns.getClusterId(), ns.getNameserviceId());
+ ret.add(info);
+ }
+ return ret;
+ }
+
+ @Override
+ public void setNamespaceInfo(Set<FederationNamespaceInfo> namespaceInfo) {
+ int index = 0;
+ for (FederationNamespaceInfo item : namespaceInfo) {
+ FederationNamespaceInfoProto.Builder itemBuilder =
+ FederationNamespaceInfoProto.newBuilder();
+ itemBuilder.setClusterId(item.getClusterId());
+ itemBuilder.setBlockPoolId(item.getBlockPoolId());
+ itemBuilder.setNameserviceId(item.getNameserviceId());
+ this.translator.getBuilder().addNamespaceInfos(index,
+ itemBuilder.build());
+ index++;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/NamenodeHeartbeatRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/NamenodeHeartbeatRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/NamenodeHeartbeatRequestPBImpl.java
new file mode 100644
index 0000000..d1fc73f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/NamenodeHeartbeatRequestPBImpl.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeHeartbeatRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeHeartbeatRequestProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeHeartbeatRequestProtoOrBuilder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipRecordProto;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.MembershipStatePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * NamenodeHeartbeatRequest.
+ */
+public class NamenodeHeartbeatRequestPBImpl
+ extends NamenodeHeartbeatRequest implements PBRecord {
+
+ private FederationProtocolPBTranslator<NamenodeHeartbeatRequestProto, Builder,
+ NamenodeHeartbeatRequestProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<NamenodeHeartbeatRequestProto,
+ Builder,
+ NamenodeHeartbeatRequestProtoOrBuilder>(
+ NamenodeHeartbeatRequestProto.class);
+
+ public NamenodeHeartbeatRequestPBImpl() {
+ }
+
+ @Override
+ public NamenodeHeartbeatRequestProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public MembershipState getNamenodeMembership() throws IOException {
+ NamenodeMembershipRecordProto membershipProto =
+ this.translator.getProtoOrBuilder().getNamenodeMembership();
+ MembershipState membership =
+ StateStoreSerializer.newRecord(MembershipState.class);
+ if (membership instanceof MembershipStatePBImpl) {
+ MembershipStatePBImpl membershipPB = (MembershipStatePBImpl)membership;
+ membershipPB.setProto(membershipProto);
+ return membershipPB;
+ } else {
+ throw new IOException("Cannot get membership from request");
+ }
+ }
+
+ @Override
+ public void setNamenodeMembership(MembershipState membership)
+ throws IOException {
+ if (membership instanceof MembershipStatePBImpl) {
+ MembershipStatePBImpl membershipPB = (MembershipStatePBImpl)membership;
+ NamenodeMembershipRecordProto membershipProto =
+ (NamenodeMembershipRecordProto)membershipPB.getProto();
+ this.translator.getBuilder().setNamenodeMembership(membershipProto);
+ } else {
+ throw new IOException("Cannot set mount table entry");
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/NamenodeHeartbeatResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/NamenodeHeartbeatResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/NamenodeHeartbeatResponsePBImpl.java
new file mode 100644
index 0000000..c243a6f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/NamenodeHeartbeatResponsePBImpl.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeHeartbeatResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeHeartbeatResponseProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * NamenodeHeartbeatResponse.
+ */
+public class NamenodeHeartbeatResponsePBImpl extends NamenodeHeartbeatResponse
+ implements PBRecord {
+
+ private FederationProtocolPBTranslator<NamenodeHeartbeatResponseProto,
+ NamenodeHeartbeatResponseProto.Builder,
+ NamenodeHeartbeatResponseProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<NamenodeHeartbeatResponseProto,
+ NamenodeHeartbeatResponseProto.Builder,
+ NamenodeHeartbeatResponseProtoOrBuilder>(
+ NamenodeHeartbeatResponseProto.class);
+
+ public NamenodeHeartbeatResponsePBImpl() {
+ }
+
+ @Override
+ public NamenodeHeartbeatResponseProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public boolean getResult() {
+ return this.translator.getProtoOrBuilder().getStatus();
+ }
+
+ @Override
+ public void setResult(boolean result) {
+ this.translator.getBuilder().setStatus(result);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateNamenodeRegistrationRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateNamenodeRegistrationRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateNamenodeRegistrationRequestPBImpl.java
new file mode 100644
index 0000000..5091360
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateNamenodeRegistrationRequestPBImpl.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateNamenodeRegistrationRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateNamenodeRegistrationRequestProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateNamenodeRegistrationRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * OverrideNamenodeRegistrationRequest.
+ */
+public class UpdateNamenodeRegistrationRequestPBImpl
+ extends UpdateNamenodeRegistrationRequest implements PBRecord {
+
+ private FederationProtocolPBTranslator<
+ UpdateNamenodeRegistrationRequestProto,
+ UpdateNamenodeRegistrationRequestProto.Builder,
+ UpdateNamenodeRegistrationRequestProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<
+ UpdateNamenodeRegistrationRequestProto,
+ UpdateNamenodeRegistrationRequestProto.Builder,
+ UpdateNamenodeRegistrationRequestProtoOrBuilder>(
+ UpdateNamenodeRegistrationRequestProto.class);
+
+ public UpdateNamenodeRegistrationRequestPBImpl() {
+ }
+
+ @Override
+ public UpdateNamenodeRegistrationRequestProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message protocol) {
+ this.translator.setProto(protocol);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public String getNameserviceId() {
+ return this.translator.getProtoOrBuilder().getNameserviceId();
+ }
+
+ @Override
+ public String getNamenodeId() {
+ return this.translator.getProtoOrBuilder().getNamenodeId();
+ }
+
+ @Override
+ public FederationNamenodeServiceState getState() {
+ return FederationNamenodeServiceState
+ .valueOf(this.translator.getProtoOrBuilder().getState());
+ }
+
+ @Override
+ public void setNameserviceId(String nsId) {
+ this.translator.getBuilder().setNameserviceId(nsId);
+ }
+
+ @Override
+ public void setNamenodeId(String nnId) {
+ this.translator.getBuilder().setNamenodeId(nnId);
+ }
+
+ @Override
+ public void setState(FederationNamenodeServiceState state) {
+ this.translator.getBuilder().setState(state.toString());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateNamenodeRegistrationResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateNamenodeRegistrationResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateNamenodeRegistrationResponsePBImpl.java
new file mode 100644
index 0000000..4558f06
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateNamenodeRegistrationResponsePBImpl.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateNamenodeRegistrationResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateNamenodeRegistrationResponseProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateNamenodeRegistrationResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * OverrideNamenodeRegistrationResponse.
+ */
+public class UpdateNamenodeRegistrationResponsePBImpl
+ extends UpdateNamenodeRegistrationResponse implements PBRecord {
+
+ private FederationProtocolPBTranslator<
+ UpdateNamenodeRegistrationResponseProto,
+ UpdateNamenodeRegistrationResponseProto.Builder,
+ UpdateNamenodeRegistrationResponseProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<
+ UpdateNamenodeRegistrationResponseProto,
+ UpdateNamenodeRegistrationResponseProto.Builder,
+ UpdateNamenodeRegistrationResponseProtoOrBuilder>(
+ UpdateNamenodeRegistrationResponseProto.class);
+
+ public UpdateNamenodeRegistrationResponsePBImpl() {
+ }
+
+ @Override
+ public UpdateNamenodeRegistrationResponseProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public boolean getResult() {
+ return this.translator.getProtoOrBuilder().getStatus();
+ }
+
+ @Override
+ public void setResult(boolean result) {
+ this.translator.getBuilder().setStatus(result);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/package-info.java
new file mode 100644
index 0000000..43c94be
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/package-info.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Protobuf implementations of FederationProtocolBase request/response objects
+ * used by state store APIs. Each state store API is defined in the
+ * org.apache.hadoop.hdfs.server.federation.store.protocol package.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipState.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipState.java
new file mode 100644
index 0000000..ab0ff0a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipState.java
@@ -0,0 +1,329 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.ACTIVE;
+import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.EXPIRED;
+import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.UNAVAILABLE;
+
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * Data schema for storing NN registration information in the
+ * {@link org.apache.hadoop.hdfs.server.federation.store.StateStoreService
+ * FederationStateStoreService}.
+ */
+public abstract class MembershipState extends BaseRecord
+ implements FederationNamenodeContext {
+
+ /** Expiration time in ms for this entry. */
+ private static long expirationMs;
+
+
+ /** Comparator based on the name.*/
+ public static final Comparator<MembershipState> NAME_COMPARATOR =
+ new Comparator<MembershipState>() {
+ public int compare(MembershipState m1, MembershipState m2) {
+ return m1.compareNameTo(m2);
+ }
+ };
+
+
+ /**
+ * Constructors.
+ */
+ public MembershipState() {
+ super();
+ }
+
+ /**
+ * Create a new membership instance.
+ * @return Membership instance.
+ * @throws IOException
+ */
+ public static MembershipState newInstance() {
+ MembershipState record =
+ StateStoreSerializer.newRecord(MembershipState.class);
+ record.init();
+ return record;
+ }
+
+ /**
+ * Create a new membership instance.
+ *
+ * @param router Identifier of the router.
+ * @param nameservice Identifier of the nameservice.
+ * @param namenode Identifier of the namenode.
+ * @param clusterId Identifier of the cluster.
+ * @param blockPoolId Identifier of the blockpool.
+ * @param rpcAddress RPC address.
+ * @param serviceAddress Service RPC address.
+ * @param lifelineAddress Lifeline RPC address.
+ * @param webAddress HTTP address.
+ * @param state State of the federation.
+ * @param safemode If the safe mode is enabled.
+ * @return Membership instance.
+ * @throws IOException If we cannot create the instance.
+ */
+ public static MembershipState newInstance(String router, String nameservice,
+ String namenode, String clusterId, String blockPoolId, String rpcAddress,
+ String serviceAddress, String lifelineAddress, String webAddress,
+ FederationNamenodeServiceState state, boolean safemode) {
+
+ MembershipState record = MembershipState.newInstance();
+ record.setRouterId(router);
+ record.setNameserviceId(nameservice);
+ record.setNamenodeId(namenode);
+ record.setRpcAddress(rpcAddress);
+ record.setServiceAddress(serviceAddress);
+ record.setLifelineAddress(lifelineAddress);
+ record.setWebAddress(webAddress);
+ record.setIsSafeMode(safemode);
+ record.setState(state);
+ record.setClusterId(clusterId);
+ record.setBlockPoolId(blockPoolId);
+ record.validate();
+ return record;
+ }
+
+ public abstract void setRouterId(String routerId);
+
+ public abstract String getRouterId();
+
+ public abstract void setNameserviceId(String nameserviceId);
+
+ public abstract void setNamenodeId(String namenodeId);
+
+ public abstract void setWebAddress(String webAddress);
+
+ public abstract void setRpcAddress(String rpcAddress);
+
+ public abstract void setServiceAddress(String serviceAddress);
+
+ public abstract void setLifelineAddress(String lifelineAddress);
+
+ public abstract void setIsSafeMode(boolean isSafeMode);
+
+ public abstract void setClusterId(String clusterId);
+
+ public abstract void setBlockPoolId(String blockPoolId);
+
+ public abstract void setState(FederationNamenodeServiceState state);
+
+ public abstract String getNameserviceId();
+
+ public abstract String getNamenodeId();
+
+ public abstract String getClusterId();
+
+ public abstract String getBlockPoolId();
+
+ public abstract String getRpcAddress();
+
+ public abstract String getServiceAddress();
+
+ public abstract String getLifelineAddress();
+
+ public abstract String getWebAddress();
+
+ public abstract boolean getIsSafeMode();
+
+ public abstract FederationNamenodeServiceState getState();
+
+ public abstract void setStats(MembershipStats stats);
+
+ public abstract MembershipStats getStats() throws IOException;
+
+ public abstract void setLastContact(long contact);
+
+ public abstract long getLastContact();
+
+ @Override
+ public boolean like(BaseRecord o) {
+ if (o instanceof MembershipState) {
+ MembershipState other = (MembershipState)o;
+ if (getRouterId() != null &&
+ !getRouterId().equals(other.getRouterId())) {
+ return false;
+ }
+ if (getNameserviceId() != null &&
+ !getNameserviceId().equals(other.getNameserviceId())) {
+ return false;
+ }
+ if (getNamenodeId() != null &&
+ !getNamenodeId().equals(other.getNamenodeId())) {
+ return false;
+ }
+ if (getRpcAddress() != null &&
+ !getRpcAddress().equals(other.getRpcAddress())) {
+ return false;
+ }
+ if (getClusterId() != null &&
+ !getClusterId().equals(other.getClusterId())) {
+ return false;
+ }
+ if (getBlockPoolId() != null &&
+ !getBlockPoolId().equals(other.getBlockPoolId())) {
+ return false;
+ }
+ if (getState() != null &&
+ !getState().equals(other.getState())) {
+ return false;
+ }
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return getRouterId() + "->" + getNameserviceId() + ":" + getNamenodeId()
+ + ":" + getRpcAddress() + "-" + getState();
+ }
+
+ @Override
+ public SortedMap<String, String> getPrimaryKeys() {
+ SortedMap<String, String> map = new TreeMap<String, String>();
+ map.put("routerId", getRouterId());
+ map.put("nameserviceId", getNameserviceId());
+ map.put("namenodeId", getNamenodeId());
+ return map;
+ }
+
+ /**
+ * Check if the namenode is available.
+ *
+ * @return If the namenode is available.
+ */
+ public boolean isAvailable() {
+ return getState() == ACTIVE;
+ }
+
+ /**
+ * Validates the entry. Throws an IllegalArgementException if the data record
+ * is missing required information.
+ */
+ @Override
+ public boolean validate() {
+ boolean ret = super.validate();
+ if (getNameserviceId() == null || getNameserviceId().length() == 0) {
+ //LOG.error("Invalid registration, no nameservice specified " + this);
+ ret = false;
+ }
+ if (getWebAddress() == null || getWebAddress().length() == 0) {
+ //LOG.error("Invalid registration, no web address specified " + this);
+ ret = false;
+ }
+ if (getRpcAddress() == null || getRpcAddress().length() == 0) {
+ //LOG.error("Invalid registration, no rpc address specified " + this);
+ ret = false;
+ }
+ if (!isBadState() &&
+ (getBlockPoolId().isEmpty() || getBlockPoolId().length() == 0)) {
+ //LOG.error("Invalid registration, no block pool specified " + this);
+ ret = false;
+ }
+ return ret;
+ }
+
+
+ /**
+ * Overrides the cached getBlockPoolId() with an update. The state will be
+ * reset when the cache is flushed
+ *
+ * @param newState Service state of the namenode.
+ */
+ public void overrideState(FederationNamenodeServiceState newState) {
+ this.setState(newState);
+ }
+
+ /**
+ * Sort by nameservice, namenode, and router.
+ *
+ * @param other Another membership to compare to.
+ * @return If this object goes before the parameter.
+ */
+ public int compareNameTo(MembershipState other) {
+ int ret = this.getNameserviceId().compareTo(other.getNameserviceId());
+ if (ret == 0) {
+ ret = this.getNamenodeId().compareTo(other.getNamenodeId());
+ }
+ if (ret == 0) {
+ ret = this.getRouterId().compareTo(other.getRouterId());
+ }
+ return ret;
+ }
+
+ /**
+ * Get the identifier of this namenode registration.
+ * @return Identifier of the namenode.
+ */
+ public String getNamenodeKey() {
+ return getNamenodeKey(this.getNameserviceId(), this.getNamenodeId());
+ }
+
+ /**
+ * Generate the identifier for a Namenode in the HDFS federation.
+ *
+ * @param nsId Nameservice of the Namenode.
+ * @param nnId Namenode within the Nameservice (HA).
+ * @return Namenode identifier within the federation.
+ */
+ public static String getNamenodeKey(String nsId, String nnId) {
+ return nsId + "-" + nnId;
+ }
+
+ /**
+ * Check if the membership is in a bad state (expired or unavailable).
+ * @return If the membership is in a bad state (expired or unavailable).
+ */
+ private boolean isBadState() {
+ return this.getState() == EXPIRED || this.getState() == UNAVAILABLE;
+ }
+
+ @Override
+ public boolean checkExpired(long currentTime) {
+ if (super.checkExpired(currentTime)) {
+ this.setState(EXPIRED);
+ // Commit it
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public long getExpirationMs() {
+ return MembershipState.expirationMs;
+ }
+
+ /**
+ * Set the expiration time for this class.
+ *
+ * @param time Expiration time in milliseconds.
+ */
+ public static void setExpirationMs(long time) {
+ MembershipState.expirationMs = time;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipStats.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipStats.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipStats.java
new file mode 100644
index 0000000..0bd19d9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipStats.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+import java.io.IOException;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * Data schema for storing NN stats in the
+ * {@link org.apache.hadoop.hdfs.server.federation.store.StateStoreService
+ * StateStoreService}.
+ */
+public abstract class MembershipStats extends BaseRecord {
+
+ public static MembershipStats newInstance() throws IOException {
+ MembershipStats record =
+ StateStoreSerializer.newRecord(MembershipStats.class);
+ record.init();
+ return record;
+ }
+
+ public abstract void setTotalSpace(long space);
+
+ public abstract long getTotalSpace();
+
+ public abstract void setAvailableSpace(long space);
+
+ public abstract long getAvailableSpace();
+
+ public abstract void setNumOfFiles(long files);
+
+ public abstract long getNumOfFiles();
+
+ public abstract void setNumOfBlocks(long blocks);
+
+ public abstract long getNumOfBlocks();
+
+ public abstract void setNumOfBlocksMissing(long blocks);
+
+ public abstract long getNumOfBlocksMissing();
+
+ public abstract void setNumOfBlocksPendingReplication(long blocks);
+
+ public abstract long getNumOfBlocksPendingReplication();
+
+ public abstract void setNumOfBlocksUnderReplicated(long blocks);
+
+ public abstract long getNumOfBlocksUnderReplicated();
+
+ public abstract void setNumOfBlocksPendingDeletion(long blocks);
+
+ public abstract long getNumOfBlocksPendingDeletion();
+
+ public abstract void setNumOfActiveDatanodes(int nodes);
+
+ public abstract int getNumOfActiveDatanodes();
+
+ public abstract void setNumOfDeadDatanodes(int nodes);
+
+ public abstract int getNumOfDeadDatanodes();
+
+ public abstract void setNumOfDecommissioningDatanodes(int nodes);
+
+ public abstract int getNumOfDecommissioningDatanodes();
+
+ public abstract void setNumOfDecomActiveDatanodes(int nodes);
+
+ public abstract int getNumOfDecomActiveDatanodes();
+
+ public abstract void setNumOfDecomDeadDatanodes(int nodes);
+
+ public abstract int getNumOfDecomDeadDatanodes();
+
+ @Override
+ public SortedMap<String, String> getPrimaryKeys() {
+ // This record is not stored directly, no key needed
+ SortedMap<String, String> map = new TreeMap<String, String>();
+ return map;
+ }
+
+ @Override
+ public long getExpirationMs() {
+ // This record is not stored directly, no expiration needed
+ return -1;
+ }
+
+ @Override
+ public void setDateModified(long time) {
+ // We don't store this record directly
+ }
+
+ @Override
+ public long getDateModified() {
+ // We don't store this record directly
+ return 0;
+ }
+
+ @Override
+ public void setDateCreated(long time) {
+ // We don't store this record directly
+ }
+
+ @Override
+ public long getDateCreated() {
+ // We don't store this record directly
+ return 0;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatePBImpl.java
new file mode 100644
index 0000000..805c2af
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatePBImpl.java
@@ -0,0 +1,334 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipRecordProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipRecordProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipRecordProtoOrBuilder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipStatsRecordProto;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.FederationProtocolPBTranslator;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipStats;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the MembershipState record.
+ */
+public class MembershipStatePBImpl extends MembershipState implements PBRecord {
+
+ private FederationProtocolPBTranslator<NamenodeMembershipRecordProto, Builder,
+ NamenodeMembershipRecordProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<NamenodeMembershipRecordProto,
+ Builder, NamenodeMembershipRecordProtoOrBuilder>(
+ NamenodeMembershipRecordProto.class);
+
+ public MembershipStatePBImpl() {
+ }
+
+ public MembershipStatePBImpl(NamenodeMembershipRecordProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public NamenodeMembershipRecordProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public void setRouterId(String routerId) {
+ Builder builder = this.translator.getBuilder();
+ if (routerId == null) {
+ builder.clearRouterId();
+ } else {
+ builder.setRouterId(routerId);
+ }
+ }
+
+ @Override
+ public void setNameserviceId(String nameserviceId) {
+ Builder builder = this.translator.getBuilder();
+ if (nameserviceId == null) {
+ builder.clearNameserviceId();
+ } else {
+ builder.setNameserviceId(nameserviceId);
+ }
+ }
+
+ @Override
+ public void setNamenodeId(String namenodeId) {
+ Builder builder = this.translator.getBuilder();
+ if (namenodeId == null) {
+ builder.clearNamenodeId();
+ } else {
+ builder.setNamenodeId(namenodeId);
+ }
+ }
+
+ @Override
+ public void setWebAddress(String webAddress) {
+ Builder builder = this.translator.getBuilder();
+ if (webAddress == null) {
+ builder.clearWebAddress();
+ } else {
+ builder.setWebAddress(webAddress);
+ }
+ }
+
+ @Override
+ public void setRpcAddress(String rpcAddress) {
+ Builder builder = this.translator.getBuilder();
+ if (rpcAddress == null) {
+ builder.clearRpcAddress();
+ } else {
+ builder.setRpcAddress(rpcAddress);
+ }
+ }
+
+ @Override
+ public void setServiceAddress(String serviceAddress) {
+ this.translator.getBuilder().setServiceAddress(serviceAddress);
+ }
+
+ @Override
+ public void setLifelineAddress(String lifelineAddress) {
+ Builder builder = this.translator.getBuilder();
+ if (lifelineAddress == null) {
+ builder.clearLifelineAddress();
+ } else {
+ builder.setLifelineAddress(lifelineAddress);
+ }
+ }
+
+ @Override
+ public void setIsSafeMode(boolean isSafeMode) {
+ Builder builder = this.translator.getBuilder();
+ builder.setIsSafeMode(isSafeMode);
+ }
+
+ @Override
+ public void setClusterId(String clusterId) {
+ Builder builder = this.translator.getBuilder();
+ if (clusterId == null) {
+ builder.clearClusterId();
+ } else {
+ builder.setClusterId(clusterId);
+ }
+ }
+
+ @Override
+ public void setBlockPoolId(String blockPoolId) {
+ Builder builder = this.translator.getBuilder();
+ if (blockPoolId == null) {
+ builder.clearBlockPoolId();
+ } else {
+ builder.setBlockPoolId(blockPoolId);
+ }
+ }
+
+ @Override
+ public void setState(FederationNamenodeServiceState state) {
+ Builder builder = this.translator.getBuilder();
+ if (state == null) {
+ builder.clearState();
+ } else {
+ builder.setState(state.toString());
+ }
+ }
+
+ @Override
+ public String getRouterId() {
+ NamenodeMembershipRecordProtoOrBuilder proto =
+ this.translator.getProtoOrBuilder();
+ if (!proto.hasRouterId()) {
+ return null;
+ }
+ return proto.getRouterId();
+ }
+
+ @Override
+ public String getNameserviceId() {
+ NamenodeMembershipRecordProtoOrBuilder proto =
+ this.translator.getProtoOrBuilder();
+ if (!proto.hasNameserviceId()) {
+ return null;
+ }
+ return this.translator.getProtoOrBuilder().getNameserviceId();
+ }
+
+ @Override
+ public String getNamenodeId() {
+ NamenodeMembershipRecordProtoOrBuilder proto =
+ this.translator.getProtoOrBuilder();
+ if (!proto.hasNamenodeId()) {
+ return null;
+ }
+ return this.translator.getProtoOrBuilder().getNamenodeId();
+ }
+
+ @Override
+ public String getClusterId() {
+ NamenodeMembershipRecordProtoOrBuilder proto =
+ this.translator.getProtoOrBuilder();
+ if (!proto.hasClusterId()) {
+ return null;
+ }
+ return this.translator.getProtoOrBuilder().getClusterId();
+ }
+
+ @Override
+ public String getBlockPoolId() {
+ NamenodeMembershipRecordProtoOrBuilder proto =
+ this.translator.getProtoOrBuilder();
+ if (!proto.hasBlockPoolId()) {
+ return null;
+ }
+ return this.translator.getProtoOrBuilder().getBlockPoolId();
+ }
+
+ @Override
+ public String getRpcAddress() {
+ NamenodeMembershipRecordProtoOrBuilder proto =
+ this.translator.getProtoOrBuilder();
+ if (!proto.hasRpcAddress()) {
+ return null;
+ }
+ return this.translator.getProtoOrBuilder().getRpcAddress();
+ }
+
+ @Override
+ public String getServiceAddress() {
+ NamenodeMembershipRecordProtoOrBuilder proto =
+ this.translator.getProtoOrBuilder();
+ if (!proto.hasServiceAddress()) {
+ return null;
+ }
+ return this.translator.getProtoOrBuilder().getServiceAddress();
+ }
+
+ @Override
+ public String getWebAddress() {
+ NamenodeMembershipRecordProtoOrBuilder proto =
+ this.translator.getProtoOrBuilder();
+ if (!proto.hasWebAddress()) {
+ return null;
+ }
+ return this.translator.getProtoOrBuilder().getWebAddress();
+ }
+
+ @Override
+ public String getLifelineAddress() {
+ NamenodeMembershipRecordProtoOrBuilder proto =
+ this.translator.getProtoOrBuilder();
+ if (!proto.hasLifelineAddress()) {
+ return null;
+ }
+ return this.translator.getProtoOrBuilder().getLifelineAddress();
+ }
+
+ @Override
+ public boolean getIsSafeMode() {
+ return this.translator.getProtoOrBuilder().getIsSafeMode();
+ }
+
+ @Override
+ public FederationNamenodeServiceState getState() {
+ FederationNamenodeServiceState ret =
+ FederationNamenodeServiceState.UNAVAILABLE;
+ NamenodeMembershipRecordProtoOrBuilder proto =
+ this.translator.getProtoOrBuilder();
+ if (!proto.hasState()) {
+ return null;
+ }
+ try {
+ ret = FederationNamenodeServiceState.valueOf(proto.getState());
+ } catch (IllegalArgumentException e) {
+ // Ignore this error
+ }
+ return ret;
+ }
+
+ @Override
+ public void setStats(MembershipStats stats) {
+ if (stats instanceof MembershipStatsPBImpl) {
+ MembershipStatsPBImpl statsPB = (MembershipStatsPBImpl)stats;
+ NamenodeMembershipStatsRecordProto statsProto =
+ (NamenodeMembershipStatsRecordProto)statsPB.getProto();
+ this.translator.getBuilder().setStats(statsProto);
+ }
+ }
+
+ @Override
+ public MembershipStats getStats() throws IOException {
+ NamenodeMembershipStatsRecordProto statsProto =
+ this.translator.getProtoOrBuilder().getStats();
+ MembershipStats stats =
+ StateStoreSerializer.newRecord(MembershipStats.class);
+ if (stats instanceof MembershipStatsPBImpl) {
+ MembershipStatsPBImpl statsPB = (MembershipStatsPBImpl)stats;
+ statsPB.setProto(statsProto);
+ return statsPB;
+ } else {
+ throw new IOException("Cannot get stats for the membership");
+ }
+ }
+
+ @Override
+ public void setLastContact(long contact) {
+ this.translator.getBuilder().setLastContact(contact);
+ }
+
+ @Override
+ public long getLastContact() {
+ return this.translator.getProtoOrBuilder().getLastContact();
+ }
+
+ @Override
+ public void setDateModified(long time) {
+ this.translator.getBuilder().setDateModified(time);
+ }
+
+ @Override
+ public long getDateModified() {
+ return this.translator.getProtoOrBuilder().getDateModified();
+ }
+
+ @Override
+ public void setDateCreated(long time) {
+ this.translator.getBuilder().setDateCreated(time);
+ }
+
+ @Override
+ public long getDateCreated() {
+ return this.translator.getProtoOrBuilder().getDateCreated();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatsPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatsPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatsPBImpl.java
new file mode 100644
index 0000000..9f0a167
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatsPBImpl.java
@@ -0,0 +1,191 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipStatsRecordProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipStatsRecordProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipStatsRecordProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.FederationProtocolPBTranslator;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipStats;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the MembershipStats record.
+ */
+public class MembershipStatsPBImpl extends MembershipStats
+ implements PBRecord {
+
+ private FederationProtocolPBTranslator<NamenodeMembershipStatsRecordProto,
+ Builder, NamenodeMembershipStatsRecordProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<NamenodeMembershipStatsRecordProto,
+ Builder, NamenodeMembershipStatsRecordProtoOrBuilder>(
+ NamenodeMembershipStatsRecordProto.class);
+
+ public MembershipStatsPBImpl() {
+ }
+
+ @Override
+ public NamenodeMembershipStatsRecordProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public void setTotalSpace(long space) {
+ this.translator.getBuilder().setTotalSpace(space);
+ }
+
+ @Override
+ public long getTotalSpace() {
+ return this.translator.getProtoOrBuilder().getTotalSpace();
+ }
+
+ @Override
+ public void setAvailableSpace(long space) {
+ this.translator.getBuilder().setAvailableSpace(space);
+ }
+
+ @Override
+ public long getAvailableSpace() {
+ return this.translator.getProtoOrBuilder().getAvailableSpace();
+ }
+
+ @Override
+ public void setNumOfFiles(long files) {
+ this.translator.getBuilder().setNumOfFiles(files);
+ }
+
+ @Override
+ public long getNumOfFiles() {
+ return this.translator.getProtoOrBuilder().getNumOfFiles();
+ }
+
+ @Override
+ public void setNumOfBlocks(long blocks) {
+ this.translator.getBuilder().setNumOfBlocks(blocks);
+ }
+
+ @Override
+ public long getNumOfBlocks() {
+ return this.translator.getProtoOrBuilder().getNumOfBlocks();
+ }
+
+ @Override
+ public void setNumOfBlocksMissing(long blocks) {
+ this.translator.getBuilder().setNumOfBlocksMissing(blocks);
+ }
+
+ @Override
+ public long getNumOfBlocksMissing() {
+ return this.translator.getProtoOrBuilder().getNumOfBlocksMissing();
+ }
+
+ @Override
+ public void setNumOfBlocksPendingReplication(long blocks) {
+ this.translator.getBuilder().setNumOfBlocksPendingReplication(blocks);
+ }
+
+ @Override
+ public long getNumOfBlocksPendingReplication() {
+ return this.translator.getProtoOrBuilder()
+ .getNumOfBlocksPendingReplication();
+ }
+
+ @Override
+ public void setNumOfBlocksUnderReplicated(long blocks) {
+ this.translator.getBuilder().setNumOfBlocksUnderReplicated(blocks);
+ }
+
+ @Override
+ public long getNumOfBlocksUnderReplicated() {
+ return this.translator.getProtoOrBuilder().getNumOfBlocksUnderReplicated();
+ }
+
+ @Override
+ public void setNumOfBlocksPendingDeletion(long blocks) {
+ this.translator.getBuilder().setNumOfBlocksPendingDeletion(blocks);
+ }
+
+ @Override
+ public long getNumOfBlocksPendingDeletion() {
+ return this.translator.getProtoOrBuilder().getNumOfBlocksPendingDeletion();
+ }
+
+ @Override
+ public void setNumOfActiveDatanodes(int nodes) {
+ this.translator.getBuilder().setNumOfActiveDatanodes(nodes);
+ }
+
+ @Override
+ public int getNumOfActiveDatanodes() {
+ return this.translator.getProtoOrBuilder().getNumOfActiveDatanodes();
+ }
+
+ @Override
+ public void setNumOfDeadDatanodes(int nodes) {
+ this.translator.getBuilder().setNumOfDeadDatanodes(nodes);
+ }
+
+ @Override
+ public int getNumOfDeadDatanodes() {
+ return this.translator.getProtoOrBuilder().getNumOfDeadDatanodes();
+ }
+
+ @Override
+ public void setNumOfDecommissioningDatanodes(int nodes) {
+ this.translator.getBuilder().setNumOfDecommissioningDatanodes(nodes);
+ }
+
+ @Override
+ public int getNumOfDecommissioningDatanodes() {
+ return this.translator.getProtoOrBuilder()
+ .getNumOfDecommissioningDatanodes();
+ }
+
+ @Override
+ public void setNumOfDecomActiveDatanodes(int nodes) {
+ this.translator.getBuilder().setNumOfDecomActiveDatanodes(nodes);
+ }
+
+ @Override
+ public int getNumOfDecomActiveDatanodes() {
+ return this.translator.getProtoOrBuilder().getNumOfDecomActiveDatanodes();
+ }
+
+ @Override
+ public void setNumOfDecomDeadDatanodes(int nodes) {
+ this.translator.getBuilder().setNumOfDecomDeadDatanodes(nodes);
+ }
+
+ @Override
+ public int getNumOfDecomDeadDatanodes() {
+ return this.translator.getProtoOrBuilder().getNumOfDecomDeadDatanodes();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/FederationProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/FederationProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/FederationProtocol.proto
new file mode 100644
index 0000000..487fe46
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/FederationProtocol.proto
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.hadoop.hdfs.federation.protocol.proto";
+option java_outer_classname = "HdfsServerFederationProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+package hadoop.hdfs;
+
+
+/////////////////////////////////////////////////
+// Membership
+/////////////////////////////////////////////////
+
+message NamenodeMembershipStatsRecordProto {
+ optional uint64 totalSpace = 1;
+ optional uint64 availableSpace = 2;
+
+ optional uint64 numOfFiles = 10;
+ optional uint64 numOfBlocks = 11;
+ optional uint64 numOfBlocksMissing = 12;
+ optional uint64 numOfBlocksPendingReplication = 13;
+ optional uint64 numOfBlocksUnderReplicated = 14;
+ optional uint64 numOfBlocksPendingDeletion = 15;
+
+ optional uint32 numOfActiveDatanodes = 20;
+ optional uint32 numOfDeadDatanodes = 21;
+ optional uint32 numOfDecommissioningDatanodes = 22;
+ optional uint32 numOfDecomActiveDatanodes = 23;
+ optional uint32 numOfDecomDeadDatanodes = 24;
+}
+
+message NamenodeMembershipRecordProto {
+ optional uint64 dateCreated = 1;
+ optional uint64 dateModified = 2;
+ optional uint64 lastContact = 3;
+ optional string routerId = 4;
+ optional string nameserviceId = 5;
+ optional string namenodeId = 6;
+ optional string clusterId = 7;
+ optional string blockPoolId = 8;
+ optional string webAddress = 9;
+ optional string rpcAddress = 10;
+ optional string serviceAddress = 11;
+ optional string lifelineAddress = 12;
+ optional string state = 13;
+ optional bool isSafeMode = 14;
+
+ optional NamenodeMembershipStatsRecordProto stats = 15;
+}
+
+message FederationNamespaceInfoProto {
+ optional string blockPoolId = 1;
+ optional string clusterId = 2;
+ optional string nameserviceId = 3;
+}
+
+message GetNamenodeRegistrationsRequestProto {
+ optional NamenodeMembershipRecordProto membership = 1;
+}
+
+message GetNamenodeRegistrationsResponseProto {
+ repeated NamenodeMembershipRecordProto namenodeMemberships = 1;
+}
+
+message GetExpiredRegistrationsRequestProto {
+}
+
+message GetNamespaceInfoRequestProto {
+}
+
+message GetNamespaceInfoResponseProto {
+ repeated FederationNamespaceInfoProto namespaceInfos = 1;
+}
+
+message UpdateNamenodeRegistrationRequestProto {
+ optional string nameserviceId = 1;
+ optional string namenodeId = 2;
+ optional string state = 3;
+}
+
+message UpdateNamenodeRegistrationResponseProto {
+ optional bool status = 1;
+}
+
+message NamenodeHeartbeatRequestProto {
+ optional NamenodeMembershipRecordProto namenodeMembership = 1;
+}
+
+message NamenodeHeartbeatResponseProto {
+ optional bool status = 1;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index f4d60c1..3189468 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4736,7 +4736,7 @@
<property>
<name>dfs.federation.router.namenode.resolver.client.class</name>
- <value>org.apache.hadoop.hdfs.server.federation.MockResolver</value>
+ <value>org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver</value>
<description>
Class to resolve the namenode for a subcluster.
</description>
@@ -4766,4 +4766,20 @@
</description>
</property>
+ <property>
+ <name>dfs.federation.router.cache.ttl</name>
+ <value>60000</value>
+ <description>
+ How often to refresh the State Store caches in milliseconds.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.store.membership.expiration</name>
+ <value>300000</value>
+ <description>
+ Expiration time in milliseconds for a membership record.
+ </description>
+ </property>
+
</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/40] hadoop git commit: HADOOP-14520. WASB: Block compaction for
Azure Block Blobs. Contributed by Georgi Chalakov
Posted by in...@apache.org.
HADOOP-14520. WASB: Block compaction for Azure Block Blobs.
Contributed by Georgi Chalakov
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/13eda500
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/13eda500
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/13eda500
Branch: refs/heads/HDFS-10467
Commit: 13eda5000304099d1145631f9be13ce8a00b600d
Parents: d77ed23
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 7 18:35:03 2017 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 7 18:35:03 2017 +0100
----------------------------------------------------------------------
.../fs/azure/AzureNativeFileSystemStore.java | 73 +-
.../hadoop/fs/azure/BlockBlobAppendStream.java | 1301 +++++++++++-------
.../hadoop/fs/azure/NativeAzureFileSystem.java | 77 +-
.../hadoop/fs/azure/NativeFileSystemStore.java | 5 +-
.../fs/azure/SecureStorageInterfaceImpl.java | 10 +-
.../hadoop/fs/azure/SelfRenewingLease.java | 10 +-
.../hadoop/fs/azure/StorageInterface.java | 3 +-
.../hadoop/fs/azure/StorageInterfaceImpl.java | 12 +-
.../fs/azure/SyncableDataOutputStream.java | 11 +
.../hadoop-azure/src/site/markdown/index.md | 34 +
.../hadoop/fs/azure/MockStorageInterface.java | 3 +-
.../azure/TestAzureConcurrentOutOfBandIo.java | 6 +-
...estNativeAzureFileSystemBlockCompaction.java | 266 ++++
.../src/test/resources/log4j.properties | 1 +
14 files changed, 1273 insertions(+), 539 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13eda500/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index bd8ac68..639862f 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -203,6 +203,23 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
private Set<String> pageBlobDirs;
/**
+ * Configuration key to indicate the set of directories in WASB where we
+ * should store files as block blobs with block compaction enabled.
+ *
+ * Entries can be directory paths relative to the container (e.g. "/path") or
+ * fully qualified wasb:// URIs (e.g.
+ * wasb://container@example.blob.core.windows.net/path)
+ */
+ public static final String KEY_BLOCK_BLOB_WITH_COMPACTION_DIRECTORIES =
+ "fs.azure.block.blob.with.compaction.dir";
+
+ /**
+ * The set of directories where we should store files as block blobs with
+ * block compaction enabled.
+ */
+ private Set<String> blockBlobWithCompationDirs;
+
+ /**
* Configuration key to indicate the set of directories in WASB where
* we should do atomic folder rename synchronized with createNonRecursive.
*/
@@ -527,6 +544,12 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
// User-agent
userAgentId = conf.get(USER_AGENT_ID_KEY, USER_AGENT_ID_DEFAULT);
+ // Extract the directories that should contain block blobs with compaction
+ blockBlobWithCompationDirs = getDirectorySet(
+ KEY_BLOCK_BLOB_WITH_COMPACTION_DIRECTORIES);
+ LOG.debug("Block blobs with compaction directories: {}",
+ setToString(blockBlobWithCompationDirs));
+
// Extract directories that should have atomic rename applied.
atomicRenameDirs = getDirectorySet(KEY_ATOMIC_RENAME_DIRECTORIES);
String hbaseRoot;
@@ -1165,6 +1188,17 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
}
/**
+ * Checks if the given key in Azure Storage should be stored as a block blobs
+ * with compaction enabled instead of normal block blob.
+ *
+ * @param key blob name
+ * @return true, if the file is in directory with block compaction enabled.
+ */
+ public boolean isBlockBlobWithCompactionKey(String key) {
+ return isKeyForDirectorySet(key, blockBlobWithCompationDirs);
+ }
+
+ /**
* Checks if the given key in Azure storage should have synchronized
* atomic folder rename createNonRecursive implemented.
*/
@@ -1356,7 +1390,9 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
}
@Override
- public DataOutputStream storefile(String key, PermissionStatus permissionStatus)
+ public DataOutputStream storefile(String keyEncoded,
+ PermissionStatus permissionStatus,
+ String key)
throws AzureException {
try {
@@ -1417,12 +1453,26 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
// Get the blob reference from the store's container and
// return it.
- CloudBlobWrapper blob = getBlobReference(key);
+ CloudBlobWrapper blob = getBlobReference(keyEncoded);
storePermissionStatus(blob, permissionStatus);
// Create the output stream for the Azure blob.
//
- OutputStream outputStream = openOutputStream(blob);
+ OutputStream outputStream;
+
+ if (isBlockBlobWithCompactionKey(key)) {
+ BlockBlobAppendStream blockBlobOutputStream = new BlockBlobAppendStream(
+ (CloudBlockBlobWrapper) blob,
+ keyEncoded,
+ this.uploadBlockSizeBytes,
+ true,
+ getInstrumentedContext());
+
+ outputStream = blockBlobOutputStream;
+ } else {
+ outputStream = openOutputStream(blob);
+ }
+
DataOutputStream dataOutStream = new SyncableDataOutputStream(outputStream);
return dataOutStream;
} catch (Exception e) {
@@ -2869,10 +2919,21 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
CloudBlobWrapper blob = this.container.getBlockBlobReference(key);
- BlockBlobAppendStream appendStream = new BlockBlobAppendStream((CloudBlockBlobWrapper) blob, key, bufferSize, getInstrumentedContext());
- appendStream.initialize();
+ OutputStream outputStream;
+
+ BlockBlobAppendStream blockBlobOutputStream = new BlockBlobAppendStream(
+ (CloudBlockBlobWrapper) blob,
+ key,
+ bufferSize,
+ isBlockBlobWithCompactionKey(key),
+ getInstrumentedContext());
+
+ outputStream = blockBlobOutputStream;
+
+ DataOutputStream dataOutStream = new SyncableDataOutputStream(
+ outputStream);
- return new DataOutputStream(appendStream);
+ return dataOutStream;
} catch(Exception ex) {
throw new AzureException(ex);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13eda500/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
index afb9379..84342cd 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
@@ -22,122 +22,256 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
+import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.HashMap;
-import java.util.Locale;
+import java.util.Iterator;
import java.util.List;
import java.util.UUID;
import java.util.Random;
-import java.util.TimeZone;
+import java.util.concurrent.ConcurrentLinkedDeque;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.Semaphore;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FSExceptionMessages;
import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.fs.StreamCapabilities;
+import org.apache.hadoop.fs.Syncable;
import org.apache.hadoop.fs.azure.StorageInterface.CloudBlockBlobWrapper;
-import org.eclipse.jetty.util.log.Log;
+import org.apache.hadoop.io.ElasticByteBufferPool;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.microsoft.azure.storage.AccessCondition;
import com.microsoft.azure.storage.OperationContext;
import com.microsoft.azure.storage.StorageException;
+import com.microsoft.azure.storage.StorageErrorCodeStrings;
import com.microsoft.azure.storage.blob.BlobRequestOptions;
import com.microsoft.azure.storage.blob.BlockEntry;
import com.microsoft.azure.storage.blob.BlockListingFilter;
+import com.microsoft.azure.storage.blob.BlockSearchMode;
+
+import static org.apache.hadoop.fs.StreamCapabilities.StreamCapability.HFLUSH;
+import static org.apache.hadoop.fs.StreamCapabilities.StreamCapability.HSYNC;
/**
- * Stream object that implememnts append for Block Blobs in WASB.
+ * Stream object that implements append for Block Blobs in WASB.
+ *
+ * The stream object implements hflush/hsync and block compaction. Block
+ * compaction is the process of replacing a sequence of small blocks with one
+ * big block. Azure Block blobs supports up to 50000 blocks and every
+ * hflush/hsync generates one block. When the number of blocks is above 32000,
+ * the process of compaction decreases the total number of blocks, if possible.
+ * If compaction is disabled, hflush/hsync are empty functions.
+ *
+ * The stream object uses background threads for uploading the blocks and the
+ * block blob list. Blocks can be uploaded concurrently. However, when the block
+ * list is uploaded, block uploading should stop. If a block is uploaded before
+ * the block list and the block id is not in the list, the block will be lost.
+ * If the block is uploaded after the block list and the block id is in the
+ * list, the block list upload will fail. The exclusive access for the block
+ * list upload is managed by uploadingSemaphore.
*/
-public class BlockBlobAppendStream extends OutputStream {
+public class BlockBlobAppendStream extends OutputStream implements Syncable,
+ StreamCapabilities {
+
+ /**
+ * The name of the blob/file.
+ */
private final String key;
- private final int bufferSize;
- private ByteArrayOutputStream outBuffer;
- private final CloudBlockBlobWrapper blob;
- private final OperationContext opContext;
/**
- * Variable to track if the stream has been closed.
+ * This variable tracks if this is new blob or existing one.
+ */
+ private boolean blobExist;
+
+ /**
+ * When the blob exist, to to prevent concurrent write we take a lease.
+ * Taking a lease is not necessary for new blobs.
*/
- private boolean closed = false;
+ private SelfRenewingLease lease = null;
/**
- * Variable to track if the append lease is released.
+ * The support for process of compaction is optional.
*/
+ private final boolean compactionEnabled;
- private volatile boolean leaseFreed;
+ /**
+ * The number of blocks above each block compaction is triggered.
+ */
+ private static final int DEFAULT_ACTIVATE_COMPACTION_BLOCK_COUNT = 32000;
/**
- * Variable to track if the append stream has been
- * initialized.
+ * The number of blocks above each block compaction is triggered.
*/
+ private int activateCompactionBlockCount
+ = DEFAULT_ACTIVATE_COMPACTION_BLOCK_COUNT;
- private boolean initialized = false;
+ /**
+ * The size of the output buffer. Writes store the data in outBuffer until
+ * either the size is above maxBlockSize or hflush/hsync is called.
+ */
+ private final AtomicInteger maxBlockSize;
/**
- * Last IOException encountered
+ * The current buffer where writes are stored.
*/
- private volatile IOException lastError = null;
+ private ByteBuffer outBuffer;
/**
- * List to keep track of the uncommitted azure storage
- * block ids
+ * The size of the blob that has been successfully stored in the Azure Blob
+ * service.
*/
- private final List<BlockEntry> uncommittedBlockEntries;
+ private final AtomicLong committedBlobLength = new AtomicLong(0);
- private static final int UNSET_BLOCKS_COUNT = -1;
+ /**
+ * Position of last block in the blob.
+ */
+ private volatile long blobLength = 0;
/**
- * Variable to hold the next block id to be used for azure
- * storage blocks.
+ * Minutes waiting before the close operation timed out.
*/
- private long nextBlockCount = UNSET_BLOCKS_COUNT;
+ private static final int CLOSE_UPLOAD_DELAY = 10;
/**
- * Variable to hold the block id prefix to be used for azure
- * storage blocks from azure-storage-java sdk version 4.2.0 onwards
+ * Keep alive time for the threadpool.
*/
- private String blockIdPrefix = null;
+ private static final int THREADPOOL_KEEP_ALIVE = 30;
+ /**
+ * Azure Block Blob used for the stream.
+ */
+ private final CloudBlockBlobWrapper blob;
+
+ /**
+ * Azure Storage operation context.
+ */
+ private final OperationContext opContext;
+
+ /**
+ * Commands send from client calls to the background thread pool.
+ */
+ private abstract class UploadCommand {
+
+ // the blob offset for the command
+ private final long commandBlobOffset;
+
+ // command completion latch
+ private final CountDownLatch completed = new CountDownLatch(1);
+
+ UploadCommand(long offset) {
+ this.commandBlobOffset = offset;
+ }
+
+ long getCommandBlobOffset() {
+ return commandBlobOffset;
+ }
+
+ void await() throws InterruptedException {
+ completed.await();
+ }
+
+ void awaitAsDependent() throws InterruptedException {
+ await();
+ }
+
+ void setCompleted() {
+ completed.countDown();
+ }
- private final Random sequenceGenerator = new Random();
+ void execute() throws InterruptedException, IOException {}
+
+ void dump() {}
+ }
+
+ /**
+ * The list of recent commands. Before block list is committed, all the block
+ * listed in the list must be uploaded. activeBlockCommands is used for
+ * enumerating the blocks and waiting on the latch until the block is
+ * uploaded.
+ */
+ private final ConcurrentLinkedQueue<UploadCommand> activeBlockCommands
+ = new ConcurrentLinkedQueue<>();
+
+ /**
+ * Variable to track if the stream has been closed.
+ */
+ private volatile boolean closed = false;
+
+ /**
+ * First IOException encountered.
+ */
+ private final AtomicReference<IOException> firstError
+ = new AtomicReference<>();
+
+ /**
+ * Flag set when the first error has been thrown.
+ */
+ private boolean firstErrorThrown = false;
/**
- * Time to wait to renew lease in milliseconds
+ * Semaphore for serializing block uploads with NativeAzureFileSystem.
+ *
+ * The semaphore starts with number of permits equal to the number of block
+ * upload threads. Each block upload thread needs one permit to start the
+ * upload. The put block list acquires all the permits before the block list
+ * is committed.
*/
- private static final int LEASE_RENEWAL_PERIOD = 10000;
+ private final Semaphore uploadingSemaphore = new Semaphore(
+ MAX_NUMBER_THREADS_IN_THREAD_POOL,
+ true);
/**
- * Number of times to retry for lease renewal
+ * Queue storing buffers with the size of the Azure block ready for
+ * reuse. The pool allows reusing the blocks instead of allocating new
+ * blocks. After the data is sent to the service, the buffer is returned
+ * back to the queue
*/
- private static final int MAX_LEASE_RENEWAL_RETRY_COUNT = 3;
+ private final ElasticByteBufferPool poolReadyByteBuffers
+ = new ElasticByteBufferPool();
/**
- * Time to wait before retrying to set the lease
+ * The blob's block list.
*/
- private static final int LEASE_RENEWAL_RETRY_SLEEP_PERIOD = 500;
+ private final List<BlockEntry> blockEntries = new ArrayList<>(
+ DEFAULT_CAPACITY_BLOCK_ENTRIES);
+
+ private static final int DEFAULT_CAPACITY_BLOCK_ENTRIES = 1024;
/**
- * Metadata key used on the blob to indicate append lease is active
+ * The uncommitted blob's block list.
*/
- public static final String APPEND_LEASE = "append_lease";
+ private final ConcurrentLinkedDeque<BlockEntry> uncommittedBlockEntries
+ = new ConcurrentLinkedDeque<>();
+
+ /**
+ * Variable to hold the next block id to be used for azure storage blocks.
+ */
+ private static final int UNSET_BLOCKS_COUNT = -1;
+ private long nextBlockCount = UNSET_BLOCKS_COUNT;
/**
- * Timeout value for the append lease in millisecs. If the lease is not
- * renewed within 30 seconds then another thread can acquire the append lease
- * on the blob
+ * Variable to hold the block id prefix to be used for azure storage blocks.
*/
- public static final int APPEND_LEASE_TIMEOUT = 30000;
+ private String blockIdPrefix = null;
/**
- * Metdata key used on the blob to indicate last modified time of append lease
+ * Maximum number of threads in block upload thread pool.
*/
- public static final String APPEND_LEASE_LAST_MODIFIED = "append_lease_last_modified";
+ private static final int MAX_NUMBER_THREADS_IN_THREAD_POOL = 4;
/**
* Number of times block upload needs is retried.
@@ -145,17 +279,33 @@ public class BlockBlobAppendStream extends OutputStream {
private static final int MAX_BLOCK_UPLOAD_RETRIES = 3;
/**
- * Wait time between block upload retries in millisecs.
+ * Wait time between block upload retries in milliseconds.
*/
private static final int BLOCK_UPLOAD_RETRY_INTERVAL = 1000;
- private static final Logger LOG = LoggerFactory.getLogger(BlockBlobAppendStream.class);
+ /**
+ * Logger.
+ */
+ private static final Logger LOG =
+ LoggerFactory.getLogger(BlockBlobAppendStream.class);
+ /**
+ * The absolute maximum of blocks for a blob. It includes committed and
+ * temporary blocks.
+ */
private static final int MAX_BLOCK_COUNT = 100000;
+ /**
+ * The upload thread pool executor.
+ */
private ThreadPoolExecutor ioThreadPool;
/**
+ * Azure Storage access conditions for the blob.
+ */
+ private final AccessCondition accessCondition = new AccessCondition();
+
+ /**
* Atomic integer to provide thread id for thread names for uploader threads.
*/
private final AtomicInteger threadSequenceNumber;
@@ -163,106 +313,123 @@ public class BlockBlobAppendStream extends OutputStream {
/**
* Prefix to be used for thread names for uploader threads.
*/
- private static final String THREAD_ID_PREFIX = "BlockBlobAppendStream";
-
- private static final String UTC_STR = "UTC";
+ private static final String THREAD_ID_PREFIX = "append-blockblob";
+ /**
+ * BlockBlobAppendStream constructor.
+ *
+ * @param blob
+ * Azure Block Blob
+ * @param aKey
+ * blob's name
+ * @param bufferSize
+ * the maximum size of a blob block.
+ * @param compactionEnabled
+ * is the compaction process enabled for this blob
+ * @param opContext
+ * Azure Store operation context for the blob
+ * @throws IOException
+ * if an I/O error occurs. In particular, an IOException may be
+ * thrown if the output stream cannot be used for append operations
+ */
public BlockBlobAppendStream(final CloudBlockBlobWrapper blob,
- final String aKey, final int bufferSize, final OperationContext opContext)
+ final String aKey,
+ final int bufferSize,
+ final boolean compactionEnabled,
+ final OperationContext opContext)
throws IOException {
- if (null == aKey || 0 == aKey.length()) {
- throw new IllegalArgumentException(
- "Illegal argument: The key string is null or empty");
- }
-
- if (0 >= bufferSize) {
- throw new IllegalArgumentException(
- "Illegal argument bufferSize cannot be zero or negative");
- }
-
+ Preconditions.checkArgument(StringUtils.isNotEmpty(aKey));
+ Preconditions.checkArgument(bufferSize >= 0);
this.blob = blob;
this.opContext = opContext;
this.key = aKey;
- this.bufferSize = bufferSize;
+ this.maxBlockSize = new AtomicInteger(bufferSize);
this.threadSequenceNumber = new AtomicInteger(0);
this.blockIdPrefix = null;
- setBlocksCountAndBlockIdPrefix();
+ this.compactionEnabled = compactionEnabled;
+ this.blobExist = true;
+ this.outBuffer = poolReadyByteBuffers.getBuffer(false, maxBlockSize.get());
- this.outBuffer = new ByteArrayOutputStream(bufferSize);
- this.uncommittedBlockEntries = new ArrayList<BlockEntry>();
-
- // Acquire append lease on the blob.
try {
- //Set the append lease if the value of the append lease is false
- if (!updateBlobAppendMetadata(true, false)) {
- LOG.error("Unable to set Append Lease on the Blob : {} "
- + "Possibly because another client already has a create or append stream open on the Blob", key);
- throw new IOException("Unable to set Append lease on the Blob. "
- + "Possibly because another client already had an append stream open on the Blob.");
- }
- } catch (StorageException ex) {
- LOG.error("Encountered Storage exception while acquiring append "
- + "lease on blob : {}. Storage Exception : {} ErrorCode : {}",
- key, ex, ex.getErrorCode());
+ // download the block list
+ blockEntries.addAll(
+ blob.downloadBlockList(
+ BlockListingFilter.COMMITTED,
+ new BlobRequestOptions(),
+ opContext));
+
+ blobLength = blob.getProperties().getLength();
+
+ committedBlobLength.set(blobLength);
- throw new IOException(ex);
+ // Acquiring lease on the blob.
+ lease = new SelfRenewingLease(blob, true);
+ accessCondition.setLeaseID(lease.getLeaseID());
+
+ } catch (StorageException ex) {
+ if (ex.getErrorCode().equals(StorageErrorCodeStrings.BLOB_NOT_FOUND)) {
+ blobExist = false;
+ }
+ else if (ex.getErrorCode().equals(
+ StorageErrorCodeStrings.LEASE_ALREADY_PRESENT)) {
+ throw new AzureException(
+ "Unable to set Append lease on the Blob: " + ex, ex);
+ }
+ else {
+ LOG.debug(
+ "Encountered storage exception."
+ + " StorageException : {} ErrorCode : {}",
+ ex,
+ ex.getErrorCode());
+ throw new AzureException(ex);
+ }
}
- leaseFreed = false;
+ setBlocksCountAndBlockIdPrefix(blockEntries);
+
+ this.ioThreadPool = new ThreadPoolExecutor(
+ MAX_NUMBER_THREADS_IN_THREAD_POOL,
+ MAX_NUMBER_THREADS_IN_THREAD_POOL,
+ THREADPOOL_KEEP_ALIVE,
+ TimeUnit.SECONDS,
+ new LinkedBlockingQueue<>(),
+ new UploaderThreadFactory());
}
/**
- * Helper method that starts an Append Lease renewer thread and the
- * thread pool.
+ * Set payload size of the stream.
+ * It is intended to be used for unit testing purposes only.
*/
- public synchronized void initialize() {
-
- if (initialized) {
- return;
- }
- /*
- * Start the thread for Append lease renewer.
- */
- Thread appendLeaseRenewer = new Thread(new AppendRenewer());
- appendLeaseRenewer.setDaemon(true);
- appendLeaseRenewer.setName(String.format("%s-AppendLeaseRenewer", key));
- appendLeaseRenewer.start();
-
- /*
- * Parameters to ThreadPoolExecutor:
- * corePoolSize : the number of threads to keep in the pool, even if they are idle,
- * unless allowCoreThreadTimeOut is set
- * maximumPoolSize : the maximum number of threads to allow in the pool
- * keepAliveTime - when the number of threads is greater than the core,
- * this is the maximum time that excess idle threads will
- * wait for new tasks before terminating.
- * unit - the time unit for the keepAliveTime argument
- * workQueue - the queue to use for holding tasks before they are executed
- * This queue will hold only the Runnable tasks submitted by the execute method.
- */
- this.ioThreadPool = new ThreadPoolExecutor(4, 4, 2, TimeUnit.SECONDS,
- new LinkedBlockingQueue<Runnable>(), new UploaderThreadFactory());
+ @VisibleForTesting
+ synchronized void setMaxBlockSize(int size) {
+ maxBlockSize.set(size);
- initialized = true;
+ // it is for testing only so we can abandon the previously allocated
+ // payload
+ this.outBuffer = ByteBuffer.allocate(maxBlockSize.get());
}
/**
- * Get the blob name.
- *
- * @return String Blob name.
+ * Set compaction parameters.
+ * It is intended to be used for unit testing purposes only.
*/
- public String getKey() {
- return key;
+ @VisibleForTesting
+ void setCompactionBlockCount(int activationCount) {
+ activateCompactionBlockCount = activationCount;
}
/**
- * Get the backing blob.
- * @return buffer size of the stream.
+ * Get the list of block entries. It is used for testing purposes only.
+ * @return List of block entries.
*/
- public int getBufferSize() {
- return bufferSize;
+ @VisibleForTesting
+ List<BlockEntry> getBlockList() throws StorageException, IOException {
+ return blob.downloadBlockList(
+ BlockListingFilter.COMMITTED,
+ new BlobRequestOptions(),
+ opContext);
}
/**
@@ -283,21 +450,6 @@ public class BlockBlobAppendStream extends OutputStream {
}
/**
- * Writes b.length bytes from the specified byte array to this output stream.
- *
- * @param data
- * the byte array to write.
- *
- * @throws IOException
- * if an I/O error occurs. In particular, an IOException may be
- * thrown if the output stream has been closed.
- */
- @Override
- public void write(final byte[] data) throws IOException {
- write(data, 0, data.length);
- }
-
- /**
* Writes length bytes from the specified byte array starting at offset to
* this output stream.
*
@@ -312,529 +464,678 @@ public class BlockBlobAppendStream extends OutputStream {
* thrown if the output stream has been closed.
*/
@Override
- public void write(final byte[] data, final int offset, final int length)
+ public synchronized void write(final byte[] data, int offset, int length)
throws IOException {
+ Preconditions.checkArgument(data != null, "null data");
if (offset < 0 || length < 0 || length > data.length - offset) {
- throw new IndexOutOfBoundsException("write API in append stream called with invalid arguments");
- }
-
- writeInternal(data, offset, length);
- }
-
- @Override
- public synchronized void close() throws IOException {
-
- if (!initialized) {
- throw new IOException("Trying to close an uninitialized Append stream");
+ throw new IndexOutOfBoundsException();
}
if (closed) {
- return;
+ throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
}
- if (leaseFreed) {
- throw new IOException(String.format("Attempting to close an append stream on blob : %s "
- + " that does not have lease on the Blob. Failing close", key));
- }
+ while (outBuffer.remaining() < length) {
+
+ int remaining = outBuffer.remaining();
+ outBuffer.put(data, offset, remaining);
+
+ // upload payload to azure storage
+ addBlockUploadCommand();
- if (outBuffer.size() > 0) {
- uploadBlockToStorage(outBuffer.toByteArray());
+ offset += remaining;
+ length -= remaining;
}
- ioThreadPool.shutdown();
+ outBuffer.put(data, offset, length);
+ }
- try {
- if (!ioThreadPool.awaitTermination(10, TimeUnit.MINUTES)) {
- LOG.error("Time out occurred while waiting for IO request to finish in append"
- + " for blob : {}", key);
- NativeAzureFileSystemHelper.logAllLiveStackTraces();
- throw new IOException("Timed out waiting for IO requests to finish");
- }
- } catch(InterruptedException intrEx) {
- // Restore the interrupted status
- Thread.currentThread().interrupt();
- LOG.error("Upload block operation in append interrupted for blob {}. Failing close", key);
- throw new IOException("Append Commit interrupted.");
- }
+ /**
+ * Flushes this output stream and forces any buffered output bytes to be
+ * written out. If any data remains in the payload it is committed to the
+ * service. Data is queued for writing and forced out to the service
+ * before the call returns.
+ */
+ @Override
+ public void flush() throws IOException {
- // Calling commit after all blocks are succesfully uploaded.
- if (lastError == null) {
- commitAppendBlocks();
+ if (closed) {
+ // calling close() after the stream is closed starts with call to flush()
+ return;
}
- // Perform cleanup.
- cleanup();
+ addBlockUploadCommand();
- if (lastError != null) {
- throw lastError;
+ if (committedBlobLength.get() < blobLength) {
+ try {
+ // wait until the block list is committed
+ addFlushCommand().await();
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ }
}
}
/**
- * Helper method that cleans up the append stream.
+ * Force all data in the output stream to be written to Azure storage.
+ * Wait to return until this is complete.
*/
- private synchronized void cleanup() {
-
- closed = true;
-
- try {
- // Set the value of append lease to false if the value is set to true.
- updateBlobAppendMetadata(false, true);
- } catch(StorageException ex) {
- LOG.debug("Append metadata update on the Blob : {} encountered Storage Exception : {} "
- + "Error Code : {}",
- key, ex, ex.getErrorCode());
- lastError = new IOException(ex);
+ @Override
+ public void hsync() throws IOException {
+ // when block compaction is disabled, hsync is empty function
+ if (compactionEnabled) {
+ flush();
}
+ }
- leaseFreed = true;
+ /**
+ * Force all data in the output stream to be written to Azure storage.
+ * Wait to return until this is complete.
+ */
+ @Override
+ public void hflush() throws IOException {
+ // when block compaction is disabled, hflush is empty function
+ if (compactionEnabled) {
+ flush();
+ }
}
/**
- * Method to commit all the uncommited blocks to azure storage.
- * If the commit fails then blocks are automatically cleaned up
- * by Azure storage.
- * @throws IOException
+ * The Synchronization capabilities of this stream depend upon the compaction
+ * policy.
+ * @param capability string to query the stream support for.
+ * @return true for hsync and hflush when compaction is enabled.
*/
- private synchronized void commitAppendBlocks() throws IOException {
+ @Override
+ public boolean hasCapability(String capability) {
+ return compactionEnabled
+ && (capability.equalsIgnoreCase(HSYNC.getValue())
+ || capability.equalsIgnoreCase((HFLUSH.getValue())));
+ }
- SelfRenewingLease lease = null;
+ /**
+ * Force all data in the output stream to be written to Azure storage.
+ * Wait to return until this is complete. Close the access to the stream and
+ * shutdown the upload thread pool.
+ * If the blob was created, its lease will be released.
+ * Any error encountered caught in threads and stored will be rethrown here
+ * after cleanup.
+ */
+ @Override
+ public synchronized void close() throws IOException {
- try {
- if (uncommittedBlockEntries.size() > 0) {
+ LOG.debug("close {} ", key);
- //Acquiring lease on the blob.
- lease = new SelfRenewingLease(blob);
+ if (closed) {
+ return;
+ }
- // Downloading existing blocks
- List<BlockEntry> blockEntries = blob.downloadBlockList(BlockListingFilter.COMMITTED,
- new BlobRequestOptions(), opContext);
+ // Upload the last block regardless of compactionEnabled flag
+ flush();
- // Adding uncommitted blocks.
- blockEntries.addAll(uncommittedBlockEntries);
+ // Initiates an orderly shutdown in which previously submitted tasks are
+ // executed.
+ ioThreadPool.shutdown();
- AccessCondition accessCondition = new AccessCondition();
- accessCondition.setLeaseID(lease.getLeaseID());
- blob.commitBlockList(blockEntries, accessCondition, new BlobRequestOptions(), opContext);
- uncommittedBlockEntries.clear();
+ try {
+ // wait up to CLOSE_UPLOAD_DELAY minutes to upload all the blocks
+ if (!ioThreadPool.awaitTermination(CLOSE_UPLOAD_DELAY, TimeUnit.MINUTES)) {
+ LOG.error("Time out occurred while close() is waiting for IO request to"
+ + " finish in append"
+ + " for blob : {}",
+ key);
+ NativeAzureFileSystemHelper.logAllLiveStackTraces();
+ throw new AzureException("Timed out waiting for IO requests to finish");
}
- } catch(StorageException ex) {
- LOG.error("Storage exception encountered during block commit phase of append for blob"
- + " : {} Storage Exception : {} Error Code: {}", key, ex, ex.getErrorCode());
- throw new IOException("Encountered Exception while committing append blocks", ex);
- } finally {
- if (lease != null) {
+ } catch(InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ }
+
+ // release the lease
+ if (firstError.get() == null && blobExist) {
try {
lease.free();
- } catch(StorageException ex) {
- LOG.debug("Exception encountered while releasing lease for "
- + "blob : {} StorageException : {} ErrorCode : {}", key, ex, ex.getErrorCode());
- // Swallowing exception here as the lease is cleaned up by the SelfRenewingLease object.
+ } catch (StorageException ex) {
+ LOG.debug("Lease free update blob {} encountered Storage Exception:"
+ + " {} Error Code : {}",
+ key,
+ ex,
+ ex.getErrorCode());
+ maybeSetFirstError(new AzureException(ex));
}
- }
+ }
+
+ closed = true;
+
+ // finally, throw the first exception raised if it has not
+ // been thrown elsewhere.
+ if (firstError.get() != null && !firstErrorThrown) {
+ throw firstError.get();
}
}
/**
- * Helper method used to generate the blockIDs. The algorithm used is similar to the Azure
- * storage SDK.
+ * Helper method used to generate the blockIDs. The algorithm used is similar
+ * to the Azure storage SDK.
*/
- private void setBlocksCountAndBlockIdPrefix() throws IOException {
+ private void setBlocksCountAndBlockIdPrefix(List<BlockEntry> blockEntries) {
- try {
+ if (nextBlockCount == UNSET_BLOCKS_COUNT && blockIdPrefix == null) {
- if (nextBlockCount == UNSET_BLOCKS_COUNT && blockIdPrefix==null) {
+ Random sequenceGenerator = new Random();
- List<BlockEntry> blockEntries =
- blob.downloadBlockList(BlockListingFilter.COMMITTED, new BlobRequestOptions(), opContext);
+ String blockZeroBlockId = (!blockEntries.isEmpty())
+ ? blockEntries.get(0).getId()
+ : "";
+ String prefix = UUID.randomUUID().toString() + "-";
+ String sampleNewerVersionBlockId = generateNewerVersionBlockId(prefix,
+ 0);
- String blockZeroBlockId = (blockEntries.size() > 0) ? blockEntries.get(0).getId() : "";
- String prefix = UUID.randomUUID().toString() + "-";
- String sampleNewerVersionBlockId = generateNewerVersionBlockId(prefix, 0);
+ if (!blockEntries.isEmpty()
+ && blockZeroBlockId.length() < sampleNewerVersionBlockId.length()) {
- if (blockEntries.size() > 0 && blockZeroBlockId.length() < sampleNewerVersionBlockId.length()) {
+ // If blob has already been created with 2.2.0, append subsequent blocks
+ // with older version (2.2.0) blockId compute nextBlockCount, the way it
+ // was done before; and don't use blockIdPrefix
+ this.blockIdPrefix = "";
+ nextBlockCount = (long) (sequenceGenerator.nextInt(Integer.MAX_VALUE))
+ + sequenceGenerator.nextInt(
+ Integer.MAX_VALUE - MAX_BLOCK_COUNT);
+ nextBlockCount += blockEntries.size();
- // If blob has already been created with 2.2.0, append subsequent blocks with older version (2.2.0) blockId
- // compute nextBlockCount, the way it was done before; and don't use blockIdPrefix
- this.blockIdPrefix = "";
- nextBlockCount = (long) (sequenceGenerator.nextInt(Integer.MAX_VALUE))
- + sequenceGenerator.nextInt(Integer.MAX_VALUE - MAX_BLOCK_COUNT);
- nextBlockCount += blockEntries.size();
-
- } else {
-
- // If there are no existing blocks, create the first block with newer version (4.2.0) blockId
- // If blob has already been created with 4.2.0, append subsequent blocks with newer version (4.2.0) blockId
- this.blockIdPrefix = prefix;
- nextBlockCount = blockEntries.size();
-
- }
+ } else {
+ // If there are no existing blocks, create the first block with newer
+ // version (4.2.0) blockId. If blob has already been created with 4.2.0,
+ // append subsequent blocks with newer version (4.2.0) blockId
+ this.blockIdPrefix = prefix;
+ nextBlockCount = blockEntries.size();
}
-
- } catch (StorageException ex) {
- LOG.debug("Encountered storage exception during setting next Block Count and BlockId prefix."
- + " StorageException : {} ErrorCode : {}", ex, ex.getErrorCode());
- throw new IOException(ex);
}
}
/**
- * Helper method that generates the next block id for uploading a block to azure storage.
+ * Helper method that generates the next block id for uploading a block to
+ * azure storage.
* @return String representing the block ID generated.
- * @throws IOException
+ * @throws IOException if the stream is in invalid state
*/
private String generateBlockId() throws IOException {
- if (nextBlockCount == UNSET_BLOCKS_COUNT) {
- throw new IOException("Append Stream in invalid state. nextBlockCount not set correctly");
- }
-
- if (this.blockIdPrefix == null) {
- throw new IOException("Append Stream in invalid state. blockIdPrefix not set correctly");
- }
-
- if (!this.blockIdPrefix.equals("")) {
-
- return generateNewerVersionBlockId(this.blockIdPrefix, nextBlockCount++);
-
- } else {
-
- return generateOlderVersionBlockId(nextBlockCount++);
-
+ if (nextBlockCount == UNSET_BLOCKS_COUNT || blockIdPrefix == null) {
+ throw new AzureException(
+ "Append Stream in invalid state. nextBlockCount not set correctly");
}
+ return (!blockIdPrefix.isEmpty())
+ ? generateNewerVersionBlockId(blockIdPrefix, nextBlockCount++)
+ : generateOlderVersionBlockId(nextBlockCount++);
}
/**
- * Helper method that generates an older (2.2.0) version blockId
+ * Helper method that generates an older (2.2.0) version blockId.
* @return String representing the block ID generated.
*/
private String generateOlderVersionBlockId(long id) {
- byte[] blockIdInBytes = getBytesFromLong(id);
- return new String(Base64.encodeBase64(blockIdInBytes), StandardCharsets.UTF_8);
+ byte[] blockIdInBytes = new byte[8];
+ for (int m = 0; m < 8; m++) {
+ blockIdInBytes[7 - m] = (byte) ((id >> (8 * m)) & 0xFF);
+ }
+
+ return new String(
+ Base64.encodeBase64(blockIdInBytes),
+ StandardCharsets.UTF_8);
}
/**
- * Helper method that generates an newer (4.2.0) version blockId
+ * Helper method that generates an newer (4.2.0) version blockId.
* @return String representing the block ID generated.
*/
private String generateNewerVersionBlockId(String prefix, long id) {
String blockIdSuffix = String.format("%06d", id);
- byte[] blockIdInBytes = (prefix + blockIdSuffix).getBytes(StandardCharsets.UTF_8);
+ byte[] blockIdInBytes =
+ (prefix + blockIdSuffix).getBytes(StandardCharsets.UTF_8);
return new String(Base64.encodeBase64(blockIdInBytes), StandardCharsets.UTF_8);
}
/**
- * Returns a byte array that represents the data of a <code>long</code> value. This
- * utility method is copied from com.microsoft.azure.storage.core.Utility class.
- * This class is marked as internal, hence we clone the method here and not express
- * dependency on the Utility Class
- *
- * @param value
- * The value from which the byte array will be returned.
- *
- * @return A byte array that represents the data of the specified <code>long</code> value.
+ * This is shared between upload block Runnable and CommitBlockList. The
+ * method captures retry logic
+ * @param blockId block name
+ * @param dataPayload block content
*/
- private static byte[] getBytesFromLong(final long value) {
+ private void writeBlockRequestInternal(String blockId,
+ ByteBuffer dataPayload,
+ boolean bufferPoolBuffer) {
+ IOException lastLocalException = null;
+
+ int uploadRetryAttempts = 0;
+ while (uploadRetryAttempts < MAX_BLOCK_UPLOAD_RETRIES) {
+ try {
+ long startTime = System.nanoTime();
+
+ blob.uploadBlock(blockId, accessCondition, new ByteArrayInputStream(
+ dataPayload.array()), dataPayload.position(),
+ new BlobRequestOptions(), opContext);
- final byte[] tempArray = new byte[8];
+ LOG.debug("upload block finished for {} ms. block {} ",
+ TimeUnit.NANOSECONDS.toMillis(
+ System.nanoTime() - startTime), blockId);
+ break;
+
+ } catch(Exception ioe) {
+ LOG.debug("Encountered exception during uploading block for Blob {}"
+ + " Exception : {}", key, ioe);
+ uploadRetryAttempts++;
+ lastLocalException = new AzureException(
+ "Encountered Exception while uploading block: " + ioe, ioe);
+ try {
+ Thread.sleep(
+ BLOCK_UPLOAD_RETRY_INTERVAL * (uploadRetryAttempts + 1));
+ } catch(InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ break;
+ }
+ }
+ }
- for (int m = 0; m < 8; m++) {
- tempArray[7 - m] = (byte) ((value >> (8 * m)) & 0xFF);
+ if (bufferPoolBuffer) {
+ poolReadyByteBuffers.putBuffer(dataPayload);
}
- return tempArray;
+ if (uploadRetryAttempts == MAX_BLOCK_UPLOAD_RETRIES) {
+ maybeSetFirstError(lastLocalException);
+ }
}
/**
- * Helper method that creates a thread to upload a block to azure storage.
- * @param payload
- * @throws IOException
+ * Set {@link #firstError} to the exception if it is not already set.
+ * @param exception exception to save
*/
- private synchronized void uploadBlockToStorage(byte[] payload)
- throws IOException {
-
- // upload payload to azure storage
- String blockId = generateBlockId();
-
- // Since uploads of the Azure storage are done in parallel threads, we go ahead
- // add the blockId in the uncommitted list. If the upload of the block fails
- // we don't commit the blockIds.
- BlockEntry blockEntry = new BlockEntry(blockId);
- blockEntry.setSize(payload.length);
- uncommittedBlockEntries.add(blockEntry);
- ioThreadPool.execute(new WriteRequest(payload, blockId));
+ private void maybeSetFirstError(IOException exception) {
+ firstError.compareAndSet(null, exception);
}
/**
- * Helper method to updated the Blob metadata during Append lease operations.
- * Blob metadata is updated to holdLease value only if the current lease
- * status is equal to testCondition and the last update on the blob metadata
- * is less that 30 secs old.
- * @param holdLease
- * @param testCondition
- * @return true if the updated lease operation was successful or false otherwise
- * @throws StorageException
+ * Throw the first error caught if it has not been raised already
+ * @throws IOException if one is caught and needs to be thrown.
*/
- private boolean updateBlobAppendMetadata(boolean holdLease, boolean testCondition)
- throws StorageException {
-
- SelfRenewingLease lease = null;
- StorageException lastStorageException = null;
- int leaseRenewalRetryCount = 0;
-
- /*
- * Updating the Blob metadata honours following algorithm based on
- * 1) If the append lease metadata is present
- * 2) Last updated time of the append lease
- * 3) Previous value of the Append lease metadata.
- *
- * The algorithm:
- * 1) If append lease metadata is not part of the Blob. In this case
- * this is the first client to Append so we update the metadata.
- * 2) If append lease metadata is present and timeout has occurred.
- * In this case irrespective of what the value of the append lease is we update the metadata.
- * 3) If append lease metadata is present and is equal to testCondition value (passed as parameter)
- * and timeout has not occurred, we update the metadata.
- * 4) If append lease metadata is present and is not equal to testCondition value (passed as parameter)
- * and timeout has not occurred, we do not update metadata and return false.
- *
- */
- while (leaseRenewalRetryCount < MAX_LEASE_RENEWAL_RETRY_COUNT) {
-
- lastStorageException = null;
-
- synchronized(this) {
- try {
-
- final Calendar currentCalendar = Calendar
- .getInstance(Locale.US);
- currentCalendar.setTimeZone(TimeZone.getTimeZone(UTC_STR));
- long currentTime = currentCalendar.getTime().getTime();
+ private void maybeThrowFirstError() throws IOException {
+ if (firstError.get() != null) {
+ firstErrorThrown = true;
+ throw firstError.get();
+ }
+ }
- // Acquire lease on the blob.
- lease = new SelfRenewingLease(blob);
+ /**
+ * Write block list. The method captures retry logic
+ */
+ private void writeBlockListRequestInternal() {
- blob.downloadAttributes(opContext);
- HashMap<String, String> metadata = blob.getMetadata();
+ IOException lastLocalException = null;
- if (metadata.containsKey(APPEND_LEASE)
- && currentTime - Long.parseLong(
- metadata.get(APPEND_LEASE_LAST_MODIFIED)) <= BlockBlobAppendStream.APPEND_LEASE_TIMEOUT
- && !metadata.get(APPEND_LEASE).equals(Boolean.toString(testCondition))) {
- return false;
- }
+ int uploadRetryAttempts = 0;
+ while (uploadRetryAttempts < MAX_BLOCK_UPLOAD_RETRIES) {
+ try {
- metadata.put(APPEND_LEASE, Boolean.toString(holdLease));
- metadata.put(APPEND_LEASE_LAST_MODIFIED, Long.toString(currentTime));
- blob.setMetadata(metadata);
- AccessCondition accessCondition = new AccessCondition();
- accessCondition.setLeaseID(lease.getLeaseID());
- blob.uploadMetadata(accessCondition, null, opContext);
- return true;
+ long startTime = System.nanoTime();
- } catch (StorageException ex) {
-
- lastStorageException = ex;
- LOG.debug("Lease renewal for Blob : {} encountered Storage Exception : {} "
- + "Error Code : {}",
- key, ex, ex.getErrorCode());
- leaseRenewalRetryCount++;
-
- } finally {
-
- if (lease != null) {
- try {
- lease.free();
- } catch(StorageException ex) {
- LOG.debug("Encountered Storage exception while releasing lease for Blob {} "
- + "during Append metadata operation. Storage Exception {} "
- + "Error Code : {} ", key, ex, ex.getErrorCode());
- } finally {
- lease = null;
- }
- }
- }
- }
+ blob.commitBlockList(blockEntries, accessCondition,
+ new BlobRequestOptions(), opContext);
- if (leaseRenewalRetryCount == MAX_LEASE_RENEWAL_RETRY_COUNT) {
- throw lastStorageException;
- } else {
+ LOG.debug("Upload block list took {} ms for blob {} ",
+ TimeUnit.NANOSECONDS.toMillis(
+ System.nanoTime() - startTime), key);
+ break;
+
+ } catch(Exception ioe) {
+ LOG.debug("Encountered exception during uploading block for Blob {}"
+ + " Exception : {}", key, ioe);
+ uploadRetryAttempts++;
+ lastLocalException = new AzureException(
+ "Encountered Exception while uploading block: " + ioe, ioe);
try {
- Thread.sleep(LEASE_RENEWAL_RETRY_SLEEP_PERIOD);
- } catch(InterruptedException ex) {
- LOG.debug("Blob append metadata updated method interrupted");
+ Thread.sleep(
+ BLOCK_UPLOAD_RETRY_INTERVAL * (uploadRetryAttempts + 1));
+ } catch(InterruptedException ie) {
Thread.currentThread().interrupt();
+ break;
}
}
}
- // The code should not enter here because the while loop will
- // always be executed and if the while loop is executed we
- // would returning from the while loop.
- return false;
+ if (uploadRetryAttempts == MAX_BLOCK_UPLOAD_RETRIES) {
+ maybeSetFirstError(lastLocalException);
+ }
}
/**
- * This is the only method that should be writing to outBuffer to maintain consistency of the outBuffer.
- * @param data
- * @param offset
- * @param length
- * @throws IOException
+ * A ThreadFactory that creates uploader thread with
+ * meaningful names helpful for debugging purposes.
*/
- private synchronized void writeInternal(final byte[] data, final int offset, final int length)
- throws IOException {
+ class UploaderThreadFactory implements ThreadFactory {
- if (!initialized) {
- throw new IOException("Trying to write to an un-initialized Append stream");
+ @Override
+ public Thread newThread(Runnable r) {
+ Thread t = new Thread(r);
+ t.setName(String.format("%s-%d", THREAD_ID_PREFIX,
+ threadSequenceNumber.getAndIncrement()));
+ return t;
}
+ }
- if (closed) {
- throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
- }
+ /**
+ * Upload block commands.
+ */
+ private class UploadBlockCommand extends UploadCommand {
- if (leaseFreed) {
- throw new IOException(String.format("Write called on a append stream not holding lease. Failing Write"));
- }
+ // the block content for upload
+ private final ByteBuffer payload;
- byte[] currentData = new byte[length];
- System.arraycopy(data, offset, currentData, 0, length);
+ // description of the block
+ private final BlockEntry entry;
- // check to see if the data to be appended exceeds the
- // buffer size. If so we upload a block to azure storage.
- while ((outBuffer.size() + currentData.length) > bufferSize) {
+ UploadBlockCommand(String blockId, ByteBuffer payload) {
- byte[] payload = new byte[bufferSize];
+ super(blobLength);
- // Add data from the existing buffer
- System.arraycopy(outBuffer.toByteArray(), 0, payload, 0, outBuffer.size());
+ BlockEntry blockEntry = new BlockEntry(blockId);
+ blockEntry.setSize(payload.position());
+ blockEntry.setSearchMode(BlockSearchMode.LATEST);
- // Updating the available size in the payload
- int availableSpaceInPayload = bufferSize - outBuffer.size();
+ this.payload = payload;
+ this.entry = blockEntry;
- // Adding data from the current call
- System.arraycopy(currentData, 0, payload, outBuffer.size(), availableSpaceInPayload);
+ uncommittedBlockEntries.add(blockEntry);
+ }
- uploadBlockToStorage(payload);
+ /**
+ * Execute command.
+ */
+ void execute() throws InterruptedException {
+
+ uploadingSemaphore.acquire(1);
+ writeBlockRequestInternal(entry.getId(), payload, true);
+ uploadingSemaphore.release(1);
- // updating the currentData buffer
- byte[] tempBuffer = new byte[currentData.length - availableSpaceInPayload];
- System.arraycopy(currentData, availableSpaceInPayload,
- tempBuffer, 0, currentData.length - availableSpaceInPayload);
- currentData = tempBuffer;
- outBuffer = new ByteArrayOutputStream(bufferSize);
}
- outBuffer.write(currentData);
+ void dump() {
+ LOG.debug("upload block {} size: {} for blob {}",
+ entry.getId(),
+ entry.getSize(),
+ key);
+ }
}
/**
- * Runnable instance that uploads the block of data to azure storage.
- *
- *
+ * Upload blob block list commands.
*/
- private class WriteRequest implements Runnable {
- private final byte[] dataPayload;
- private final String blockId;
+ private class UploadBlockListCommand extends UploadCommand {
+
+ private BlockEntry lastBlock = null;
+
+ UploadBlockListCommand() {
+ super(blobLength);
- public WriteRequest(byte[] dataPayload, String blockId) {
- this.dataPayload = dataPayload;
- this.blockId = blockId;
+ if (!uncommittedBlockEntries.isEmpty()) {
+ lastBlock = uncommittedBlockEntries.getLast();
+ }
}
- @Override
- public void run() {
+ void awaitAsDependent() throws InterruptedException {
+ // empty. later commit block does not need to wait previous commit block
+ // lists.
+ }
- int uploadRetryAttempts = 0;
- IOException lastLocalException = null;
- while (uploadRetryAttempts < MAX_BLOCK_UPLOAD_RETRIES) {
- try {
+ void dump() {
+ LOG.debug("commit block list with {} blocks for blob {}",
+ uncommittedBlockEntries.size(), key);
+ }
+
+ /**
+ * Execute command.
+ */
+ public void execute() throws InterruptedException, IOException {
+
+ if (committedBlobLength.get() >= getCommandBlobOffset()) {
+ LOG.debug("commit already applied for {}", key);
+ return;
+ }
+
+ if (lastBlock == null) {
+ LOG.debug("nothing to commit for {}", key);
+ return;
+ }
+
+ LOG.debug("active commands: {} for {}", activeBlockCommands.size(), key);
+
+ for (UploadCommand activeCommand : activeBlockCommands) {
+ if (activeCommand.getCommandBlobOffset() < getCommandBlobOffset()) {
+ activeCommand.dump();
+ activeCommand.awaitAsDependent();
+ } else {
+ break;
+ }
+ }
+
+ // stop all uploads until the block list is committed
+ uploadingSemaphore.acquire(MAX_NUMBER_THREADS_IN_THREAD_POOL);
+
+ BlockEntry uncommittedBlock;
+ do {
+ uncommittedBlock = uncommittedBlockEntries.poll();
+ blockEntries.add(uncommittedBlock);
+ } while (uncommittedBlock != lastBlock);
+
+ if (blockEntries.size() > activateCompactionBlockCount) {
+ LOG.debug("Block compaction: activated with {} blocks for {}",
+ blockEntries.size(), key);
+
+ // Block compaction
+ long startCompaction = System.nanoTime();
+ blockCompaction();
+ LOG.debug("Block compaction finished for {} ms with {} blocks for {}",
+ TimeUnit.NANOSECONDS.toMillis(
+ System.nanoTime() - startCompaction),
+ blockEntries.size(), key);
+ }
+
+ writeBlockListRequestInternal();
+
+ uploadingSemaphore.release(MAX_NUMBER_THREADS_IN_THREAD_POOL);
- blob.uploadBlock(blockId, new ByteArrayInputStream(dataPayload),
- dataPayload.length, new BlobRequestOptions(), opContext);
+ // remove blocks previous commands
+ for (Iterator<UploadCommand> it = activeBlockCommands.iterator();
+ it.hasNext();) {
+ UploadCommand activeCommand = it.next();
+ if (activeCommand.getCommandBlobOffset() <= getCommandBlobOffset()) {
+ it.remove();
+ } else {
break;
- } catch(Exception ioe) {
- Log.getLog().debug("Encountered exception during uploading block for Blob : {} Exception : {}", key, ioe);
- uploadRetryAttempts++;
- lastLocalException = new IOException("Encountered Exception while uploading block", ioe);
- try {
- Thread.sleep(BLOCK_UPLOAD_RETRY_INTERVAL);
- } catch(InterruptedException ie) {
- Thread.currentThread().interrupt();
- break;
+ }
+ }
+
+ committedBlobLength.set(getCommandBlobOffset());
+ }
+
+ /**
+ * Internal output stream with read access to the internal buffer.
+ */
+ private class ByteArrayOutputStreamInternal extends ByteArrayOutputStream {
+
+ ByteArrayOutputStreamInternal(int size) {
+ super(size);
+ }
+
+ byte[] getByteArray() {
+ return buf;
+ }
+ }
+
+ /**
+ * Block compaction process.
+ *
+ * Block compaction is only enabled when the number of blocks exceeds
+ * activateCompactionBlockCount. The algorithm searches for the longest
+ * segment [b..e) where (e-b) > 2 && |b| + |b+1| ... |e-1| < maxBlockSize
+ * such that size(b1) + size(b2) + ... + size(bn) < maximum-block-size.
+ * It then downloads the blocks in the sequence, concatenates the data to
+ * form a single block, uploads this new block, and updates the block
+ * list to replace the sequence of blocks with the new block.
+ */
+ private void blockCompaction() throws IOException {
+ //current segment [segmentBegin, segmentEnd) and file offset/size of the
+ // current segment
+ int segmentBegin = 0, segmentEnd = 0;
+ long segmentOffsetBegin = 0, segmentOffsetEnd = 0;
+
+ //longest segment [maxSegmentBegin, maxSegmentEnd) and file offset/size of
+ // the longest segment
+ int maxSegmentBegin = 0, maxSegmentEnd = 0;
+ long maxSegmentOffsetBegin = 0, maxSegmentOffsetEnd = 0;
+
+ for (BlockEntry block : blockEntries) {
+ segmentEnd++;
+ segmentOffsetEnd += block.getSize();
+ if (segmentOffsetEnd - segmentOffsetBegin > maxBlockSize.get()) {
+ if (segmentEnd - segmentBegin > 2) {
+ if (maxSegmentEnd - maxSegmentBegin < segmentEnd - segmentBegin) {
+ maxSegmentBegin = segmentBegin;
+ maxSegmentEnd = segmentEnd;
+ maxSegmentOffsetBegin = segmentOffsetBegin;
+ maxSegmentOffsetEnd = segmentOffsetEnd - block.getSize();
+ }
}
+ segmentBegin = segmentEnd - 1;
+ segmentOffsetBegin = segmentOffsetEnd - block.getSize();
}
}
- if (uploadRetryAttempts == MAX_BLOCK_UPLOAD_RETRIES) {
- lastError = lastLocalException;
+ if (maxSegmentEnd - maxSegmentBegin > 1) {
+
+ LOG.debug("Block compaction: {} blocks for {}",
+ maxSegmentEnd - maxSegmentBegin, key);
+
+ // download synchronously all the blocks from the azure storage
+ ByteArrayOutputStreamInternal blockOutputStream
+ = new ByteArrayOutputStreamInternal(maxBlockSize.get());
+
+ try {
+ long length = maxSegmentOffsetEnd - maxSegmentOffsetBegin;
+ blob.downloadRange(maxSegmentOffsetBegin, length, blockOutputStream,
+ new BlobRequestOptions(), opContext);
+ } catch(StorageException ex) {
+ LOG.error(
+ "Storage exception encountered during block compaction phase"
+ + " : {} Storage Exception : {} Error Code: {}",
+ key, ex, ex.getErrorCode());
+ throw new AzureException(
+ "Encountered Exception while committing append blocks " + ex, ex);
+ }
+
+ // upload synchronously new block to the azure storage
+ String blockId = generateBlockId();
+
+ ByteBuffer byteBuffer = ByteBuffer.wrap(
+ blockOutputStream.getByteArray());
+ byteBuffer.position(blockOutputStream.size());
+
+ writeBlockRequestInternal(blockId, byteBuffer, false);
+
+ // replace blocks from the longest segment with new block id
+ blockEntries.subList(maxSegmentBegin + 1, maxSegmentEnd - 1).clear();
+ BlockEntry newBlock = blockEntries.get(maxSegmentBegin);
+ newBlock.setId(blockId);
+ newBlock.setSearchMode(BlockSearchMode.LATEST);
+ newBlock.setSize(maxSegmentOffsetEnd - maxSegmentOffsetBegin);
}
}
}
/**
- * A ThreadFactory that creates uploader thread with
- * meaningful names helpful for debugging purposes.
+ * Prepare block upload command and queue the command in thread pool executor.
*/
- class UploaderThreadFactory implements ThreadFactory {
+ private synchronized void addBlockUploadCommand() throws IOException {
+
+ maybeThrowFirstError();
+
+ if (blobExist && lease.isFreed()) {
+ throw new AzureException(String.format(
+ "Attempting to upload a block on blob : %s "
+ + " that does not have lease on the Blob. Failing upload", key));
+ }
+
+ int blockSize = outBuffer.position();
+ if (blockSize > 0) {
+ UploadCommand command = new UploadBlockCommand(generateBlockId(),
+ outBuffer);
+ activeBlockCommands.add(command);
+
+ blobLength += blockSize;
+ outBuffer = poolReadyByteBuffers.getBuffer(false, maxBlockSize.get());
+
+ ioThreadPool.execute(new WriteRequest(command));
- @Override
- public Thread newThread(Runnable r) {
- Thread t = new Thread(r);
- t.setName(String.format("%s-%s-%d", THREAD_ID_PREFIX, key,
- threadSequenceNumber.getAndIncrement()));
- return t;
}
}
/**
- * A deamon thread that renews the Append lease on the blob.
- * The thread sleeps for LEASE_RENEWAL_PERIOD time before renewing
- * the lease. If an error is encountered while renewing the lease
- * then an lease is released by this thread, which fails all other
- * operations.
+ * Prepare block list commit command and queue the command in thread pool
+ * executor.
*/
- private class AppendRenewer implements Runnable {
+ private synchronized UploadCommand addFlushCommand() throws IOException {
- @Override
- public void run() {
+ maybeThrowFirstError();
- while (!leaseFreed) {
+ if (blobExist && lease.isFreed()) {
+ throw new AzureException(
+ String.format("Attempting to upload block list on blob : %s"
+ + " that does not have lease on the Blob. Failing upload", key));
+ }
- try {
- Thread.sleep(LEASE_RENEWAL_PERIOD);
- } catch (InterruptedException ie) {
- LOG.debug("Appender Renewer thread interrupted");
- Thread.currentThread().interrupt();
- }
+ UploadCommand command = new UploadBlockListCommand();
+ activeBlockCommands.add(command);
- Log.getLog().debug("Attempting to renew append lease on {}", key);
+ ioThreadPool.execute(new WriteRequest(command));
- try {
- if (!leaseFreed) {
- // Update the blob metadata to renew the append lease
- if (!updateBlobAppendMetadata(true, true)) {
- LOG.error("Unable to re-acquire append lease on the Blob {} ", key);
- leaseFreed = true;
- }
- }
- } catch (StorageException ex) {
+ return command;
+ }
- LOG.debug("Lease renewal for Blob : {} encountered "
- + "Storage Exception : {} Error Code : {}", key, ex, ex.getErrorCode());
+ /**
+ * Runnable instance that uploads the block of data to azure storage.
+ */
+ private class WriteRequest implements Runnable {
+ private final UploadCommand command;
- // We swallow the exception here because if the blob metadata is not updated for
- // APPEND_LEASE_TIMEOUT period, another thread would be able to detect this and
- // continue forward if it needs to append.
- leaseFreed = true;
- }
+ WriteRequest(UploadCommand command) {
+ this.command = command;
+ }
+
+ @Override
+ public void run() {
+
+ try {
+ command.dump();
+ long startTime = System.nanoTime();
+ command.execute();
+ command.setCompleted();
+ LOG.debug("command finished for {} ms",
+ TimeUnit.NANOSECONDS.toMillis(
+ System.nanoTime() - startTime));
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ } catch (Exception ex) {
+ LOG.debug(
+ "Encountered exception during execution of command for Blob :"
+ + " {} Exception : {}", key, ex);
+ firstError.compareAndSet(null, new AzureException(ex));
}
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13eda500/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 0bde124..280c0e0 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -62,6 +62,8 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.Seekable;
+import org.apache.hadoop.fs.StreamCapabilities;
+import org.apache.hadoop.fs.Syncable;
import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation;
import org.apache.hadoop.fs.azure.metrics.AzureFileSystemMetricsSystem;
import org.apache.hadoop.fs.azure.security.Constants;
@@ -352,9 +354,9 @@ public class NativeAzureFileSystem extends FileSystem {
}
/**
- * This is an exact copy of org.codehaus.jettison.json.JSONObject.quote
+ * This is an exact copy of org.codehaus.jettison.json.JSONObject.quote
* method.
- *
+ *
* Produce a string in double quotes with backslash sequences in all the
* right places. A backslash will be inserted within </, allowing JSON
* text to be delivered in HTML. In JSON text, a string cannot contain a
@@ -947,11 +949,11 @@ public class NativeAzureFileSystem extends FileSystem {
}
}
- private class NativeAzureFsOutputStream extends OutputStream {
- // We should not override flush() to actually close current block and flush
- // to DFS, this will break applications that assume flush() is a no-op.
- // Applications are advised to use Syncable.hflush() for that purpose.
- // NativeAzureFsOutputStream needs to implement Syncable if needed.
+ /**
+ * Azure output stream; wraps an inner stream of different types.
+ */
+ public class NativeAzureFsOutputStream extends OutputStream
+ implements Syncable, StreamCapabilities {
private String key;
private String keyEncoded;
private OutputStream out;
@@ -983,6 +985,48 @@ public class NativeAzureFileSystem extends FileSystem {
setEncodedKey(anEncodedKey);
}
+ /**
+ * Get a reference to the wrapped output stream.
+ *
+ * @return the underlying output stream
+ */
+ @InterfaceAudience.LimitedPrivate({"HDFS"})
+ public OutputStream getOutStream() {
+ return out;
+ }
+
+ @Override // Syncable
+ public void hflush() throws IOException {
+ if (out instanceof Syncable) {
+ ((Syncable) out).hflush();
+ } else {
+ flush();
+ }
+ }
+
+ @Override // Syncable
+ public void hsync() throws IOException {
+ if (out instanceof Syncable) {
+ ((Syncable) out).hsync();
+ } else {
+ flush();
+ }
+ }
+
+ /**
+ * Propagate probe of stream capabilities to nested stream
+ * (if supported), else return false.
+ * @param capability string to query the stream support for.
+ * @return true if the nested stream supports the specific capability.
+ */
+ @Override // StreamCapability
+ public boolean hasCapability(String capability) {
+ if (out instanceof StreamCapabilities) {
+ return ((StreamCapabilities) out).hasCapability(capability);
+ }
+ return false;
+ }
+
@Override
public synchronized void close() throws IOException {
if (out != null) {
@@ -990,8 +1034,11 @@ public class NativeAzureFileSystem extends FileSystem {
// before returning to the caller.
//
out.close();
- restoreKey();
- out = null;
+ try {
+ restoreKey();
+ } finally {
+ out = null;
+ }
}
}
@@ -1045,10 +1092,10 @@ public class NativeAzureFileSystem extends FileSystem {
/**
* Writes <code>len</code> from the specified byte array starting at offset
* <code>off</code> to the output stream. The general contract for write(b,
- * off, len) is that some of the bytes in the array <code>
- * b</code b> are written to the output stream in order; element
- * <code>b[off]</code> is the first byte written and
- * <code>b[off+len-1]</code> is the last byte written by this operation.
+ * off, len) is that some of the bytes in the array <code>b</code>
+ * are written to the output stream in order; element <code>b[off]</code>
+ * is the first byte written and <code>b[off+len-1]</code> is the last
+ * byte written by this operation.
*
* @param b
* Byte array to be written.
@@ -1749,7 +1796,7 @@ public class NativeAzureFileSystem extends FileSystem {
OutputStream bufOutStream;
if (store.isPageBlobKey(key)) {
// Store page blobs directly in-place without renames.
- bufOutStream = store.storefile(key, permissionStatus);
+ bufOutStream = store.storefile(key, permissionStatus, key);
} else {
// This is a block blob, so open the output blob stream based on the
// encoded key.
@@ -1777,7 +1824,7 @@ public class NativeAzureFileSystem extends FileSystem {
// these
// blocks.
bufOutStream = new NativeAzureFsOutputStream(store.storefile(
- keyEncoded, permissionStatus), key, keyEncoded);
+ keyEncoded, permissionStatus, key), key, keyEncoded);
}
// Construct the data output stream from the buffered output stream.
FSDataOutputStream fsOut = new FSDataOutputStream(bufOutStream, statistics);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13eda500/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeFileSystemStore.java
index 1c7309f..57a729d 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeFileSystemStore.java
@@ -50,8 +50,9 @@ interface NativeFileSystemStore {
InputStream retrieve(String key, long byteRangeStart) throws IOException;
- DataOutputStream storefile(String key, PermissionStatus permissionStatus)
- throws AzureException;
+ DataOutputStream storefile(String keyEncoded,
+ PermissionStatus permissionStatus,
+ String key) throws AzureException;
boolean isPageBlobKey(String key);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13eda500/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
index 5dbb6bc..7c2722e 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
@@ -519,7 +519,7 @@ public class SecureStorageInterfaceImpl extends StorageInterface {
@Override
public SelfRenewingLease acquireLease() throws StorageException {
- return new SelfRenewingLease(this);
+ return new SelfRenewingLease(this, false);
}
}
@@ -557,10 +557,12 @@ public class SecureStorageInterfaceImpl extends StorageInterface {
}
@Override
- public void uploadBlock(String blockId, InputStream sourceStream,
+ public void uploadBlock(String blockId, AccessCondition accessCondition,
+ InputStream sourceStream,
long length, BlobRequestOptions options,
OperationContext opContext) throws IOException, StorageException {
- ((CloudBlockBlob) getBlob()).uploadBlock(blockId, sourceStream, length, null, options, opContext);
+ ((CloudBlockBlob) getBlob()).uploadBlock(blockId, sourceStream, length,
+ accessCondition, options, opContext);
}
@Override
@@ -593,4 +595,4 @@ public class SecureStorageInterfaceImpl extends StorageInterface {
null, options, opContext);
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13eda500/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java
index 00d5e99..10956f7 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java
@@ -30,6 +30,8 @@ import com.microsoft.azure.storage.blob.CloudBlob;
import java.util.concurrent.atomic.AtomicInteger;
+import static com.microsoft.azure.storage.StorageErrorCodeStrings.LEASE_ALREADY_PRESENT;
+
/**
* An Azure blob lease that automatically renews itself indefinitely
* using a background thread. Use it to synchronize distributed processes,
@@ -66,7 +68,7 @@ public class SelfRenewingLease {
@VisibleForTesting
static final int LEASE_ACQUIRE_RETRY_INTERVAL = 2000;
- public SelfRenewingLease(CloudBlobWrapper blobWrapper)
+ public SelfRenewingLease(CloudBlobWrapper blobWrapper, boolean throwIfPresent)
throws StorageException {
this.leaseFreed = false;
@@ -79,10 +81,14 @@ public class SelfRenewingLease {
leaseID = blob.acquireLease(LEASE_TIMEOUT, null);
} catch (StorageException e) {
+ if (throwIfPresent && e.getErrorCode().equals(LEASE_ALREADY_PRESENT)) {
+ throw e;
+ }
+
// Throw again if we don't want to keep waiting.
// We expect it to be that the lease is already present,
// or in some cases that the blob does not exist.
- if (!"LeaseAlreadyPresent".equals(e.getErrorCode())) {
+ if (!LEASE_ALREADY_PRESENT.equals(e.getErrorCode())) {
LOG.info(
"Caught exception when trying to get lease on blob "
+ blobWrapper.getUri().toString() + ". " + e.getMessage());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13eda500/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
index 8b6b082..e03d731 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
@@ -665,6 +665,7 @@ abstract class StorageInterface {
*
* @param blockId A String that represents the Base-64 encoded block ID. Note for a given blob
* the length of all Block IDs must be identical.
+ * @param accessCondition An {@link AccessCondition} object that represents the access conditions for the blob.
* @param sourceStream An {@link InputStream} object that represents the input stream to write to the
* block blob.
* @param length A long which represents the length, in bytes, of the stream data,
@@ -678,7 +679,7 @@ abstract class StorageInterface {
* @throws IOException If an I/O error occurred.
* @throws StorageException If a storage service error occurred.
*/
- void uploadBlock(String blockId, InputStream sourceStream,
+ void uploadBlock(String blockId, AccessCondition accessCondition, InputStream sourceStream,
long length, BlobRequestOptions options,
OperationContext opContext) throws IOException, StorageException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13eda500/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
index d3d0370..41a4dbb 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
@@ -277,7 +277,7 @@ class StorageInterfaceImpl extends StorageInterface {
return new CloudBlockBlobWrapperImpl(container.getBlockBlobReference(relativePath));
}
-
+
@Override
public CloudBlobWrapper getPageBlobReference(String relativePath)
throws URISyntaxException, StorageException {
@@ -286,7 +286,7 @@ class StorageInterfaceImpl extends StorageInterface {
}
}
-
+
abstract static class CloudBlobWrapperImpl implements CloudBlobWrapper {
private final CloudBlob blob;
@@ -441,10 +441,10 @@ class StorageInterfaceImpl extends StorageInterface {
@Override
public SelfRenewingLease acquireLease() throws StorageException {
- return new SelfRenewingLease(this);
+ return new SelfRenewingLease(this, false);
}
}
-
+
//
// CloudBlockBlobWrapperImpl
@@ -479,10 +479,10 @@ class StorageInterfaceImpl extends StorageInterface {
}
@Override
- public void uploadBlock(String blockId, InputStream sourceStream,
+ public void uploadBlock(String blockId, AccessCondition accessCondition, InputStream sourceStream,
long length, BlobRequestOptions options,
OperationContext opContext) throws IOException, StorageException {
- ((CloudBlockBlob) getBlob()).uploadBlock(blockId, sourceStream, length, null, options, opContext);
+ ((CloudBlockBlob) getBlob()).uploadBlock(blockId, sourceStream, length, accessCondition, options, opContext);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13eda500/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SyncableDataOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SyncableDataOutputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SyncableDataOutputStream.java
index a52fdb7..fc8796b 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SyncableDataOutputStream.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SyncableDataOutputStream.java
@@ -24,6 +24,7 @@ import java.io.OutputStream;
import org.apache.hadoop.fs.StreamCapabilities;
import org.apache.hadoop.fs.Syncable;
+import org.apache.hadoop.classification.InterfaceAudience;
/**
* Support the Syncable interface on top of a DataOutputStream.
@@ -38,6 +39,16 @@ public class SyncableDataOutputStream extends DataOutputStream
super(out);
}
+ /**
+ * Get a reference to the wrapped output stream.
+ *
+ * @return the underlying output stream
+ */
+ @InterfaceAudience.LimitedPrivate({"HDFS"})
+ public OutputStream getOutStream() {
+ return out;
+ }
+
@Override
public boolean hasCapability(String capability) {
if (out instanceof StreamCapabilities) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13eda500/hadoop-tools/hadoop-azure/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/index.md b/hadoop-tools/hadoop-azure/src/site/markdown/index.md
index 758650d..466bf0b 100644
--- a/hadoop-tools/hadoop-azure/src/site/markdown/index.md
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/index.md
@@ -153,6 +153,40 @@ line argument:
```
+### Block Blob with Compaction Support and Configuration
+
+Block blobs are the default kind of blob and are good for most big-data use
+cases. However, block blobs have strict limit of 50,000 blocks per blob.
+To prevent reaching the limit WASB, by default, does not upload new block to
+the service after every `hflush()` or `hsync()`.
+
+For most of the cases, combining data from multiple `write()` calls in
+blocks of 4Mb is a good optimization. But, in others cases, like HBase log files,
+every call to `hflush()` or `hsync()` must upload the data to the service.
+
+Block blobs with compaction upload the data to the cloud service after every
+`hflush()`/`hsync()`. To mitigate the limit of 50000 blocks, `hflush()
+`/`hsync()` runs once compaction process, if number of blocks in the blob
+is above 32,000.
+
+Block compaction search and replaces a sequence of small blocks with one big
+block. That means there is associated cost with block compaction: reading
+small blocks back to the client and writing it again as one big block.
+
+In order to have the files you create be block blobs with block compaction
+enabled, the client must set the configuration variable
+`fs.azure.block.blob.with.compaction.dir` to a comma-separated list of
+folder names.
+
+For example:
+
+```xml
+<property>
+ <name>fs.azure.block.blob.with.compaction.dir</name>
+ <value>/hbase/WALs,/data/myblobfiles</value>
+</property>
+```
+
### Page Blob Support and Configuration
The Azure Blob Storage interface for Hadoop supports two kinds of blobs,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13eda500/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
index 4f26d9f..e0ae7b4 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
@@ -551,7 +551,8 @@ public class MockStorageInterface extends StorageInterface {
throw new UnsupportedOperationException("downloadBlockList not used in Mock Tests");
}
@Override
- public void uploadBlock(String blockId, InputStream sourceStream,
+ public void uploadBlock(String blockId, AccessCondition accessCondition,
+ InputStream sourceStream,
long length, BlobRequestOptions options,
OperationContext opContext) throws IOException, StorageException {
throw new UnsupportedOperationException("uploadBlock not used in Mock Tests");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13eda500/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureConcurrentOutOfBandIo.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureConcurrentOutOfBandIo.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureConcurrentOutOfBandIo.java
index 7ea7534..a10a366 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureConcurrentOutOfBandIo.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureConcurrentOutOfBandIo.java
@@ -107,7 +107,8 @@ public class TestAzureConcurrentOutOfBandIo {
//
outputStream = writerStorageAccount.getStore().storefile(
key,
- new PermissionStatus("", "", FsPermission.getDefault()));
+ new PermissionStatus("", "", FsPermission.getDefault()),
+ key);
Arrays.fill(dataBlockWrite, (byte) (i % 256));
for (int j = 0; j < NUMBER_OF_BLOCKS; j++) {
@@ -141,7 +142,8 @@ public class TestAzureConcurrentOutOfBandIo {
// reading. This eliminates the race between the reader and writer threads.
OutputStream outputStream = testAccount.getStore().storefile(
"WASB_String.txt",
- new PermissionStatus("", "", FsPermission.getDefault()));
+ new PermissionStatus("", "", FsPermission.getDefault()),
+ "WASB_String.txt");
Arrays.fill(dataBlockWrite, (byte) 255);
for (int i = 0; i < NUMBER_OF_BLOCKS; i++) {
outputStream.write(dataBlockWrite);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/40] hadoop git commit: HDFS-10630. Federation State Store FS
Implementation. Contributed by Jason Kace and Inigo Goiri.
Posted by in...@apache.org.
HDFS-10630. Federation State Store FS Implementation. Contributed by Jason Kace and Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3384811c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3384811c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3384811c
Branch: refs/heads/HDFS-10467
Commit: 3384811cc478bdcda798063a58bf91bb94ffc98c
Parents: 951200d
Author: Inigo Goiri <in...@apache.org>
Authored: Tue May 2 15:49:53 2017 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu Sep 7 13:19:50 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 14 +
.../federation/router/PeriodicService.java | 198 ++++++++
.../StateStoreConnectionMonitorService.java | 67 +++
.../federation/store/StateStoreService.java | 152 +++++-
.../federation/store/StateStoreUtils.java | 51 +-
.../store/driver/StateStoreDriver.java | 31 +-
.../driver/StateStoreRecordOperations.java | 17 +-
.../store/driver/impl/StateStoreBaseImpl.java | 31 +-
.../driver/impl/StateStoreFileBaseImpl.java | 429 ++++++++++++++++
.../store/driver/impl/StateStoreFileImpl.java | 161 +++++++
.../driver/impl/StateStoreFileSystemImpl.java | 178 +++++++
.../driver/impl/StateStoreSerializableImpl.java | 77 +++
.../federation/store/records/BaseRecord.java | 20 +-
.../server/federation/store/records/Query.java | 66 +++
.../src/main/resources/hdfs-default.xml | 16 +
.../store/FederationStateStoreTestUtils.java | 232 +++++++++
.../store/driver/TestStateStoreDriverBase.java | 483 +++++++++++++++++++
.../store/driver/TestStateStoreFile.java | 64 +++
.../store/driver/TestStateStoreFileSystem.java | 88 ++++
19 files changed, 2329 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3384811c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 6a984cb..bb3087c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.hdfs;
+import java.util.concurrent.TimeUnit;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
@@ -25,6 +27,8 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRackFaultTolerant;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.RamDiskReplicaLruTracker;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl;
import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreSerializerPBImpl;
import org.apache.hadoop.http.HttpConfig;
@@ -1137,6 +1141,16 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
FEDERATION_STORE_SERIALIZER_CLASS_DEFAULT =
StateStoreSerializerPBImpl.class;
+ public static final String FEDERATION_STORE_DRIVER_CLASS =
+ FEDERATION_STORE_PREFIX + "driver.class";
+ public static final Class<? extends StateStoreDriver>
+ FEDERATION_STORE_DRIVER_CLASS_DEFAULT = StateStoreFileImpl.class;
+
+ public static final String FEDERATION_STORE_CONNECTION_TEST_MS =
+ FEDERATION_STORE_PREFIX + "connection.test";
+ public static final long FEDERATION_STORE_CONNECTION_TEST_MS_DEFAULT =
+ TimeUnit.MINUTES.toMillis(1);
+
// dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry
@Deprecated
public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3384811c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java
new file mode 100644
index 0000000..5e12222
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java
@@ -0,0 +1,198 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.service.ServiceStateException;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+/**
+ * Service to periodically execute a runnable.
+ */
+public abstract class PeriodicService extends AbstractService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(PeriodicService.class);
+
+ /** Default interval in milliseconds for the periodic service. */
+ private static final long DEFAULT_INTERVAL_MS = TimeUnit.MINUTES.toMillis(1);
+
+
+ /** Interval for running the periodic service in milliseconds. */
+ private long intervalMs;
+ /** Name of the service. */
+ private final String serviceName;
+
+ /** Scheduler for the periodic service. */
+ private final ScheduledExecutorService scheduler;
+
+ /** If the service is running. */
+ private volatile boolean isRunning = false;
+
+ /** How many times we run. */
+ private long runCount;
+ /** How many errors we got. */
+ private long errorCount;
+ /** When was the last time we executed this service successfully. */
+ private long lastRun;
+
+ /**
+ * Create a new periodic update service.
+ *
+ * @param name Name of the service.
+ */
+ public PeriodicService(String name) {
+ this(name, DEFAULT_INTERVAL_MS);
+ }
+
+ /**
+ * Create a new periodic update service.
+ *
+ * @param name Name of the service.
+ * @param interval Interval for the periodic service in milliseconds.
+ */
+ public PeriodicService(String name, long interval) {
+ super(name);
+ this.serviceName = name;
+ this.intervalMs = interval;
+
+ ThreadFactory threadFactory = new ThreadFactoryBuilder()
+ .setNameFormat(this.getName() + "-%d")
+ .build();
+ this.scheduler = Executors.newScheduledThreadPool(1, threadFactory);
+ }
+
+ /**
+ * Set the interval for the periodic service.
+ *
+ * @param interval Interval in milliseconds.
+ */
+ protected void setIntervalMs(long interval) {
+ if (getServiceState() == STATE.STARTED) {
+ throw new ServiceStateException("Periodic service already started");
+ } else {
+ this.intervalMs = interval;
+ }
+ }
+
+ /**
+ * Get the interval for the periodic service.
+ *
+ * @return Interval in milliseconds.
+ */
+ protected long getIntervalMs() {
+ return this.intervalMs;
+ }
+
+ /**
+ * Get how many times we failed to run the periodic service.
+ *
+ * @return Times we failed to run the periodic service.
+ */
+ protected long getErrorCount() {
+ return this.errorCount;
+ }
+
+ /**
+ * Get how many times we run the periodic service.
+ *
+ * @return Times we run the periodic service.
+ */
+ protected long getRunCount() {
+ return this.runCount;
+ }
+
+ /**
+ * Get the last time the periodic service was executed.
+ *
+ * @return Last time the periodic service was executed.
+ */
+ protected long getLastUpdate() {
+ return this.lastRun;
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ super.serviceStart();
+ LOG.info("Starting periodic service {}", this.serviceName);
+ startPeriodic();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ stopPeriodic();
+ LOG.info("Stopping periodic service {}", this.serviceName);
+ super.serviceStop();
+ }
+
+ /**
+ * Stop the periodic task.
+ */
+ protected synchronized void stopPeriodic() {
+ if (this.isRunning) {
+ LOG.info("{} is shutting down", this.serviceName);
+ this.isRunning = false;
+ this.scheduler.shutdownNow();
+ }
+ }
+
+ /**
+ * Start the periodic execution.
+ */
+ protected synchronized void startPeriodic() {
+ stopPeriodic();
+
+ // Create the runnable service
+ Runnable updateRunnable = new Runnable() {
+ @Override
+ public void run() {
+ LOG.debug("Running {} update task", serviceName);
+ try {
+ if (!isRunning) {
+ return;
+ }
+ periodicInvoke();
+ runCount++;
+ lastRun = Time.now();
+ } catch (Exception ex) {
+ errorCount++;
+ LOG.warn(serviceName + " service threw an exception", ex);
+ }
+ }
+ };
+
+ // Start the execution of the periodic service
+ this.isRunning = true;
+ this.scheduler.scheduleWithFixedDelay(
+ updateRunnable, 0, this.intervalMs, TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Method that the service will run periodically.
+ */
+ protected abstract void periodicInvoke();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3384811c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreConnectionMonitorService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreConnectionMonitorService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreConnectionMonitorService.java
new file mode 100644
index 0000000..4d279c5
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreConnectionMonitorService.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.router.PeriodicService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Service to periodically monitor the connection of the StateStore
+ * {@link StateStoreService} data store and to re-open the connection
+ * to the data store if required.
+ */
+public class StateStoreConnectionMonitorService extends PeriodicService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StateStoreConnectionMonitorService.class);
+
+ /** Service that maintains the State Store connection. */
+ private final StateStoreService stateStore;
+
+
+ /**
+ * Create a new service to monitor the connectivity of the state store driver.
+ *
+ * @param store Instance of the state store to be monitored.
+ */
+ public StateStoreConnectionMonitorService(StateStoreService store) {
+ super(StateStoreConnectionMonitorService.class.getSimpleName());
+ this.stateStore = store;
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ this.setIntervalMs(conf.getLong(
+ DFSConfigKeys.FEDERATION_STORE_CONNECTION_TEST_MS,
+ DFSConfigKeys.FEDERATION_STORE_CONNECTION_TEST_MS_DEFAULT));
+
+ super.serviceInit(conf);
+ }
+
+ @Override
+ public void periodicInvoke() {
+ LOG.debug("Checking state store connection");
+ if (!stateStore.isDriverReady()) {
+ LOG.info("Attempting to open state store driver.");
+ stateStore.loadDriver();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3384811c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
index 866daa3..df207e0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
@@ -15,45 +15,168 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.hdfs.server.federation.store;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.LinkedList;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
/**
* A service to initialize a
* {@link org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver
- * StateStoreDriver} and maintain the connection to the data store. There
- * are multiple state store driver connections supported:
+ * StateStoreDriver} and maintain the connection to the data store. There are
+ * multiple state store driver connections supported:
* <ul>
- * <li>File {@link org.apache.hadoop.hdfs.server.federation.store.driver.impl.
+ * <li>File
+ * {@link org.apache.hadoop.hdfs.server.federation.store.driver.impl.
* StateStoreFileImpl StateStoreFileImpl}
- * <li>ZooKeeper {@link org.apache.hadoop.hdfs.server.federation.store.driver.
- * impl.StateStoreZooKeeperImpl StateStoreZooKeeperImpl}
+ * <li>ZooKeeper
+ * {@link org.apache.hadoop.hdfs.server.federation.store.driver.impl.
+ * StateStoreZooKeeperImpl StateStoreZooKeeperImpl}
* </ul>
* <p>
- * The service also supports the dynamic registration of data interfaces such as
- * the following:
+ * The service also supports the dynamic registration of record stores like:
* <ul>
- * <li>{@link MembershipStateStore}: state of the Namenodes in the
+ * <li>{@link MembershipStore}: state of the Namenodes in the
* federation.
* <li>{@link MountTableStore}: Mount table between to subclusters.
* See {@link org.apache.hadoop.fs.viewfs.ViewFs ViewFs}.
- * <li>{@link RouterStateStore}: State of the routers in the federation.
* </ul>
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class StateStoreService extends CompositeService {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StateStoreService.class);
+
+
+ /** State Store configuration. */
+ private Configuration conf;
+
/** Identifier for the service. */
private String identifier;
- // Stub class
- public StateStoreService(String name) {
- super(name);
+ /** Driver for the back end connection. */
+ private StateStoreDriver driver;
+
+ /** Service to maintain data store connection. */
+ private StateStoreConnectionMonitorService monitorService;
+
+
+ public StateStoreService() {
+ super(StateStoreService.class.getName());
+ }
+
+ /**
+ * Initialize the State Store and the connection to the backend.
+ *
+ * @param config Configuration for the State Store.
+ * @throws IOException
+ */
+ @Override
+ protected void serviceInit(Configuration config) throws Exception {
+ this.conf = config;
+
+ // Create implementation of State Store
+ Class<? extends StateStoreDriver> driverClass = this.conf.getClass(
+ DFSConfigKeys.FEDERATION_STORE_DRIVER_CLASS,
+ DFSConfigKeys.FEDERATION_STORE_DRIVER_CLASS_DEFAULT,
+ StateStoreDriver.class);
+ this.driver = ReflectionUtils.newInstance(driverClass, this.conf);
+
+ if (this.driver == null) {
+ throw new IOException("Cannot create driver for the State Store");
+ }
+
+ // Check the connection to the State Store periodically
+ this.monitorService = new StateStoreConnectionMonitorService(this);
+ this.addService(monitorService);
+
+ super.serviceInit(this.conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ loadDriver();
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ closeDriver();
+
+ super.serviceStop();
+ }
+
+ /**
+ * List of records supported by this State Store.
+ *
+ * @return List of supported record classes.
+ */
+ public Collection<Class<? extends BaseRecord>> getSupportedRecords() {
+ // TODO add list of records
+ return new LinkedList<>();
+ }
+
+ /**
+ * Load the State Store driver. If successful, refresh cached data tables.
+ */
+ public void loadDriver() {
+ synchronized (this.driver) {
+ if (!isDriverReady()) {
+ String driverName = this.driver.getClass().getSimpleName();
+ if (this.driver.init(conf, getIdentifier(), getSupportedRecords())) {
+ LOG.info("Connection to the State Store driver {} is open and ready",
+ driverName);
+ } else {
+ LOG.error("Cannot initialize State Store driver {}", driverName);
+ }
+ }
+ }
+ }
+
+ /**
+ * Check if the driver is ready to be used.
+ *
+ * @return If the driver is ready.
+ */
+ public boolean isDriverReady() {
+ return this.driver.isDriverReady();
+ }
+
+ /**
+ * Manually shuts down the driver.
+ *
+ * @throws Exception If the driver cannot be closed.
+ */
+ @VisibleForTesting
+ public void closeDriver() throws Exception {
+ if (this.driver != null) {
+ this.driver.close();
+ }
+ }
+
+ /**
+ * Get the state store driver.
+ *
+ * @return State store driver.
+ */
+ public StateStoreDriver getDriver() {
+ return this.driver;
}
/**
@@ -74,4 +197,5 @@ public class StateStoreService extends CompositeService {
public void setIdentifier(String id) {
this.identifier = id;
}
-}
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3384811c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUtils.java
index 8c681df..0a36619 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUtils.java
@@ -17,17 +17,22 @@
*/
package org.apache.hadoop.hdfs.server.federation.store;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.Query;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
- * Set of utility functions used to query, create, update and delete data
- * records in the state store.
+ * Set of utility functions used to work with the State Store.
*/
public final class StateStoreUtils {
- private static final Log LOG = LogFactory.getLog(StateStoreUtils.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StateStoreUtils.class);
+
private StateStoreUtils() {
// Utility class
@@ -52,7 +57,7 @@ public final class StateStoreUtils {
// Check if we went too far
if (actualClazz.equals(BaseRecord.class)) {
- LOG.error("We went too far (" + actualClazz + ") with " + clazz);
+ LOG.error("We went too far ({}) with {}", actualClazz, clazz);
actualClazz = clazz;
}
return actualClazz;
@@ -69,4 +74,36 @@ public final class StateStoreUtils {
Class<? extends BaseRecord> getRecordClass(final T record) {
return getRecordClass(record.getClass());
}
-}
+
+ /**
+ * Get the base class name for a record. If we get an implementation of a
+ * record we will return the real parent record class.
+ *
+ * @param clazz Class of the data record to check.
+ * @return Name of the base class for the record.
+ */
+ public static <T extends BaseRecord> String getRecordName(
+ final Class<T> clazz) {
+ return getRecordClass(clazz).getSimpleName();
+ }
+
+ /**
+ * Filters a list of records to find all records matching the query.
+ *
+ * @param query Map of field names and objects to use to filter results.
+ * @param records List of data records to filter.
+ * @return List of all records matching the query (or empty list if none
+ * match), null if the data set could not be filtered.
+ */
+ public static <T extends BaseRecord> List<T> filterMultiple(
+ final Query<T> query, final Iterable<T> records) {
+
+ List<T> matchingList = new ArrayList<>();
+ for (T record : records) {
+ if (query.matches(record)) {
+ matchingList.add(record);
+ }
+ }
+ return matchingList;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3384811c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java
index a1527df..90111bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java
@@ -18,15 +18,16 @@
package org.apache.hadoop.hdfs.server.federation.store.driver;
import java.net.InetAddress;
-import java.util.List;
+import java.util.Collection;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
import org.apache.hadoop.hdfs.server.federation.store.StateStoreUnavailableException;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils;
import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Driver class for an implementation of a {@link StateStoreService}
@@ -35,7 +36,8 @@ import org.apache.hadoop.util.Time;
*/
public abstract class StateStoreDriver implements StateStoreRecordOperations {
- private static final Log LOG = LogFactory.getLog(StateStoreDriver.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StateStoreDriver.class);
/** State Store configuration. */
@@ -47,13 +49,14 @@ public abstract class StateStoreDriver implements StateStoreRecordOperations {
/**
* Initialize the state store connection.
+ *
* @param config Configuration for the driver.
* @param id Identifier for the driver.
* @param records Records that are supported.
* @return If initialized and ready, false if failed to initialize driver.
*/
public boolean init(final Configuration config, final String id,
- final List<Class<? extends BaseRecord>> records) {
+ final Collection<Class<? extends BaseRecord>> records) {
this.conf = config;
this.identifier = id;
@@ -62,8 +65,20 @@ public abstract class StateStoreDriver implements StateStoreRecordOperations {
LOG.warn("The identifier for the State Store connection is not set");
}
- // TODO stub
- return false;
+ boolean success = initDriver();
+ if (!success) {
+ LOG.error("Cannot intialize driver for {}", getDriverName());
+ return false;
+ }
+
+ for (Class<? extends BaseRecord> cls : records) {
+ String recordString = StateStoreUtils.getRecordName(cls);
+ if (!initRecordStorage(recordString, cls)) {
+ LOG.error("Cannot initialize record store for {}", cls.getSimpleName());
+ return false;
+ }
+ }
+ return true;
}
/**
@@ -169,4 +184,4 @@ public abstract class StateStoreDriver implements StateStoreRecordOperations {
}
return hostname;
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3384811c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
index 739eeba..e76a733 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
@@ -19,11 +19,11 @@ package org.apache.hadoop.hdfs.server.federation.store.driver;
import java.io.IOException;
import java.util.List;
-import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.Query;
import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
import org.apache.hadoop.io.retry.AtMostOnce;
import org.apache.hadoop.io.retry.Idempotent;
@@ -67,14 +67,14 @@ public interface StateStoreRecordOperations {
* Get a single record from the store that matches the query.
*
* @param clazz Class of record to fetch.
- * @param query Map of field names and objects to filter results.
+ * @param query Query to filter results.
* @return A single record matching the query. Null if there are no matching
* records or more than one matching record in the store.
* @throws IOException If multiple records match or if the data store cannot
* be queried.
*/
@Idempotent
- <T extends BaseRecord> T get(Class<T> clazz, Map<String, String> query)
+ <T extends BaseRecord> T get(Class<T> clazz, Query<T> query)
throws IOException;
/**
@@ -83,14 +83,14 @@ public interface StateStoreRecordOperations {
* supports filtering it should overwrite this method.
*
* @param clazz Class of record to fetch.
- * @param query Map of field names and objects to filter results.
+ * @param query Query to filter results.
* @return Records of type clazz that match the query or empty list if none
* are found.
* @throws IOException Throws exception if unable to query the data store.
*/
@Idempotent
<T extends BaseRecord> List<T> getMultiple(
- Class<T> clazz, Map<String, String> query) throws IOException;
+ Class<T> clazz, Query<T> query) throws IOException;
/**
* Creates a single record. Optionally updates an existing record with same
@@ -152,13 +152,12 @@ public interface StateStoreRecordOperations {
* Remove multiple records of a specific class that match a query. Requires
* the getAll implementation to fetch fresh records on each call.
*
- * @param clazz Class of record to remove.
- * @param filter matching filter to remove.
+ * @param query Query to filter what to remove.
* @return The number of records removed.
* @throws IOException Throws exception if unable to query the data store.
*/
@AtMostOnce
- <T extends BaseRecord> int remove(Class<T> clazz, Map<String, String> filter)
+ <T extends BaseRecord> int remove(Class<T> clazz, Query<T> query)
throws IOException;
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3384811c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreBaseImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreBaseImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreBaseImpl.java
index b711fa9..1bd35f2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreBaseImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreBaseImpl.java
@@ -17,14 +17,17 @@
*/
package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
+import static org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils.filterMultiple;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
import org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils;
import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.Query;
+import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
/**
* Base implementation of a State Store driver. It contains default
@@ -41,7 +44,7 @@ public abstract class StateStoreBaseImpl extends StateStoreDriver {
@Override
public <T extends BaseRecord> T get(
- Class<T> clazz, Map<String, String> query) throws IOException {
+ Class<T> clazz, Query<T> query) throws IOException {
List<T> records = getMultiple(clazz, query);
if (records.size() > 1) {
throw new IOException("Found more than one object in collection");
@@ -53,17 +56,31 @@ public abstract class StateStoreBaseImpl extends StateStoreDriver {
}
@Override
+ public <T extends BaseRecord> List<T> getMultiple(
+ Class<T> clazz, Query<T> query) throws IOException {
+ QueryResult<T> result = get(clazz);
+ List<T> records = result.getRecords();
+ List<T> ret = filterMultiple(query, records);
+ if (ret == null) {
+ throw new IOException("Cannot fetch records from the store");
+ }
+ return ret;
+ }
+
+ @Override
public <T extends BaseRecord> boolean put(
T record, boolean allowUpdate, boolean errorIfExists) throws IOException {
- List<T> singletonList = new ArrayList<T>();
+ List<T> singletonList = new ArrayList<>();
singletonList.add(record);
return putAll(singletonList, allowUpdate, errorIfExists);
}
@Override
public <T extends BaseRecord> boolean remove(T record) throws IOException {
- Map<String, String> primaryKeys = record.getPrimaryKeys();
- Class<? extends BaseRecord> clazz = StateStoreUtils.getRecordClass(record);
- return remove(clazz, primaryKeys) == 1;
+ final Query<T> query = new Query<T>(record);
+ Class<? extends BaseRecord> clazz = record.getClass();
+ @SuppressWarnings("unchecked")
+ Class<T> recordClass = (Class<T>)StateStoreUtils.getRecordClass(clazz);
+ return remove(recordClass, query) == 1;
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3384811c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
new file mode 100644
index 0000000..d7c00ff
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
@@ -0,0 +1,429 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
+
+import static org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils.filterMultiple;
+import static org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils.getRecordClass;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreUnavailableException;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.Query;
+import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * {@link StateStoreDriver} implementation based on a local file.
+ */
+public abstract class StateStoreFileBaseImpl
+ extends StateStoreSerializableImpl {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StateStoreFileBaseImpl.class);
+
+ /** If it is initialized. */
+ private boolean initialized = false;
+
+ /** Name of the file containing the data. */
+ private static final String DATA_FILE_NAME = "records.data";
+
+
+ /**
+ * Lock reading records.
+ *
+ * @param clazz Class of the record.
+ */
+ protected abstract <T extends BaseRecord> void lockRecordRead(Class<T> clazz);
+
+ /**
+ * Unlock reading records.
+ *
+ * @param clazz Class of the record.
+ */
+ protected abstract <T extends BaseRecord> void unlockRecordRead(
+ Class<T> clazz);
+
+ /**
+ * Lock writing records.
+ *
+ * @param clazz Class of the record.
+ */
+ protected abstract <T extends BaseRecord> void lockRecordWrite(
+ Class<T> clazz);
+
+ /**
+ * Unlock writing records.
+ *
+ * @param clazz Class of the record.
+ */
+ protected abstract <T extends BaseRecord> void unlockRecordWrite(
+ Class<T> clazz);
+
+ /**
+ * Get the reader for the file system.
+ *
+ * @param clazz Class of the record.
+ */
+ protected abstract <T extends BaseRecord> BufferedReader getReader(
+ Class<T> clazz, String sub);
+
+ /**
+ * Get the writer for the file system.
+ *
+ * @param clazz Class of the record.
+ */
+ protected abstract <T extends BaseRecord> BufferedWriter getWriter(
+ Class<T> clazz, String sub);
+
+ /**
+ * Check if a path exists.
+ *
+ * @param path Path to check.
+ * @return If the path exists.
+ */
+ protected abstract boolean exists(String path);
+
+ /**
+ * Make a directory.
+ *
+ * @param path Path of the directory to create.
+ * @return If the directory was created.
+ */
+ protected abstract boolean mkdir(String path);
+
+ /**
+ * Get root directory.
+ *
+ * @return Root directory.
+ */
+ protected abstract String getRootDir();
+
+ /**
+ * Set the driver as initialized.
+ *
+ * @param ini If the driver is initialized.
+ */
+ public void setInitialized(boolean ini) {
+ this.initialized = ini;
+ }
+
+ @Override
+ public boolean initDriver() {
+ String rootDir = getRootDir();
+ try {
+ if (rootDir == null) {
+ LOG.error("Invalid root directory, unable to initialize driver.");
+ return false;
+ }
+
+ // Check root path
+ if (!exists(rootDir)) {
+ if (!mkdir(rootDir)) {
+ LOG.error("Cannot create State Store root directory {}", rootDir);
+ return false;
+ }
+ }
+ } catch (Exception ex) {
+ LOG.error(
+ "Cannot initialize filesystem using root directory {}", rootDir, ex);
+ return false;
+ }
+ setInitialized(true);
+ return true;
+ }
+
+ @Override
+ public <T extends BaseRecord> boolean initRecordStorage(
+ String className, Class<T> recordClass) {
+
+ String dataDirPath = getRootDir() + "/" + className;
+ try {
+ // Create data directories for files
+ if (!exists(dataDirPath)) {
+ LOG.info("{} data directory doesn't exist, creating it", dataDirPath);
+ if (!mkdir(dataDirPath)) {
+ LOG.error("Cannot create data directory {}", dataDirPath);
+ return false;
+ }
+ String dataFilePath = dataDirPath + "/" + DATA_FILE_NAME;
+ if (!exists(dataFilePath)) {
+ // Create empty file
+ List<T> emtpyList = new ArrayList<>();
+ if(!writeAll(emtpyList, recordClass)) {
+ LOG.error("Cannot create data file {}", dataFilePath);
+ return false;
+ }
+ }
+ }
+ } catch (Exception ex) {
+ LOG.error("Cannot create data directory {}", dataDirPath, ex);
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Read all lines from a file and deserialize into the desired record type.
+ *
+ * @param reader Open handle for the file.
+ * @param recordClass Record class to create.
+ * @param includeDates True if dateModified/dateCreated are serialized.
+ * @return List of records.
+ * @throws IOException
+ */
+ private <T extends BaseRecord> List<T> getAllFile(
+ BufferedReader reader, Class<T> clazz, boolean includeDates)
+ throws IOException {
+
+ List<T> ret = new ArrayList<T>();
+ String line;
+ while ((line = reader.readLine()) != null) {
+ if (!line.startsWith("#") && line.length() > 0) {
+ try {
+ T record = newRecord(line, clazz, includeDates);
+ ret.add(record);
+ } catch (Exception ex) {
+ LOG.error("Cannot parse line in data source file: {}", line, ex);
+ }
+ }
+ }
+ return ret;
+ }
+
+ @Override
+ public <T extends BaseRecord> QueryResult<T> get(Class<T> clazz)
+ throws IOException {
+ return get(clazz, (String)null);
+ }
+
+ @Override
+ public <T extends BaseRecord> QueryResult<T> get(Class<T> clazz, String sub)
+ throws IOException {
+ verifyDriverReady();
+ BufferedReader reader = null;
+ lockRecordRead(clazz);
+ try {
+ reader = getReader(clazz, sub);
+ List<T> data = getAllFile(reader, clazz, true);
+ return new QueryResult<T>(data, getTime());
+ } catch (Exception ex) {
+ LOG.error("Cannot fetch records {}", clazz.getSimpleName());
+ throw new IOException("Cannot read from data store " + ex.getMessage());
+ } finally {
+ if (reader != null) {
+ try {
+ reader.close();
+ } catch (IOException e) {
+ LOG.error("Failed closing file", e);
+ }
+ }
+ unlockRecordRead(clazz);
+ }
+ }
+
+ /**
+ * Overwrite the existing data with a new data set.
+ *
+ * @param list List of records to write.
+ * @param writer BufferedWriter stream to write to.
+ * @return If the records were succesfully written.
+ */
+ private <T extends BaseRecord> boolean writeAllFile(
+ Collection<T> records, BufferedWriter writer) {
+
+ try {
+ for (BaseRecord record : records) {
+ try {
+ String data = serializeString(record);
+ writer.write(data);
+ writer.newLine();
+ } catch (IllegalArgumentException ex) {
+ LOG.error("Cannot write record {} to file", record, ex);
+ }
+ }
+ writer.flush();
+ return true;
+ } catch (IOException e) {
+ LOG.error("Cannot commit records to file", e);
+ return false;
+ }
+ }
+
+ /**
+ * Overwrite the existing data with a new data set. Replaces all records in
+ * the data store for this record class. If all records in the data store are
+ * not successfully committed, this function must return false and leave the
+ * data store unchanged.
+ *
+ * @param records List of records to write. All records must be of type
+ * recordClass.
+ * @param recordClass Class of record to replace.
+ * @return true if all operations were successful, false otherwise.
+ * @throws StateStoreUnavailableException
+ */
+ public <T extends BaseRecord> boolean writeAll(
+ Collection<T> records, Class<T> recordClass)
+ throws StateStoreUnavailableException {
+ verifyDriverReady();
+ lockRecordWrite(recordClass);
+ BufferedWriter writer = null;
+ try {
+ writer = getWriter(recordClass, null);
+ return writeAllFile(records, writer);
+ } catch (Exception e) {
+ LOG.error(
+ "Cannot add records to file for {}", recordClass.getSimpleName(), e);
+ return false;
+ } finally {
+ if (writer != null) {
+ try {
+ writer.close();
+ } catch (IOException e) {
+ LOG.error(
+ "Cannot close writer for {}", recordClass.getSimpleName(), e);
+ }
+ }
+ unlockRecordWrite(recordClass);
+ }
+ }
+
+ /**
+ * Get the data file name.
+ *
+ * @return Data file name.
+ */
+ protected String getDataFileName() {
+ return DATA_FILE_NAME;
+ }
+
+ @Override
+ public boolean isDriverReady() {
+ return this.initialized;
+ }
+
+ @Override
+ public <T extends BaseRecord> boolean putAll(
+ List<T> records, boolean allowUpdate, boolean errorIfExists)
+ throws StateStoreUnavailableException {
+ verifyDriverReady();
+
+ if (records.isEmpty()) {
+ return true;
+ }
+
+ @SuppressWarnings("unchecked")
+ Class<T> clazz = (Class<T>) getRecordClass(records.get(0).getClass());
+ QueryResult<T> result;
+ try {
+ result = get(clazz);
+ } catch (IOException e) {
+ return false;
+ }
+ Map<Object, T> writeList = new HashMap<>();
+
+ // Write all of the existing records
+ for (T existingRecord : result.getRecords()) {
+ String key = existingRecord.getPrimaryKey();
+ writeList.put(key, existingRecord);
+ }
+
+ // Add inserts and updates, overwrite any existing values
+ for (T updatedRecord : records) {
+ try {
+ updatedRecord.validate();
+ String key = updatedRecord.getPrimaryKey();
+ if (writeList.containsKey(key) && allowUpdate) {
+ // Update
+ writeList.put(key, updatedRecord);
+ // Update the mod time stamp. Many backends will use their
+ // own timestamp for the mod time.
+ updatedRecord.setDateModified(this.getTime());
+ } else if (!writeList.containsKey(key)) {
+ // Insert
+ // Create/Mod timestamps are already initialized
+ writeList.put(key, updatedRecord);
+ } else if (errorIfExists) {
+ LOG.error("Attempt to insert record {} that already exists",
+ updatedRecord);
+ return false;
+ }
+ } catch (IllegalArgumentException ex) {
+ LOG.error("Cannot write invalid record to State Store", ex);
+ return false;
+ }
+ }
+
+ // Write all
+ boolean status = writeAll(writeList.values(), clazz);
+ return status;
+ }
+
+ @Override
+ public <T extends BaseRecord> int remove(Class<T> clazz, Query<T> query)
+ throws StateStoreUnavailableException {
+ verifyDriverReady();
+
+ if (query == null) {
+ return 0;
+ }
+
+ int removed = 0;
+ // Get the current records
+ try {
+ final QueryResult<T> result = get(clazz);
+ final List<T> existingRecords = result.getRecords();
+ // Write all of the existing records except those to be removed
+ final List<T> recordsToRemove = filterMultiple(query, existingRecords);
+ removed = recordsToRemove.size();
+ final List<T> newRecords = new LinkedList<>();
+ for (T record : existingRecords) {
+ if (!recordsToRemove.contains(record)) {
+ newRecords.add(record);
+ }
+ }
+ if (!writeAll(newRecords, clazz)) {
+ throw new IOException(
+ "Cannot remove record " + clazz + " query " + query);
+ }
+ } catch (IOException e) {
+ LOG.error("Cannot remove records {} query {}", clazz, query, e);
+ }
+
+ return removed;
+ }
+
+ @Override
+ public <T extends BaseRecord> boolean removeAll(Class<T> clazz)
+ throws StateStoreUnavailableException {
+ verifyDriverReady();
+ List<T> emptyList = new ArrayList<>();
+ boolean status = writeAll(emptyList, clazz);
+ return status;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3384811c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java
new file mode 100644
index 0000000..24e9660
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java
@@ -0,0 +1,161 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.io.Files;
+
+/**
+ * StateStoreDriver implementation based on a local file.
+ */
+public class StateStoreFileImpl extends StateStoreFileBaseImpl {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StateStoreFileImpl.class);
+
+ /** Configuration keys. */
+ public static final String FEDERATION_STORE_FILE_DIRECTORY =
+ DFSConfigKeys.FEDERATION_STORE_PREFIX + "driver.file.directory";
+
+ /** Synchronization. */
+ private static final ReadWriteLock READ_WRITE_LOCK =
+ new ReentrantReadWriteLock();
+
+ /** Root directory for the state store. */
+ private String rootDirectory;
+
+
+ @Override
+ protected boolean exists(String path) {
+ File test = new File(path);
+ return test.exists();
+ }
+
+ @Override
+ protected boolean mkdir(String path) {
+ File dir = new File(path);
+ return dir.mkdirs();
+ }
+
+ @Override
+ protected String getRootDir() {
+ if (this.rootDirectory == null) {
+ String dir = getConf().get(FEDERATION_STORE_FILE_DIRECTORY);
+ if (dir == null) {
+ File tempDir = Files.createTempDir();
+ dir = tempDir.getAbsolutePath();
+ }
+ this.rootDirectory = dir;
+ }
+ return this.rootDirectory;
+ }
+
+ @Override
+ protected <T extends BaseRecord> void lockRecordWrite(Class<T> recordClass) {
+ // TODO - Synchronize via FS
+ READ_WRITE_LOCK.writeLock().lock();
+ }
+
+ @Override
+ protected <T extends BaseRecord> void unlockRecordWrite(
+ Class<T> recordClass) {
+ // TODO - Synchronize via FS
+ READ_WRITE_LOCK.writeLock().unlock();
+ }
+
+ @Override
+ protected <T extends BaseRecord> void lockRecordRead(Class<T> recordClass) {
+ // TODO - Synchronize via FS
+ READ_WRITE_LOCK.readLock().lock();
+ }
+
+ @Override
+ protected <T extends BaseRecord> void unlockRecordRead(Class<T> recordClass) {
+ // TODO - Synchronize via FS
+ READ_WRITE_LOCK.readLock().unlock();
+ }
+
+ @Override
+ protected <T extends BaseRecord> BufferedReader getReader(
+ Class<T> clazz, String sub) {
+ String filename = StateStoreUtils.getRecordName(clazz);
+ if (sub != null && sub.length() > 0) {
+ filename += "/" + sub;
+ }
+ filename += "/" + getDataFileName();
+
+ try {
+ LOG.debug("Loading file: {}", filename);
+ File file = new File(getRootDir(), filename);
+ FileInputStream fis = new FileInputStream(file);
+ InputStreamReader isr =
+ new InputStreamReader(fis, StandardCharsets.UTF_8);
+ BufferedReader reader = new BufferedReader(isr);
+ return reader;
+ } catch (Exception ex) {
+ LOG.error(
+ "Cannot open read stream for record {}", clazz.getSimpleName(), ex);
+ return null;
+ }
+ }
+
+ @Override
+ protected <T extends BaseRecord> BufferedWriter getWriter(
+ Class<T> clazz, String sub) {
+ String filename = StateStoreUtils.getRecordName(clazz);
+ if (sub != null && sub.length() > 0) {
+ filename += "/" + sub;
+ }
+ filename += "/" + getDataFileName();
+
+ try {
+ File file = new File(getRootDir(), filename);
+ FileOutputStream fos = new FileOutputStream(file, false);
+ OutputStreamWriter osw =
+ new OutputStreamWriter(fos, StandardCharsets.UTF_8);
+ BufferedWriter writer = new BufferedWriter(osw);
+ return writer;
+ } catch (IOException ex) {
+ LOG.error(
+ "Cannot open read stream for record {}", clazz.getSimpleName(), ex);
+ return null;
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ setInitialized(false);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3384811c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileSystemImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileSystemImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileSystemImpl.java
new file mode 100644
index 0000000..5968421
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileSystemImpl.java
@@ -0,0 +1,178 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * StateStoreDriver} implementation based on a filesystem. The most common uses
+ * HDFS as a backend.
+ */
+public class StateStoreFileSystemImpl extends StateStoreFileBaseImpl {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StateStoreFileSystemImpl.class);
+
+
+ /** Configuration keys. */
+ public static final String FEDERATION_STORE_FS_PATH =
+ DFSConfigKeys.FEDERATION_STORE_PREFIX + "driver.fs.path";
+
+ /** File system to back the State Store. */
+ private FileSystem fs;
+ /** Working path in the filesystem. */
+ private String workPath;
+
+ @Override
+ protected boolean exists(String path) {
+ try {
+ return fs.exists(new Path(path));
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ @Override
+ protected boolean mkdir(String path) {
+ try {
+ return fs.mkdirs(new Path(path));
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ @Override
+ protected String getRootDir() {
+ if (this.workPath == null) {
+ String rootPath = getConf().get(FEDERATION_STORE_FS_PATH);
+ URI workUri;
+ try {
+ workUri = new URI(rootPath);
+ fs = FileSystem.get(workUri, getConf());
+ } catch (Exception ex) {
+ return null;
+ }
+ this.workPath = rootPath;
+ }
+ return this.workPath;
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (fs != null) {
+ fs.close();
+ }
+ }
+
+ /**
+ * Get the folder path for the record class' data.
+ *
+ * @param cls Data record class.
+ * @return Path of the folder containing the record class' data files.
+ */
+ private Path getPathForClass(Class<? extends BaseRecord> clazz) {
+ if (clazz == null) {
+ return null;
+ }
+ // TODO extract table name from class: entry.getTableName()
+ String className = StateStoreUtils.getRecordName(clazz);
+ return new Path(workPath, className);
+ }
+
+ @Override
+ protected <T extends BaseRecord> void lockRecordRead(Class<T> clazz) {
+ // Not required, synced with HDFS leasing
+ }
+
+ @Override
+ protected <T extends BaseRecord> void unlockRecordRead(Class<T> clazz) {
+ // Not required, synced with HDFS leasing
+ }
+
+ @Override
+ protected <T extends BaseRecord> void lockRecordWrite(Class<T> clazz) {
+ // TODO -> wait for lease to be available
+ }
+
+ @Override
+ protected <T extends BaseRecord> void unlockRecordWrite(Class<T> clazz) {
+ // TODO -> ensure lease is closed for the file
+ }
+
+ @Override
+ protected <T extends BaseRecord> BufferedReader getReader(
+ Class<T> clazz, String sub) {
+
+ Path path = getPathForClass(clazz);
+ if (sub != null && sub.length() > 0) {
+ path = Path.mergePaths(path, new Path("/" + sub));
+ }
+ path = Path.mergePaths(path, new Path("/" + getDataFileName()));
+
+ try {
+ FSDataInputStream fdis = fs.open(path);
+ InputStreamReader isr =
+ new InputStreamReader(fdis, StandardCharsets.UTF_8);
+ BufferedReader reader = new BufferedReader(isr);
+ return reader;
+ } catch (IOException ex) {
+ LOG.error("Cannot open write stream for {} to {}",
+ clazz.getSimpleName(), path);
+ return null;
+ }
+ }
+
+ @Override
+ protected <T extends BaseRecord> BufferedWriter getWriter(
+ Class<T> clazz, String sub) {
+
+ Path path = getPathForClass(clazz);
+ if (sub != null && sub.length() > 0) {
+ path = Path.mergePaths(path, new Path("/" + sub));
+ }
+ path = Path.mergePaths(path, new Path("/" + getDataFileName()));
+
+ try {
+ FSDataOutputStream fdos = fs.create(path, true);
+ OutputStreamWriter osw =
+ new OutputStreamWriter(fdos, StandardCharsets.UTF_8);
+ BufferedWriter writer = new BufferedWriter(osw);
+ return writer;
+ } catch (IOException ex) {
+ LOG.error("Cannot open write stream for {} to {}",
+ clazz.getSimpleName(), path);
+ return null;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3384811c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializableImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializableImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializableImpl.java
new file mode 100644
index 0000000..e9b3fdf
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializableImpl.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+
+/**
+ * State Store driver that stores a serialization of the records. The serializer
+ * is pluggable.
+ */
+public abstract class StateStoreSerializableImpl extends StateStoreBaseImpl {
+
+ /** Default serializer for this driver. */
+ private StateStoreSerializer serializer;
+
+
+ @Override
+ public boolean init(final Configuration config, final String id,
+ final Collection<Class<? extends BaseRecord>> records) {
+ boolean ret = super.init(config, id, records);
+
+ this.serializer = StateStoreSerializer.getSerializer(config);
+
+ return ret;
+ }
+
+ /**
+ * Serialize a record using the serializer.
+ * @param record Record to serialize.
+ * @return Byte array with the serialization of the record.
+ */
+ protected <T extends BaseRecord> byte[] serialize(T record) {
+ return serializer.serialize(record);
+ }
+
+ /**
+ * Serialize a record using the serializer.
+ * @param record Record to serialize.
+ * @return String with the serialization of the record.
+ */
+ protected <T extends BaseRecord> String serializeString(T record) {
+ return serializer.serializeString(record);
+ }
+
+ /**
+ * Creates a record from an input data string.
+ * @param data Serialized text of the record.
+ * @param clazz Record class.
+ * @param includeDates If dateModified and dateCreated are serialized.
+ * @return The created record.
+ * @throws IOException
+ */
+ protected <T extends BaseRecord> T newRecord(
+ String data, Class<T> clazz, boolean includeDates) throws IOException {
+ return serializer.deserialize(data, clazz);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3384811c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java
index 4192a3d..79f99c8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java
@@ -123,6 +123,24 @@ public abstract class BaseRecord implements Comparable<BaseRecord> {
}
/**
+ * Check if this record matches a partial record.
+ *
+ * @param other Partial record.
+ * @return If this record matches.
+ */
+ public boolean like(BaseRecord other) {
+ if (other == null) {
+ return false;
+ }
+ Map<String, String> thisKeys = this.getPrimaryKeys();
+ Map<String, String> otherKeys = other.getPrimaryKeys();
+ if (thisKeys == null) {
+ return otherKeys == null;
+ }
+ return thisKeys.equals(otherKeys);
+ }
+
+ /**
* Override equals check to use primary key(s) for comparison.
*/
@Override
@@ -186,4 +204,4 @@ public abstract class BaseRecord implements Comparable<BaseRecord> {
public String toString() {
return getPrimaryKey();
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3384811c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/Query.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/Query.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/Query.java
new file mode 100644
index 0000000..3c59abf
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/Query.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+/**
+ * Check if a record matches a query. The query is usually a partial record.
+ *
+ * @param <T> Type of the record to query.
+ */
+public class Query<T extends BaseRecord> {
+
+ /** Partial object to compare against. */
+ private final T partial;
+
+
+ /**
+ * Create a query to search for a partial record.
+ *
+ * @param partial It defines the attributes to search.
+ */
+ public Query(final T part) {
+ this.partial = part;
+ }
+
+ /**
+ * Get the partial record used to query.
+ *
+ * @return The partial record used for the query.
+ */
+ public T getPartial() {
+ return this.partial;
+ }
+
+ /**
+ * Check if a record matches the primary keys or the partial record.
+ *
+ * @param other Record to check.
+ * @return If the record matches. Don't match if there is no partial.
+ */
+ public boolean matches(T other) {
+ if (this.partial == null) {
+ return false;
+ }
+ return this.partial.like(other);
+ }
+
+ @Override
+ public String toString() {
+ return "Checking: " + this.partial;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3384811c/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 3b6146a..3935023 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4655,4 +4655,20 @@
</description>
</property>
+ <property>
+ <name>dfs.federation.router.store.driver.class</name>
+ <value>org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl</value>
+ <description>
+ Class to implement the State Store. By default it uses the local disk.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.store.connection.test</name>
+ <value>60000</value>
+ <description>
+ How often to check for the connection to the State Store in milliseconds.
+ </description>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3384811c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
new file mode 100644
index 0000000..fc5aebd
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
@@ -0,0 +1,232 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.FEDERATION_STORE_DRIVER_CLASS;
+import static org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl.FEDERATION_STORE_FILE_DIRECTORY;
+import static org.junit.Assert.assertNotNull;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileBaseImpl;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.util.Time;
+
+/**
+ * Utilities to test the State Store.
+ */
+public final class FederationStateStoreTestUtils {
+
+ private FederationStateStoreTestUtils() {
+ // Utility Class
+ }
+
+ /**
+ * Get the default State Store driver implementation.
+ *
+ * @return Class of the default State Store driver implementation.
+ */
+ public static Class<? extends StateStoreDriver> getDefaultDriver() {
+ return DFSConfigKeys.FEDERATION_STORE_DRIVER_CLASS_DEFAULT;
+ }
+
+ /**
+ * Create a default State Store configuration.
+ *
+ * @return State Store configuration.
+ */
+ public static Configuration getStateStoreConfiguration() {
+ Class<? extends StateStoreDriver> clazz = getDefaultDriver();
+ return getStateStoreConfiguration(clazz);
+ }
+
+ /**
+ * Create a new State Store configuration for a particular driver.
+ *
+ * @param clazz Class of the driver to create.
+ * @return State Store configuration.
+ */
+ public static Configuration getStateStoreConfiguration(
+ Class<? extends StateStoreDriver> clazz) {
+ Configuration conf = new HdfsConfiguration(false);
+
+ conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, true);
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, "hdfs://test");
+
+ conf.setClass(FEDERATION_STORE_DRIVER_CLASS, clazz, StateStoreDriver.class);
+
+ if (clazz.isAssignableFrom(StateStoreFileBaseImpl.class)) {
+ setFileConfiguration(conf);
+ }
+ return conf;
+ }
+
+ /**
+ * Create a new State Store based on a configuration.
+ *
+ * @param configuration Configuration for the State Store.
+ * @return New State Store service.
+ * @throws IOException If it cannot create the State Store.
+ * @throws InterruptedException If we cannot wait for the store to start.
+ */
+ public static StateStoreService getStateStore(
+ Configuration configuration) throws IOException, InterruptedException {
+
+ StateStoreService stateStore = new StateStoreService();
+ assertNotNull(stateStore);
+
+ // Set unique identifier, this is normally the router address
+ String identifier = UUID.randomUUID().toString();
+ stateStore.setIdentifier(identifier);
+
+ stateStore.init(configuration);
+ stateStore.start();
+
+ // Wait for state store to connect
+ waitStateStore(stateStore, TimeUnit.SECONDS.toMillis(10));
+
+ return stateStore;
+ }
+
+ /**
+ * Wait for the State Store to initialize its driver.
+ *
+ * @param stateStore State Store.
+ * @param timeoutMs Time out in milliseconds.
+ * @throws IOException If the State Store cannot be reached.
+ * @throws InterruptedException If the sleep is interrupted.
+ */
+ public static void waitStateStore(StateStoreService stateStore,
+ long timeoutMs) throws IOException, InterruptedException {
+ long startingTime = Time.monotonicNow();
+ while (!stateStore.isDriverReady()) {
+ Thread.sleep(100);
+ if (Time.monotonicNow() - startingTime > timeoutMs) {
+ throw new IOException("Timeout waiting for State Store to connect");
+ }
+ }
+ }
+
+ /**
+ * Delete the default State Store.
+ *
+ * @throws IOException
+ */
+ public static void deleteStateStore() throws IOException {
+ Class<? extends StateStoreDriver> driverClass = getDefaultDriver();
+ deleteStateStore(driverClass);
+ }
+
+ /**
+ * Delete the State Store.
+ * @param driverClass Class of the State Store driver implementation.
+ * @throws IOException If it cannot be deleted.
+ */
+ public static void deleteStateStore(
+ Class<? extends StateStoreDriver> driverClass) throws IOException {
+
+ if (StateStoreFileBaseImpl.class.isAssignableFrom(driverClass)) {
+ String workingDirectory = System.getProperty("user.dir");
+ File dir = new File(workingDirectory + "/statestore");
+ if (dir.exists()) {
+ FileUtils.cleanDirectory(dir);
+ }
+ }
+ }
+
+ /**
+ * Set the default configuration for drivers based on files.
+ *
+ * @param conf Configuration to extend.
+ */
+ public static void setFileConfiguration(Configuration conf) {
+ String workingPath = System.getProperty("user.dir");
+ String stateStorePath = workingPath + "/statestore";
+ conf.set(FEDERATION_STORE_FILE_DIRECTORY, stateStorePath);
+ }
+
+ /**
+ * Clear all the records from the State Store.
+ *
+ * @param store State Store to remove records from.
+ * @return If the State Store was cleared.
+ * @throws IOException If it cannot clear the State Store.
+ */
+ public static boolean clearAllRecords(StateStoreService store)
+ throws IOException {
+ Collection<Class<? extends BaseRecord>> allRecords =
+ store.getSupportedRecords();
+ for (Class<? extends BaseRecord> recordType : allRecords) {
+ if (!clearRecords(store, recordType)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Clear records from a certain type from the State Store.
+ *
+ * @param store State Store to remove records from.
+ * @param recordClass Class of the records to remove.
+ * @return If the State Store was cleared.
+ * @throws IOException If it cannot clear the State Store.
+ */
+ public static <T extends BaseRecord> boolean clearRecords(
+ StateStoreService store, Class<T> recordClass) throws IOException {
+ List<T> emptyList = new ArrayList<>();
+ if (!synchronizeRecords(store, emptyList, recordClass)) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Synchronize a set of records. Remove all and keep the ones specified.
+ *
+ * @param stateStore State Store service managing the driver.
+ * @param records Records to add.
+ * @param clazz Class of the record to synchronize.
+ * @return If the synchronization succeeded.
+ * @throws IOException If it cannot connect to the State Store.
+ */
+ public static <T extends BaseRecord> boolean synchronizeRecords(
+ StateStoreService stateStore, List<T> records, Class<T> clazz)
+ throws IOException {
+ StateStoreDriver driver = stateStore.getDriver();
+ driver.verifyDriverReady();
+ if (driver.removeAll(clazz)) {
+ if (driver.putAll(records, true, false)) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/40] hadoop git commit: HDFS-10687. Federation Membership State
Store internal API. Contributed by Jason Kace and Inigo Goiri.
Posted by in...@apache.org.
HDFS-10687. Federation Membership State Store internal API. Contributed by Jason Kace and Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b43bb809
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b43bb809
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b43bb809
Branch: refs/heads/HDFS-10467
Commit: b43bb809adddb94c5bfc676eeb6f2e3eb931d35d
Parents: 4634769
Author: Inigo Goiri <in...@apache.org>
Authored: Mon Jul 31 10:55:21 2017 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu Sep 7 13:19:50 2017 -0700
----------------------------------------------------------------------
.../dev-support/findbugsExcludeFile.xml | 3 +
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 +
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 17 +-
.../resolver/MembershipNamenodeResolver.java | 290 ++++++++++++
.../federation/router/FederationUtil.java | 42 +-
.../federation/store/CachedRecordStore.java | 237 ++++++++++
.../federation/store/MembershipStore.java | 126 +++++
.../federation/store/StateStoreCache.java | 36 ++
.../store/StateStoreCacheUpdateService.java | 67 +++
.../federation/store/StateStoreService.java | 202 +++++++-
.../store/impl/MembershipStoreImpl.java | 311 +++++++++++++
.../federation/store/impl/package-info.java | 31 ++
.../GetNamenodeRegistrationsRequest.java | 52 +++
.../GetNamenodeRegistrationsResponse.java | 55 +++
.../store/protocol/GetNamespaceInfoRequest.java | 30 ++
.../protocol/GetNamespaceInfoResponse.java | 52 +++
.../protocol/NamenodeHeartbeatRequest.java | 52 +++
.../protocol/NamenodeHeartbeatResponse.java | 49 ++
.../UpdateNamenodeRegistrationRequest.java | 72 +++
.../UpdateNamenodeRegistrationResponse.java | 51 ++
.../impl/pb/FederationProtocolPBTranslator.java | 145 ++++++
.../GetNamenodeRegistrationsRequestPBImpl.java | 87 ++++
.../GetNamenodeRegistrationsResponsePBImpl.java | 99 ++++
.../impl/pb/GetNamespaceInfoRequestPBImpl.java | 60 +++
.../impl/pb/GetNamespaceInfoResponsePBImpl.java | 95 ++++
.../impl/pb/NamenodeHeartbeatRequestPBImpl.java | 93 ++++
.../pb/NamenodeHeartbeatResponsePBImpl.java | 71 +++
...UpdateNamenodeRegistrationRequestPBImpl.java | 95 ++++
...pdateNamenodeRegistrationResponsePBImpl.java | 73 +++
.../store/protocol/impl/pb/package-info.java | 29 ++
.../store/records/MembershipState.java | 329 +++++++++++++
.../store/records/MembershipStats.java | 126 +++++
.../records/impl/pb/MembershipStatePBImpl.java | 334 +++++++++++++
.../records/impl/pb/MembershipStatsPBImpl.java | 191 ++++++++
.../src/main/proto/FederationProtocol.proto | 107 +++++
.../src/main/resources/hdfs-default.xml | 18 +-
.../resolver/TestNamenodeResolver.java | 284 ++++++++++++
.../store/FederationStateStoreTestUtils.java | 23 +-
.../federation/store/TestStateStoreBase.java | 81 ++++
.../store/TestStateStoreMembershipState.java | 463 +++++++++++++++++++
.../store/driver/TestStateStoreDriverBase.java | 69 ++-
.../store/records/TestMembershipState.java | 129 ++++++
42 files changed, 4745 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
index 9582fcb..4b958b5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
@@ -15,6 +15,9 @@
<Package name="org.apache.hadoop.hdfs.qjournal.protocol" />
</Match>
<Match>
+ <Package name="org.apache.hadoop.hdfs.federation.protocol.proto" />
+ </Match>
+ <Match>
<Bug pattern="EI_EXPOSE_REP" />
</Match>
<Match>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index 425572f..cc7a975 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -331,6 +331,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<include>QJournalProtocol.proto</include>
<include>editlog.proto</include>
<include>fsimage.proto</include>
+ <include>FederationProtocol.proto</include>
</includes>
</source>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 341a502..b9c7408 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -27,6 +27,8 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRackFaultTolerant;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.RamDiskReplicaLruTracker;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver;
import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl;
import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreSerializerPBImpl;
@@ -1166,8 +1168,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
"org.apache.hadoop.hdfs.server.federation.MockResolver";
public static final String FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS =
FEDERATION_ROUTER_PREFIX + "namenode.resolver.client.class";
- public static final String FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS_DEFAULT =
- "org.apache.hadoop.hdfs.server.federation.MockResolver";
+ public static final Class<? extends ActiveNamenodeResolver>
+ FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS_DEFAULT =
+ MembershipNamenodeResolver.class;
// HDFS Router-based federation State Store
public static final String FEDERATION_STORE_PREFIX =
@@ -1189,6 +1192,16 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final long FEDERATION_STORE_CONNECTION_TEST_MS_DEFAULT =
TimeUnit.MINUTES.toMillis(1);
+ public static final String DFS_ROUTER_CACHE_TIME_TO_LIVE_MS =
+ FEDERATION_ROUTER_PREFIX + "cache.ttl";
+ public static final long DFS_ROUTER_CACHE_TIME_TO_LIVE_MS_DEFAULT =
+ TimeUnit.MINUTES.toMillis(1);
+
+ public static final String FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS =
+ FEDERATION_STORE_PREFIX + "membership.expiration";
+ public static final long FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS_DEFAULT =
+ TimeUnit.MINUTES.toMillis(5);
+
// dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry
@Deprecated
public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java
new file mode 100644
index 0000000..b0ced24
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java
@@ -0,0 +1,290 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.ACTIVE;
+import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.EXPIRED;
+import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.UNAVAILABLE;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.store.MembershipStore;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreCache;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreUnavailableException;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateNamenodeRegistrationRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Implements a cached lookup of the most recently active namenode for a
+ * particular nameservice. Relies on the {@link StateStoreService} to
+ * discover available nameservices and namenodes.
+ */
+public class MembershipNamenodeResolver
+ implements ActiveNamenodeResolver, StateStoreCache {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(MembershipNamenodeResolver.class);
+
+ /** Reference to the State Store. */
+ private final StateStoreService stateStore;
+ /** Membership State Store interface. */
+ private final MembershipStore membershipInterface;
+
+ /** Parent router ID. */
+ private String routerId;
+
+ /** Cached lookup of NN for nameservice. Invalidated on cache refresh. */
+ private Map<String, List<? extends FederationNamenodeContext>> cacheNS;
+ /** Cached lookup of NN for block pool. Invalidated on cache refresh. */
+ private Map<String, List<? extends FederationNamenodeContext>> cacheBP;
+
+
+ public MembershipNamenodeResolver(
+ Configuration conf, StateStoreService store) throws IOException {
+ this.stateStore = store;
+
+ this.cacheNS = new ConcurrentHashMap<>();
+ this.cacheBP = new ConcurrentHashMap<>();
+
+ if (this.stateStore != null) {
+ // Request cache updates from the state store
+ this.stateStore.registerCacheExternal(this);
+
+ // Initialize the interface to get the membership
+ this.membershipInterface = this.stateStore.getRegisteredRecordStore(
+ MembershipStore.class);
+ } else {
+ this.membershipInterface = null;
+ }
+
+ if (this.membershipInterface == null) {
+ throw new IOException("State Store does not have an interface for " +
+ MembershipStore.class.getSimpleName());
+ }
+ }
+
+ @Override
+ public boolean loadCache(boolean force) {
+ // Our cache depends on the store, update it first
+ try {
+ this.membershipInterface.loadCache(force);
+ } catch (IOException e) {
+ LOG.error("Cannot update membership from the State Store", e);
+ }
+
+ // Force refresh of active NN cache
+ cacheBP.clear();
+ cacheNS.clear();
+ return true;
+ }
+
+ @Override
+ public void updateActiveNamenode(
+ final String nsId, final InetSocketAddress address) throws IOException {
+
+ // Called when we have an RPC miss and successful hit on an alternate NN.
+ // Temporarily update our cache, it will be overwritten on the next update.
+ try {
+ MembershipState partial = MembershipState.newInstance();
+ String rpcAddress = address.getHostName() + ":" + address.getPort();
+ partial.setRpcAddress(rpcAddress);
+ partial.setNameserviceId(nsId);
+
+ GetNamenodeRegistrationsRequest request =
+ GetNamenodeRegistrationsRequest.newInstance(partial);
+
+ GetNamenodeRegistrationsResponse response =
+ this.membershipInterface.getNamenodeRegistrations(request);
+ List<MembershipState> records = response.getNamenodeMemberships();
+
+ if (records != null && records.size() == 1) {
+ MembershipState record = records.get(0);
+ UpdateNamenodeRegistrationRequest updateRequest =
+ UpdateNamenodeRegistrationRequest.newInstance(
+ record.getNameserviceId(), record.getNamenodeId(), ACTIVE);
+ this.membershipInterface.updateNamenodeRegistration(updateRequest);
+ }
+ } catch (StateStoreUnavailableException e) {
+ LOG.error("Cannot update {} as active, State Store unavailable", address);
+ }
+ }
+
+ @Override
+ public List<? extends FederationNamenodeContext> getNamenodesForNameserviceId(
+ final String nsId) throws IOException {
+
+ List<? extends FederationNamenodeContext> ret = cacheNS.get(nsId);
+ if (ret == null) {
+ try {
+ MembershipState partial = MembershipState.newInstance();
+ partial.setNameserviceId(nsId);
+ GetNamenodeRegistrationsRequest request =
+ GetNamenodeRegistrationsRequest.newInstance(partial);
+
+ final List<MembershipState> result =
+ getRecentRegistrationForQuery(request, true, false);
+ if (result == null || result.isEmpty()) {
+ LOG.error("Cannot locate eligible NNs for {}", nsId);
+ return null;
+ } else {
+ cacheNS.put(nsId, result);
+ ret = result;
+ }
+ } catch (StateStoreUnavailableException e) {
+ LOG.error("Cannot get active NN for {}, State Store unavailable", nsId);
+ }
+ }
+ if (ret == null) {
+ return null;
+ }
+ return Collections.unmodifiableList(ret);
+ }
+
+ @Override
+ public List<? extends FederationNamenodeContext> getNamenodesForBlockPoolId(
+ final String bpId) throws IOException {
+
+ List<? extends FederationNamenodeContext> ret = cacheBP.get(bpId);
+ if (ret == null) {
+ try {
+ MembershipState partial = MembershipState.newInstance();
+ partial.setBlockPoolId(bpId);
+ GetNamenodeRegistrationsRequest request =
+ GetNamenodeRegistrationsRequest.newInstance(partial);
+
+ final List<MembershipState> result =
+ getRecentRegistrationForQuery(request, true, false);
+ if (result == null || result.isEmpty()) {
+ LOG.error("Cannot locate eligible NNs for {}", bpId);
+ } else {
+ cacheBP.put(bpId, result);
+ ret = result;
+ }
+ } catch (StateStoreUnavailableException e) {
+ LOG.error("Cannot get active NN for {}, State Store unavailable", bpId);
+ return null;
+ }
+ }
+ if (ret == null) {
+ return null;
+ }
+ return Collections.unmodifiableList(ret);
+ }
+
+ @Override
+ public boolean registerNamenode(NamenodeStatusReport report)
+ throws IOException {
+
+ if (this.routerId == null) {
+ LOG.warn("Cannot register namenode, router ID is not known {}", report);
+ return false;
+ }
+
+ MembershipState record = MembershipState.newInstance(
+ routerId, report.getNameserviceId(), report.getNamenodeId(),
+ report.getClusterId(), report.getBlockPoolId(), report.getRpcAddress(),
+ report.getServiceAddress(), report.getLifelineAddress(),
+ report.getWebAddress(), report.getState(), report.getSafemode());
+
+ if (report.getState() != UNAVAILABLE) {
+ // Set/update our last contact time
+ record.setLastContact(Time.now());
+ }
+
+ NamenodeHeartbeatRequest request = NamenodeHeartbeatRequest.newInstance();
+ request.setNamenodeMembership(record);
+ return this.membershipInterface.namenodeHeartbeat(request).getResult();
+ }
+
+ @Override
+ public Set<FederationNamespaceInfo> getNamespaces() throws IOException {
+ GetNamespaceInfoRequest request = GetNamespaceInfoRequest.newInstance();
+ GetNamespaceInfoResponse response =
+ this.membershipInterface.getNamespaceInfo(request);
+ return response.getNamespaceInfo();
+ }
+
+ /**
+ * Picks the most relevant record registration that matches the query. Return
+ * registrations matching the query in this preference: 1) Most recently
+ * updated ACTIVE registration 2) Most recently updated STANDBY registration
+ * (if showStandby) 3) Most recently updated UNAVAILABLE registration (if
+ * showUnavailable). EXPIRED registrations are ignored.
+ *
+ * @param query The select query for NN registrations.
+ * @param excludes List of NNs to exclude from matching results.
+ * @param addUnavailable include UNAVAILABLE registrations.
+ * @param addExpired include EXPIRED registrations.
+ * @return List of memberships or null if no registrations that
+ * both match the query AND the selected states.
+ * @throws IOException
+ */
+ private List<MembershipState> getRecentRegistrationForQuery(
+ GetNamenodeRegistrationsRequest request, boolean addUnavailable,
+ boolean addExpired) throws IOException {
+
+ // Retrieve a list of all registrations that match this query.
+ // This may include all NN records for a namespace/blockpool, including
+ // duplicate records for the same NN from different routers.
+ GetNamenodeRegistrationsResponse response =
+ this.membershipInterface.getNamenodeRegistrations(request);
+
+ List<MembershipState> memberships = response.getNamenodeMemberships();
+ if (!addExpired || !addUnavailable) {
+ Iterator<MembershipState> iterator = memberships.iterator();
+ while (iterator.hasNext()) {
+ MembershipState membership = iterator.next();
+ if (membership.getState() == EXPIRED && !addExpired) {
+ iterator.remove();
+ } else if (membership.getState() == UNAVAILABLE && !addUnavailable) {
+ iterator.remove();
+ }
+ }
+ }
+
+ List<MembershipState> priorityList = new ArrayList<>();
+ priorityList.addAll(memberships);
+ Collections.sort(priorityList, new NamenodePriorityComparator());
+
+ LOG.debug("Selected most recent NN {} for query", priorityList);
+ return priorityList;
+ }
+
+ @Override
+ public void setRouterId(String router) {
+ this.routerId = router;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
index 6e7e865..0129a37 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
@@ -19,26 +19,57 @@ package org.apache.hadoop.hdfs.server.federation.router;
import java.lang.reflect.Constructor;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Utilities for managing HDFS federation.
*/
public final class FederationUtil {
- private static final Log LOG = LogFactory.getLog(FederationUtil.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(FederationUtil.class);
private FederationUtil() {
// Utility Class
}
/**
+ * Create an instance of an interface with a constructor using a context.
+ *
+ * @param conf Configuration for the class names.
+ * @param context Context object to pass to the instance.
+ * @param contextClass Type of the context passed to the constructor.
+ * @param clazz Class of the object to return.
+ * @return New instance of the specified class that implements the desired
+ * interface and a single parameter constructor containing a
+ * StateStore reference.
+ */
+ private static <T, R> T newInstance(final Configuration conf,
+ final R context, final Class<R> contextClass, final Class<T> clazz) {
+ try {
+ if (contextClass == null) {
+ // Default constructor if no context
+ Constructor<T> constructor = clazz.getConstructor();
+ return constructor.newInstance();
+ } else {
+ // Constructor with context
+ Constructor<T> constructor = clazz.getConstructor(
+ Configuration.class, contextClass);
+ return constructor.newInstance(conf, context);
+ }
+ } catch (ReflectiveOperationException e) {
+ LOG.error("Could not instantiate: {}", clazz.getSimpleName(), e);
+ return null;
+ }
+ }
+
+ /**
* Create an instance of an interface with a constructor using a state store
* constructor.
*
@@ -105,13 +136,14 @@ public final class FederationUtil {
*
* @param conf Configuration that defines the namenode resolver class.
* @param obj Context object passed to class constructor.
- * @return ActiveNamenodeResolver
+ * @return New active namenode resolver.
*/
public static ActiveNamenodeResolver newActiveNamenodeResolver(
Configuration conf, StateStoreService stateStore) {
- return newInstance(conf, stateStore, StateStoreService.class,
+ Class<? extends ActiveNamenodeResolver> clazz = conf.getClass(
DFSConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS,
DFSConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS_DEFAULT,
ActiveNamenodeResolver.class);
+ return newInstance(conf, stateStore, StateStoreService.class, clazz);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/CachedRecordStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/CachedRecordStore.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/CachedRecordStore.java
new file mode 100644
index 0000000..90a6699
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/CachedRecordStore.java
@@ -0,0 +1,237 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Record store that takes care of caching the records in memory.
+ *
+ * @param <R> Record to store by this interface.
+ */
+public abstract class CachedRecordStore<R extends BaseRecord>
+ extends RecordStore<R> implements StateStoreCache {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(CachedRecordStore.class);
+
+
+ /** Prevent loading the cache more than once every 500 ms. */
+ private static final long MIN_UPDATE_MS = 500;
+
+
+ /** Cached entries. */
+ private List<R> records = new ArrayList<>();
+
+ /** Time stamp of the cached entries. */
+ private long timestamp = -1;
+
+ /** If the cache is initialized. */
+ private boolean initialized = false;
+
+ /** Last time the cache was updated. */
+ private long lastUpdate = -1;
+
+ /** Lock to access the memory cache. */
+ private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ private final Lock readLock = readWriteLock.readLock();
+ private final Lock writeLock = readWriteLock.writeLock();
+
+ /** If it should override the expired values when loading the cache. */
+ private boolean override = false;
+
+
+ /**
+ * Create a new cached record store.
+ *
+ * @param clazz Class of the record to store.
+ * @param driver State Store driver.
+ */
+ protected CachedRecordStore(Class<R> clazz, StateStoreDriver driver) {
+ this(clazz, driver, false);
+ }
+
+ /**
+ * Create a new cached record store.
+ *
+ * @param clazz Class of the record to store.
+ * @param driver State Store driver.
+ * @param override If the entries should be override if they expire
+ */
+ protected CachedRecordStore(
+ Class<R> clazz, StateStoreDriver driver, boolean over) {
+ super(clazz, driver);
+
+ this.override = over;
+ }
+
+ /**
+ * Check that the cache of the State Store information is available.
+ *
+ * @throws StateStoreUnavailableException If the cache is not initialized.
+ */
+ private void checkCacheAvailable() throws StateStoreUnavailableException {
+ if (!this.initialized) {
+ throw new StateStoreUnavailableException(
+ "Cached State Store not initialized, " +
+ getRecordClass().getSimpleName() + " records not valid");
+ }
+ }
+
+ @Override
+ public boolean loadCache(boolean force) throws IOException {
+ // Prevent loading the cache too frequently
+ if (force || isUpdateTime()) {
+ List<R> newRecords = null;
+ long t = -1;
+ try {
+ QueryResult<R> result = getDriver().get(getRecordClass());
+ newRecords = result.getRecords();
+ t = result.getTimestamp();
+
+ // If we have any expired record, update the State Store
+ if (this.override) {
+ overrideExpiredRecords(result);
+ }
+ } catch (IOException e) {
+ LOG.error("Cannot get \"{}\" records from the State Store",
+ getRecordClass().getSimpleName());
+ this.initialized = false;
+ return false;
+ }
+
+ // Update cache atomically
+ writeLock.lock();
+ try {
+ this.records.clear();
+ this.records.addAll(newRecords);
+ this.timestamp = t;
+ this.initialized = true;
+ } finally {
+ writeLock.unlock();
+ }
+
+ lastUpdate = Time.monotonicNow();
+ }
+ return true;
+ }
+
+ /**
+ * Check if it's time to update the cache. Update it it was never updated.
+ *
+ * @return If it's time to update this cache.
+ */
+ private boolean isUpdateTime() {
+ return Time.monotonicNow() - lastUpdate > MIN_UPDATE_MS;
+ }
+
+ /**
+ * Updates the state store with any record overrides we detected, such as an
+ * expired state.
+ *
+ * @param query RecordQueryResult containing the data to be inspected.
+ * @param clazz Type of objects contained in the query.
+ * @throws IOException
+ */
+ public void overrideExpiredRecords(QueryResult<R> query) throws IOException {
+ List<R> commitRecords = new ArrayList<>();
+ List<R> newRecords = query.getRecords();
+ long currentDriverTime = query.getTimestamp();
+ if (newRecords == null || currentDriverTime <= 0) {
+ LOG.error("Cannot check overrides for record");
+ return;
+ }
+ for (R record : newRecords) {
+ if (record.checkExpired(currentDriverTime)) {
+ String recordName = StateStoreUtils.getRecordName(record.getClass());
+ LOG.info("Override State Store record {}: {}", recordName, record);
+ commitRecords.add(record);
+ }
+ }
+ if (commitRecords.size() > 0) {
+ getDriver().putAll(commitRecords, true, false);
+ }
+ }
+
+ /**
+ * Updates the state store with any record overrides we detected, such as an
+ * expired state.
+ *
+ * @param driver State store driver for the data store.
+ * @param record Record record to be updated.
+ * @param clazz Type of data record.
+ * @throws IOException
+ */
+ public void overrideExpiredRecord(R record) throws IOException {
+ List<R> newRecords = Collections.singletonList(record);
+ long time = getDriver().getTime();
+ QueryResult<R> query = new QueryResult<>(newRecords, time);
+ overrideExpiredRecords(query);
+ }
+
+ /**
+ * Get all the cached records.
+ *
+ * @return Copy of the cached records.
+ * @throws StateStoreUnavailableException If the State store is not available.
+ */
+ public List<R> getCachedRecords() throws StateStoreUnavailableException {
+ checkCacheAvailable();
+
+ List<R> ret = new LinkedList<R>();
+ this.readLock.lock();
+ try {
+ ret.addAll(this.records);
+ } finally {
+ this.readLock.unlock();
+ }
+ return ret;
+ }
+
+ /**
+ * Get all the cached records and the time stamp of the cache.
+ *
+ * @return Copy of the cached records and the time stamp.
+ * @throws StateStoreUnavailableException If the State store is not available.
+ */
+ protected QueryResult<R> getCachedRecordsAndTimeStamp()
+ throws StateStoreUnavailableException {
+ checkCacheAvailable();
+
+ this.readLock.lock();
+ try {
+ return new QueryResult<R>(this.records, this.timestamp);
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MembershipStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MembershipStore.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MembershipStore.java
new file mode 100644
index 0000000..3e8ba6b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MembershipStore.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateNamenodeRegistrationRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateNamenodeRegistrationResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+
+/**
+ * Management API for NameNode registrations stored in
+ * {@link org.apache.hadoop.hdfs.server.federation.store.records.MembershipState
+ * MembershipState} records. The {@link org.apache.hadoop.hdfs.server.
+ * federation.router.RouterHeartbeatService RouterHeartbeatService} periodically
+ * polls each NN to update the NameNode metadata(addresses, operational) and HA
+ * state(active, standby). Each NameNode may be polled by multiple
+ * {@link org.apache.hadoop.hdfs.server.federation.router.Router Router}
+ * instances.
+ * <p>
+ * Once fetched from the
+ * {@link org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver
+ * StateStoreDriver}, NameNode registrations are cached until the next query.
+ * The fetched registration data is aggregated using a quorum to determine the
+ * best/most accurate state for each NameNode. The cache is periodically updated
+ * by the @{link StateStoreCacheUpdateService}.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public abstract class MembershipStore
+ extends CachedRecordStore<MembershipState> {
+
+ protected MembershipStore(StateStoreDriver driver) {
+ super(MembershipState.class, driver, true);
+ }
+
+ /**
+ * Inserts or updates a namenode membership entry into the table.
+ *
+ * @param request Fully populated NamenodeHeartbeatRequest request.
+ * @return True if successful, false otherwise.
+ * @throws StateStoreUnavailableException Throws exception if the data store
+ * is not initialized.
+ * @throws IOException if the data store could not be queried or the query is
+ * invalid.
+ */
+ public abstract NamenodeHeartbeatResponse namenodeHeartbeat(
+ NamenodeHeartbeatRequest request) throws IOException;
+
+ /**
+ * Queries for a single cached registration entry matching the given
+ * parameters. Possible keys are the names of data structure elements Possible
+ * values are matching SQL "LIKE" targets.
+ *
+ * @param request Fully populated GetNamenodeRegistrationsRequest request.
+ * @return Single matching FederationMembershipStateEntry or null if not found
+ * or more than one entry matches.
+ * @throws StateStoreUnavailableException Throws exception if the data store
+ * is not initialized.
+ * @throws IOException if the data store could not be queried or the query is
+ * invalid.
+ */
+ public abstract GetNamenodeRegistrationsResponse getNamenodeRegistrations(
+ GetNamenodeRegistrationsRequest request) throws IOException;
+
+ /**
+ * Get the expired registrations from the registration cache.
+ *
+ * @return Expired registrations or zero-length list if none are found.
+ * @throws StateStoreUnavailableException Throws exception if the data store
+ * is not initialized.
+ * @throws IOException if the data store could not be queried or the query is
+ * invalid.
+ */
+ public abstract GetNamenodeRegistrationsResponse
+ getExpiredNamenodeRegistrations(GetNamenodeRegistrationsRequest request)
+ throws IOException;
+
+ /**
+ * Retrieves a list of registered nameservices and their associated info.
+ *
+ * @param request
+ * @return Collection of information for each registered nameservice.
+ * @throws IOException if the data store could not be queried or the query is
+ * invalid.
+ */
+ public abstract GetNamespaceInfoResponse getNamespaceInfo(
+ GetNamespaceInfoRequest request) throws IOException;
+
+ /**
+ * Overrides a cached namenode state with an updated state.
+ *
+ * @param request Fully populated OverrideNamenodeRegistrationRequest request.
+ * @return OverrideNamenodeRegistrationResponse
+ * @throws StateStoreUnavailableException if the data store is not
+ * initialized.
+ * @throws IOException if the data store could not be queried or the query is
+ * invalid.
+ */
+ public abstract UpdateNamenodeRegistrationResponse updateNamenodeRegistration(
+ UpdateNamenodeRegistrationRequest request) throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreCache.java
new file mode 100644
index 0000000..83fc501
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreCache.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import java.io.IOException;
+
+/**
+ * Interface for a cached copy of the State Store.
+ */
+public interface StateStoreCache {
+
+ /**
+ * Load the cache from the State Store. Called by the cache update service
+ * when the data has been reloaded.
+ *
+ * @param force If we force the load.
+ * @return If the cache was loaded successfully.
+ * @throws IOException If there was an error loading the cache.
+ */
+ boolean loadCache(boolean force) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreCacheUpdateService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreCacheUpdateService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreCacheUpdateService.java
new file mode 100644
index 0000000..bb8cfb0
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreCacheUpdateService.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.router.PeriodicService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Service to periodically update the {@link StateStoreService}
+ * cached information in the
+ * {@link org.apache.hadoop.hdfs.server.federation.router.Router Router}.
+ * This is for performance and removes the State Store from the critical path
+ * in common operations.
+ */
+public class StateStoreCacheUpdateService extends PeriodicService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StateStoreCacheUpdateService.class);
+
+ /** The service that manages the State Store connection. */
+ private final StateStoreService stateStore;
+
+
+ /**
+ * Create a new Cache update service.
+ *
+ * @param stateStore Implementation of the state store
+ */
+ public StateStoreCacheUpdateService(StateStoreService stateStore) {
+ super(StateStoreCacheUpdateService.class.getSimpleName());
+ this.stateStore = stateStore;
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+
+ this.setIntervalMs(conf.getLong(
+ DFSConfigKeys.DFS_ROUTER_CACHE_TIME_TO_LIVE_MS,
+ DFSConfigKeys.DFS_ROUTER_CACHE_TIME_TO_LIVE_MS_DEFAULT));
+
+ super.serviceInit(conf);
+ }
+
+ @Override
+ public void periodicInvoke() {
+ LOG.debug("Updating State Store cache");
+ stateStore.refreshCaches();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
index df207e0..73f607f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
@@ -18,17 +18,24 @@
package org.apache.hadoop.hdfs.server.federation.store;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.impl.MembershipStoreImpl;
import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -54,6 +61,9 @@ import com.google.common.annotations.VisibleForTesting;
* federation.
* <li>{@link MountTableStore}: Mount table between to subclusters.
* See {@link org.apache.hadoop.fs.viewfs.ViewFs ViewFs}.
+ * <li>{@link RebalancerStore}: Log of the rebalancing operations.
+ * <li>{@link RouterStore}: Router state in the federation.
+ * <li>{@link TokenStore}: Tokens in the federation.
* </ul>
*/
@InterfaceAudience.Private
@@ -77,8 +87,30 @@ public class StateStoreService extends CompositeService {
private StateStoreConnectionMonitorService monitorService;
+ /** Supported record stores. */
+ private final Map<
+ Class<? extends BaseRecord>, RecordStore<? extends BaseRecord>>
+ recordStores;
+
+ /** Service to maintain State Store caches. */
+ private StateStoreCacheUpdateService cacheUpdater;
+ /** Time the cache was last successfully updated. */
+ private long cacheLastUpdateTime;
+ /** List of internal caches to update. */
+ private final List<StateStoreCache> cachesToUpdateInternal;
+ /** List of external caches to update. */
+ private final List<StateStoreCache> cachesToUpdateExternal;
+
+
public StateStoreService() {
super(StateStoreService.class.getName());
+
+ // Records and stores supported by this implementation
+ this.recordStores = new HashMap<>();
+
+ // Caches to maintain
+ this.cachesToUpdateInternal = new ArrayList<>();
+ this.cachesToUpdateExternal = new ArrayList<>();
}
/**
@@ -102,10 +134,22 @@ public class StateStoreService extends CompositeService {
throw new IOException("Cannot create driver for the State Store");
}
+ // Add supported record stores
+ addRecordStore(MembershipStoreImpl.class);
+
// Check the connection to the State Store periodically
this.monitorService = new StateStoreConnectionMonitorService(this);
this.addService(monitorService);
+ // Set expirations intervals for each record
+ MembershipState.setExpirationMs(conf.getLong(
+ DFSConfigKeys.FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS,
+ DFSConfigKeys.FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS_DEFAULT));
+
+ // Cache update service
+ this.cacheUpdater = new StateStoreCacheUpdateService(this);
+ addService(this.cacheUpdater);
+
super.serviceInit(this.conf);
}
@@ -123,13 +167,56 @@ public class StateStoreService extends CompositeService {
}
/**
+ * Add a record store to the State Store. It includes adding the store, the
+ * supported record and the cache management.
+ *
+ * @param clazz Class of the record store to track.
+ * @return New record store.
+ * @throws ReflectiveOperationException
+ */
+ private <T extends RecordStore<?>> void addRecordStore(
+ final Class<T> clazz) throws ReflectiveOperationException {
+
+ assert this.getServiceState() == STATE.INITED :
+ "Cannot add record to the State Store once started";
+
+ T recordStore = RecordStore.newInstance(clazz, this.getDriver());
+ Class<? extends BaseRecord> recordClass = recordStore.getRecordClass();
+ this.recordStores.put(recordClass, recordStore);
+
+ // Subscribe for cache updates
+ if (recordStore instanceof StateStoreCache) {
+ StateStoreCache cachedRecordStore = (StateStoreCache) recordStore;
+ this.cachesToUpdateInternal.add(cachedRecordStore);
+ }
+ }
+
+ /**
+ * Get the record store in this State Store for a given interface.
+ *
+ * @param recordStoreClass Class of the record store.
+ * @return Registered record store or null if not found.
+ */
+ public <T extends RecordStore<?>> T getRegisteredRecordStore(
+ final Class<T> recordStoreClass) {
+ for (RecordStore<? extends BaseRecord> recordStore :
+ this.recordStores.values()) {
+ if (recordStoreClass.isInstance(recordStore)) {
+ @SuppressWarnings("unchecked")
+ T recordStoreChecked = (T) recordStore;
+ return recordStoreChecked;
+ }
+ }
+ return null;
+ }
+
+ /**
* List of records supported by this State Store.
*
* @return List of supported record classes.
*/
public Collection<Class<? extends BaseRecord>> getSupportedRecords() {
- // TODO add list of records
- return new LinkedList<>();
+ return this.recordStores.keySet();
}
/**
@@ -142,6 +229,7 @@ public class StateStoreService extends CompositeService {
if (this.driver.init(conf, getIdentifier(), getSupportedRecords())) {
LOG.info("Connection to the State Store driver {} is open and ready",
driverName);
+ this.refreshCaches();
} else {
LOG.error("Cannot initialize State Store driver {}", driverName);
}
@@ -198,4 +286,114 @@ public class StateStoreService extends CompositeService {
this.identifier = id;
}
+ //
+ // Cached state store data
+ //
+ /**
+ * The last time the state store cache was fully updated.
+ *
+ * @return Timestamp.
+ */
+ public long getCacheUpdateTime() {
+ return this.cacheLastUpdateTime;
+ }
+
+ /**
+ * Stops the cache update service.
+ */
+ @VisibleForTesting
+ public void stopCacheUpdateService() {
+ if (this.cacheUpdater != null) {
+ this.cacheUpdater.stop();
+ removeService(this.cacheUpdater);
+ this.cacheUpdater = null;
+ }
+ }
+
+ /**
+ * Register a cached record store for automatic periodic cache updates.
+ *
+ * @param client Client to the state store.
+ */
+ public void registerCacheExternal(StateStoreCache client) {
+ this.cachesToUpdateExternal.add(client);
+ }
+
+ /**
+ * Refresh the cache with information from the State Store. Called
+ * periodically by the CacheUpdateService to maintain data caches and
+ * versions.
+ */
+ public void refreshCaches() {
+ refreshCaches(false);
+ }
+
+ /**
+ * Refresh the cache with information from the State Store. Called
+ * periodically by the CacheUpdateService to maintain data caches and
+ * versions.
+ * @param force If we force the refresh.
+ */
+ public void refreshCaches(boolean force) {
+ boolean success = true;
+ if (isDriverReady()) {
+ List<StateStoreCache> cachesToUpdate = new LinkedList<>();
+ cachesToUpdate.addAll(cachesToUpdateInternal);
+ cachesToUpdate.addAll(cachesToUpdateExternal);
+ for (StateStoreCache cachedStore : cachesToUpdate) {
+ String cacheName = cachedStore.getClass().getSimpleName();
+ boolean result = false;
+ try {
+ result = cachedStore.loadCache(force);
+ } catch (IOException e) {
+ LOG.error("Error updating cache for {}", cacheName, e);
+ result = false;
+ }
+ if (!result) {
+ success = false;
+ LOG.error("Cache update failed for cache {}", cacheName);
+ }
+ }
+ } else {
+ success = false;
+ LOG.info("Skipping State Store cache update, driver is not ready.");
+ }
+ if (success) {
+ // Uses local time, not driver time.
+ this.cacheLastUpdateTime = Time.now();
+ }
+ }
+
+ /**
+ * Update the cache for a specific record store.
+ *
+ * @param clazz Class of the record store.
+ * @return If the cached was loaded.
+ * @throws IOException if the cache update failed.
+ */
+ public boolean loadCache(final Class<?> clazz) throws IOException {
+ return loadCache(clazz, false);
+ }
+
+ /**
+ * Update the cache for a specific record store.
+ *
+ * @param clazz Class of the record store.
+ * @param force Force the update ignoring cached periods.
+ * @return If the cached was loaded.
+ * @throws IOException if the cache update failed.
+ */
+ public boolean loadCache(Class<?> clazz, boolean force) throws IOException {
+ List<StateStoreCache> cachesToUpdate =
+ new LinkedList<StateStoreCache>();
+ cachesToUpdate.addAll(this.cachesToUpdateInternal);
+ cachesToUpdate.addAll(this.cachesToUpdateExternal);
+ for (StateStoreCache cachedStore : cachesToUpdate) {
+ if (clazz.isInstance(cachedStore)) {
+ return cachedStore.loadCache(force);
+ }
+ }
+ throw new IOException("Registered cache was not found for " + clazz);
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MembershipStoreImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MembershipStoreImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MembershipStoreImpl.java
new file mode 100644
index 0000000..c28131f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MembershipStoreImpl.java
@@ -0,0 +1,311 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.impl;
+
+import static org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils.filterMultiple;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
+import org.apache.hadoop.hdfs.server.federation.store.MembershipStore;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreCache;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateNamenodeRegistrationRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateNamenodeRegistrationResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.hdfs.server.federation.store.records.Query;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Implementation of the {@link MembershipStore} State Store API.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class MembershipStoreImpl
+ extends MembershipStore implements StateStoreCache {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(MembershipStoreImpl.class);
+
+
+ /** Reported namespaces that are not decommissioned. */
+ private final Set<FederationNamespaceInfo> activeNamespaces;
+
+ /** Namenodes (after evaluating the quorum) that are active in the cluster. */
+ private final Map<String, MembershipState> activeRegistrations;
+ /** Namenode status reports (raw) that were discarded for being too old. */
+ private final Map<String, MembershipState> expiredRegistrations;
+
+ /** Lock to access the local memory cache. */
+ private final ReadWriteLock cacheReadWriteLock =
+ new ReentrantReadWriteLock();
+ private final Lock cacheReadLock = cacheReadWriteLock.readLock();
+ private final Lock cacheWriteLock = cacheReadWriteLock.writeLock();
+
+
+ public MembershipStoreImpl(StateStoreDriver driver) {
+ super(driver);
+
+ this.activeRegistrations = new HashMap<>();
+ this.expiredRegistrations = new HashMap<>();
+ this.activeNamespaces = new TreeSet<>();
+ }
+
+ @Override
+ public GetNamenodeRegistrationsResponse getExpiredNamenodeRegistrations(
+ GetNamenodeRegistrationsRequest request) throws IOException {
+
+ GetNamenodeRegistrationsResponse response =
+ GetNamenodeRegistrationsResponse.newInstance();
+ cacheReadLock.lock();
+ try {
+ Collection<MembershipState> vals = this.expiredRegistrations.values();
+ List<MembershipState> copyVals = new ArrayList<>(vals);
+ response.setNamenodeMemberships(copyVals);
+ } finally {
+ cacheReadLock.unlock();
+ }
+ return response;
+ }
+
+ @Override
+ public GetNamespaceInfoResponse getNamespaceInfo(
+ GetNamespaceInfoRequest request) throws IOException {
+
+ Set<FederationNamespaceInfo> namespaces = new HashSet<>();
+ try {
+ cacheReadLock.lock();
+ namespaces.addAll(activeNamespaces);
+ } finally {
+ cacheReadLock.unlock();
+ }
+
+ GetNamespaceInfoResponse response =
+ GetNamespaceInfoResponse.newInstance(namespaces);
+ return response;
+ }
+
+ @Override
+ public GetNamenodeRegistrationsResponse getNamenodeRegistrations(
+ final GetNamenodeRegistrationsRequest request) throws IOException {
+
+ // TODO Cache some common queries and sorts
+ List<MembershipState> ret = null;
+
+ cacheReadLock.lock();
+ try {
+ Collection<MembershipState> registrations = activeRegistrations.values();
+ MembershipState partialMembership = request.getPartialMembership();
+ if (partialMembership == null) {
+ ret = new ArrayList<>(registrations);
+ } else {
+ Query<MembershipState> query = new Query<>(partialMembership);
+ ret = filterMultiple(query, registrations);
+ }
+ } finally {
+ cacheReadLock.unlock();
+ }
+ // Sort in ascending update date order
+ Collections.sort(ret);
+
+ GetNamenodeRegistrationsResponse response =
+ GetNamenodeRegistrationsResponse.newInstance(ret);
+ return response;
+ }
+
+ @Override
+ public NamenodeHeartbeatResponse namenodeHeartbeat(
+ NamenodeHeartbeatRequest request) throws IOException {
+
+ MembershipState record = request.getNamenodeMembership();
+ String nnId = record.getNamenodeKey();
+ MembershipState existingEntry = null;
+ cacheReadLock.lock();
+ try {
+ existingEntry = this.activeRegistrations.get(nnId);
+ } finally {
+ cacheReadLock.unlock();
+ }
+
+ if (existingEntry != null) {
+ if (existingEntry.getState() != record.getState()) {
+ LOG.info("NN registration state has changed: {} -> {}",
+ existingEntry, record);
+ } else {
+ LOG.debug("Updating NN registration: {} -> {}", existingEntry, record);
+ }
+ } else {
+ LOG.info("Inserting new NN registration: {}", record);
+ }
+
+ boolean status = getDriver().put(record, true, false);
+
+ NamenodeHeartbeatResponse response =
+ NamenodeHeartbeatResponse.newInstance(status);
+ return response;
+ }
+
+ @Override
+ public boolean loadCache(boolean force) throws IOException {
+ super.loadCache(force);
+
+ // Update local cache atomically
+ cacheWriteLock.lock();
+ try {
+ this.activeRegistrations.clear();
+ this.expiredRegistrations.clear();
+ this.activeNamespaces.clear();
+
+ // Build list of NN registrations: nnId -> registration list
+ Map<String, List<MembershipState>> nnRegistrations = new HashMap<>();
+ List<MembershipState> cachedRecords = getCachedRecords();
+ for (MembershipState membership : cachedRecords) {
+ String nnId = membership.getNamenodeKey();
+ if (membership.getState() == FederationNamenodeServiceState.EXPIRED) {
+ // Expired, RPC service does not use these
+ String key = membership.getPrimaryKey();
+ this.expiredRegistrations.put(key, membership);
+ } else {
+ // This is a valid NN registration, build a list of all registrations
+ // using the NN id to use for the quorum calculation.
+ List<MembershipState> nnRegistration =
+ nnRegistrations.get(nnId);
+ if (nnRegistration == null) {
+ nnRegistration = new LinkedList<>();
+ nnRegistrations.put(nnId, nnRegistration);
+ }
+ nnRegistration.add(membership);
+ String bpId = membership.getBlockPoolId();
+ String cId = membership.getClusterId();
+ String nsId = membership.getNameserviceId();
+ FederationNamespaceInfo nsInfo =
+ new FederationNamespaceInfo(bpId, cId, nsId);
+ this.activeNamespaces.add(nsInfo);
+ }
+ }
+
+ // Calculate most representative entry for each active NN id
+ for (List<MembershipState> nnRegistration : nnRegistrations.values()) {
+ // Run quorum based on NN state
+ MembershipState representativeRecord =
+ getRepresentativeQuorum(nnRegistration);
+ String nnKey = representativeRecord.getNamenodeKey();
+ this.activeRegistrations.put(nnKey, representativeRecord);
+ }
+ LOG.debug("Refreshed {} NN registrations from State Store",
+ cachedRecords.size());
+ } finally {
+ cacheWriteLock.unlock();
+ }
+ return true;
+ }
+
+ @Override
+ public UpdateNamenodeRegistrationResponse updateNamenodeRegistration(
+ UpdateNamenodeRegistrationRequest request) throws IOException {
+
+ boolean status = false;
+ cacheWriteLock.lock();
+ try {
+ String namenode = MembershipState.getNamenodeKey(
+ request.getNameserviceId(), request.getNamenodeId());
+ MembershipState member = this.activeRegistrations.get(namenode);
+ if (member != null) {
+ member.setState(request.getState());
+ status = true;
+ }
+ } finally {
+ cacheWriteLock.unlock();
+ }
+ UpdateNamenodeRegistrationResponse response =
+ UpdateNamenodeRegistrationResponse.newInstance(status);
+ return response;
+ }
+
+ /**
+ * Picks the most recent entry in the subset that is most agreeable on the
+ * specified field. 1) If a majority of the collection has the same value for
+ * the field, the first sorted entry within the subset the matches the
+ * majority value 2) Otherwise the first sorted entry in the set of all
+ * entries
+ *
+ * @param entries - Collection of state store record objects of the same type
+ * @param fieldName - Field name for the value to compare
+ * @return record that is most representative of the field name
+ */
+ private MembershipState getRepresentativeQuorum(
+ Collection<MembershipState> records) {
+
+ // Collate objects by field value: field value -> order set of records
+ Map<FederationNamenodeServiceState, TreeSet<MembershipState>> occurenceMap =
+ new HashMap<>();
+ for (MembershipState record : records) {
+ FederationNamenodeServiceState state = record.getState();
+ TreeSet<MembershipState> matchingSet = occurenceMap.get(state);
+ if (matchingSet == null) {
+ // TreeSet orders elements by descending date via comparators
+ matchingSet = new TreeSet<>();
+ occurenceMap.put(state, matchingSet);
+ }
+ matchingSet.add(record);
+ }
+
+ // Select largest group
+ TreeSet<MembershipState> largestSet = new TreeSet<>();
+ for (TreeSet<MembershipState> matchingSet : occurenceMap.values()) {
+ if (largestSet.size() < matchingSet.size()) {
+ largestSet = matchingSet;
+ }
+ }
+
+ // If quorum, use the newest element here
+ if (largestSet.size() > records.size() / 2) {
+ return largestSet.first();
+ // Otherwise, return most recent by class comparator
+ } else if (records.size() > 0) {
+ TreeSet<MembershipState> sortedList = new TreeSet<>(records);
+ LOG.debug("Quorum failed, using most recent: {}", sortedList.first());
+ return sortedList.first();
+ } else {
+ return null;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/package-info.java
new file mode 100644
index 0000000..1a50d15
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/package-info.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Contains implementations of the state store API interfaces. All classes
+ * derive from {@link
+ * org.apache.hadoop.hdfs.server.federation.store.RecordStore}. The API
+ * definitions are contained in the
+ * org.apache.hadoop.hdfs.server.federation.store package.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+package org.apache.hadoop.hdfs.server.federation.store.impl;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamenodeRegistrationsRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamenodeRegistrationsRequest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamenodeRegistrationsRequest.java
new file mode 100644
index 0000000..568feaf
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamenodeRegistrationsRequest.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+
+/**
+ * API request for listing namenode registrations present in the state store.
+ */
+public abstract class GetNamenodeRegistrationsRequest {
+
+ public static GetNamenodeRegistrationsRequest newInstance()
+ throws IOException {
+ return StateStoreSerializer.newRecord(
+ GetNamenodeRegistrationsRequest.class);
+ }
+
+ public static GetNamenodeRegistrationsRequest newInstance(
+ MembershipState member) throws IOException {
+ GetNamenodeRegistrationsRequest request = newInstance();
+ request.setPartialMembership(member);
+ return request;
+ }
+
+ @Public
+ @Unstable
+ public abstract MembershipState getPartialMembership();
+
+ @Public
+ @Unstable
+ public abstract void setPartialMembership(MembershipState member);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamenodeRegistrationsResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamenodeRegistrationsResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamenodeRegistrationsResponse.java
new file mode 100644
index 0000000..0d60c90
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamenodeRegistrationsResponse.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+
+/**
+ * API response for listing namenode registrations present in the state store.
+ */
+public abstract class GetNamenodeRegistrationsResponse {
+
+ public static GetNamenodeRegistrationsResponse newInstance()
+ throws IOException {
+ return StateStoreSerializer.newRecord(
+ GetNamenodeRegistrationsResponse.class);
+ }
+
+ public static GetNamenodeRegistrationsResponse newInstance(
+ List<MembershipState> records) throws IOException {
+ GetNamenodeRegistrationsResponse response = newInstance();
+ response.setNamenodeMemberships(records);
+ return response;
+ }
+
+ @Public
+ @Unstable
+ public abstract List<MembershipState> getNamenodeMemberships()
+ throws IOException;
+
+ @Public
+ @Unstable
+ public abstract void setNamenodeMemberships(
+ List<MembershipState> records) throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamespaceInfoRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamespaceInfoRequest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamespaceInfoRequest.java
new file mode 100644
index 0000000..b5cc01b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamespaceInfoRequest.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API response for listing HDFS namespaces present in the state store.
+ */
+public abstract class GetNamespaceInfoRequest {
+
+ public static GetNamespaceInfoRequest newInstance() {
+ return StateStoreSerializer.newRecord(GetNamespaceInfoRequest.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamespaceInfoResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamespaceInfoResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamespaceInfoResponse.java
new file mode 100644
index 0000000..f541453
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamespaceInfoResponse.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API response for listing HDFS namespaces present in the state store.
+ */
+public abstract class GetNamespaceInfoResponse {
+
+ public static GetNamespaceInfoResponse newInstance() {
+ return StateStoreSerializer.newRecord(GetNamespaceInfoResponse.class);
+ }
+
+ public static GetNamespaceInfoResponse newInstance(
+ Set<FederationNamespaceInfo> namespaces) throws IOException {
+ GetNamespaceInfoResponse response = newInstance();
+ response.setNamespaceInfo(namespaces);
+ return response;
+ }
+
+ @Public
+ @Unstable
+ public abstract Set<FederationNamespaceInfo> getNamespaceInfo();
+
+ @Public
+ @Unstable
+ public abstract void setNamespaceInfo(
+ Set<FederationNamespaceInfo> namespaceInfo);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/NamenodeHeartbeatRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/NamenodeHeartbeatRequest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/NamenodeHeartbeatRequest.java
new file mode 100644
index 0000000..9506026
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/NamenodeHeartbeatRequest.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+
+/**
+ * API request for registering a namenode with the state store.
+ */
+public abstract class NamenodeHeartbeatRequest {
+
+ public static NamenodeHeartbeatRequest newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(NamenodeHeartbeatRequest.class);
+ }
+
+ public static NamenodeHeartbeatRequest newInstance(MembershipState namenode)
+ throws IOException {
+ NamenodeHeartbeatRequest request = newInstance();
+ request.setNamenodeMembership(namenode);
+ return request;
+ }
+
+ @Private
+ @Unstable
+ public abstract MembershipState getNamenodeMembership()
+ throws IOException;
+
+ @Private
+ @Unstable
+ public abstract void setNamenodeMembership(MembershipState report)
+ throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/NamenodeHeartbeatResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/NamenodeHeartbeatResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/NamenodeHeartbeatResponse.java
new file mode 100644
index 0000000..acb7a6f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/NamenodeHeartbeatResponse.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API response for registering a namenode with the state store.
+ */
+public abstract class NamenodeHeartbeatResponse {
+
+ public static NamenodeHeartbeatResponse newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(NamenodeHeartbeatResponse.class);
+ }
+
+ public static NamenodeHeartbeatResponse newInstance(boolean status)
+ throws IOException {
+ NamenodeHeartbeatResponse response = newInstance();
+ response.setResult(status);
+ return response;
+ }
+
+ @Private
+ @Unstable
+ public abstract boolean getResult();
+
+ @Private
+ @Unstable
+ public abstract void setResult(boolean result);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateNamenodeRegistrationRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateNamenodeRegistrationRequest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateNamenodeRegistrationRequest.java
new file mode 100644
index 0000000..4459e33
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateNamenodeRegistrationRequest.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API request for overriding an existing namenode registration in the state
+ * store.
+ */
+public abstract class UpdateNamenodeRegistrationRequest {
+
+ public static UpdateNamenodeRegistrationRequest newInstance()
+ throws IOException {
+ return StateStoreSerializer.newRecord(
+ UpdateNamenodeRegistrationRequest.class);
+ }
+
+ public static UpdateNamenodeRegistrationRequest newInstance(
+ String nameserviceId, String namenodeId,
+ FederationNamenodeServiceState state) throws IOException {
+ UpdateNamenodeRegistrationRequest request = newInstance();
+ request.setNameserviceId(nameserviceId);
+ request.setNamenodeId(namenodeId);
+ request.setState(state);
+ return request;
+ }
+
+ @Private
+ @Unstable
+ public abstract String getNameserviceId();
+
+ @Private
+ @Unstable
+ public abstract String getNamenodeId();
+
+ @Private
+ @Unstable
+ public abstract FederationNamenodeServiceState getState();
+
+ @Private
+ @Unstable
+ public abstract void setNameserviceId(String nsId);
+
+ @Private
+ @Unstable
+ public abstract void setNamenodeId(String nnId);
+
+ @Private
+ @Unstable
+ public abstract void setState(FederationNamenodeServiceState state);
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/40] hadoop git commit: Revert "Plan/ResourceAllocation data
structure enhancements required to support recurring reservations in
ReservationSystem."
Posted by in...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3345e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java
index 03569d4..bc98e2f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java
@@ -40,7 +40,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.planning.Planne
import org.apache.hadoop.yarn.server.resourcemanager.reservation.planning.ReservationAgent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.util.Clock;
-import org.apache.hadoop.yarn.util.UTCClock;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.junit.After;
@@ -48,10 +47,6 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-/**
- * Testing the class {@link InMemoryPlan}.
- */
-@SuppressWarnings("checkstyle:nowhitespaceafter")
public class TestInMemoryPlan {
private String user = "yarn";
@@ -67,7 +62,6 @@ public class TestInMemoryPlan {
private ReservationAgent agent;
private Planner replanner;
private RMContext context;
- private long maxPeriodicity;
@Before
public void setUp() throws PlanningException {
@@ -78,7 +72,7 @@ public class TestInMemoryPlan {
clock = mock(Clock.class);
queueMetrics = mock(QueueMetrics.class);
- policy = new NoOverCommitPolicy();
+ policy = mock(SharingPolicy.class);
replanner = mock(Planner.class);
when(clock.getTime()).thenReturn(1L);
@@ -101,41 +95,15 @@ public class TestInMemoryPlan {
@Test
public void testAddReservation() {
- Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ Plan plan =
+ new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
ReservationId reservationID =
- ReservationSystemTestUtil.getNewReservationId();
+ ReservationSystemTestUtil.getNewReservationId();
int[] alloc = { 10, 10, 10, 10, 10, 10 };
int start = 100;
- ReservationAllocation rAllocation =
- createReservationAllocation(reservationID, start, alloc);
- Assert.assertNull(plan.getReservationById(reservationID));
- try {
- plan.addReservation(rAllocation, false);
- } catch (PlanningException e) {
- Assert.fail(e.getMessage());
- }
- doAssertions(plan, rAllocation);
- checkAllocation(plan, alloc, start, 0);
- }
-
- @Test
- public void testAddPeriodicReservation() throws PlanningException {
-
- maxPeriodicity = 100;
- Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, maxPeriodicity,
- context, new UTCClock());
-
- ReservationId reservationID =
- ReservationSystemTestUtil.getNewReservationId();
- int[] alloc = { 10, 50 };
- int start = 10;
- long period = 20;
- ReservationAllocation rAllocation = createReservationAllocation(
- reservationID, start, alloc, String.valueOf(period));
- // use periodicity of 1hr
- rAllocation.setPeriodicity(period);
+ ReservationAllocation rAllocation = createReservationAllocation
+ (reservationID, start, alloc);
Assert.assertNull(plan.getReservationById(reservationID));
try {
plan.addReservation(rAllocation, false);
@@ -143,54 +111,32 @@ public class TestInMemoryPlan {
Assert.fail(e.getMessage());
}
doAssertions(plan, rAllocation);
- checkAllocation(plan, alloc, start, period);
-
- RLESparseResourceAllocation available =
- plan.getAvailableResourceOverTime(user, reservationID, 150, 330, 50);
- System.out.println(available);
+ checkAllocation(plan, alloc, start);
}
- private void checkAllocation(Plan plan, int[] alloc, int start,
- long periodicity) {
- long end = start + alloc.length;
- if (periodicity > 0) {
- end = end + maxPeriodicity;
- }
+ private void checkAllocation(Plan plan, int[] alloc, int start) {
RLESparseResourceAllocation userCons =
- plan.getConsumptionForUserOverTime(user, start, end * 3);
+ plan.getConsumptionForUserOverTime(user, start, start + alloc.length);
for (int i = 0; i < alloc.length; i++) {
- // only one instance for non-periodic reservation
- if (periodicity <= 0) {
- Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])),
- plan.getTotalCommittedResources(start + i));
- Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])),
- userCons.getCapacityAtTime(start + i));
- } else {
- // periodic reservations should repeat
- long y = 0;
- Resource res = Resource.newInstance(1024 * (alloc[i]), (alloc[i]));
- while (y <= end * 2) {
- Assert.assertEquals("At time: " + start + i + y, res,
- plan.getTotalCommittedResources(start + i + y));
- Assert.assertEquals(" At time: " + (start + i + y), res,
- userCons.getCapacityAtTime(start + i + y));
- y = y + periodicity;
- }
- }
+ Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])),
+ plan.getTotalCommittedResources(start + i));
+ Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])),
+ userCons.getCapacityAtTime(start + i));
}
}
@Test
public void testAddEmptyReservation() {
- Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ Plan plan =
+ new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
ReservationId reservationID =
- ReservationSystemTestUtil.getNewReservationId();
+ ReservationSystemTestUtil.getNewReservationId();
int[] alloc = {};
int start = 100;
- ReservationAllocation rAllocation =
- createReservationAllocation(reservationID, start, alloc);
+ ReservationAllocation rAllocation = createReservationAllocation
+ (reservationID, start, alloc);
Assert.assertNull(plan.getReservationById(reservationID));
try {
plan.addReservation(rAllocation, false);
@@ -202,14 +148,15 @@ public class TestInMemoryPlan {
@Test
public void testAddReservationAlreadyExists() {
// First add a reservation
- Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ Plan plan =
+ new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
ReservationId reservationID =
- ReservationSystemTestUtil.getNewReservationId();
+ ReservationSystemTestUtil.getNewReservationId();
int[] alloc = { 10, 10, 10, 10, 10, 10 };
int start = 100;
- ReservationAllocation rAllocation =
- createReservationAllocation(reservationID, start, alloc);
+ ReservationAllocation rAllocation = createReservationAllocation
+ (reservationID, start, alloc);
Assert.assertNull(plan.getReservationById(reservationID));
try {
plan.addReservation(rAllocation, false);
@@ -217,7 +164,7 @@ public class TestInMemoryPlan {
Assert.fail(e.getMessage());
}
doAssertions(plan, rAllocation);
- checkAllocation(plan, alloc, start, 0);
+ checkAllocation(plan, alloc, start);
// Try to add it again
try {
@@ -233,15 +180,16 @@ public class TestInMemoryPlan {
@Test
public void testUpdateReservation() {
- Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ Plan plan =
+ new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
ReservationId reservationID =
ReservationSystemTestUtil.getNewReservationId();
// First add a reservation
int[] alloc = { 10, 10, 10, 10, 10, 10 };
int start = 100;
- ReservationAllocation rAllocation =
- createReservationAllocation(reservationID, start, alloc);
+ ReservationAllocation rAllocation = createReservationAllocation
+ (reservationID, start, alloc);
Assert.assertNull(plan.getReservationById(reservationID));
try {
plan.addReservation(rAllocation, false);
@@ -262,8 +210,8 @@ public class TestInMemoryPlan {
// Now update it
start = 110;
int[] updatedAlloc = { 0, 5, 10, 10, 5, 0 };
- rAllocation =
- createReservationAllocation(reservationID, start, updatedAlloc, true);
+ rAllocation = createReservationAllocation(reservationID, start,
+ updatedAlloc, true);
try {
plan.updateReservation(rAllocation);
} catch (PlanningException e) {
@@ -271,71 +219,32 @@ public class TestInMemoryPlan {
}
doAssertions(plan, rAllocation);
- userCons = plan.getConsumptionForUserOverTime(user, start,
- start + updatedAlloc.length);
+ userCons =
+ plan.getConsumptionForUserOverTime(user, start, start
+ + updatedAlloc.length);
for (int i = 0; i < updatedAlloc.length; i++) {
- Assert.assertEquals(Resource.newInstance(1024 * (updatedAlloc[i] + i),
- updatedAlloc[i] + i), plan.getTotalCommittedResources(start + i));
- Assert.assertEquals(Resource.newInstance(1024 * (updatedAlloc[i] + i),
- updatedAlloc[i] + i), userCons.getCapacityAtTime(start + i));
- }
- }
-
- @Test
- public void testUpdatePeriodicReservation() {
- Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
- // First add a reservation
- ReservationId reservationID =
- ReservationSystemTestUtil.getNewReservationId();
- int[] alloc = { 10, 20 };
- int start = 100;
- ReservationAllocation rAllocation =
- createReservationAllocation(reservationID, start, alloc);
- // use periodicity of 1hr
- long period = 3600000;
- rAllocation.getReservationDefinition()
- .setRecurrenceExpression(String.valueOf(period));
- rAllocation.setPeriodicity(period);
- Assert.assertNull(plan.getReservationById(reservationID));
- try {
- plan.addReservation(rAllocation, false);
- } catch (PlanningException e) {
- Assert.fail(e.getMessage());
- }
- System.out.println(plan.toString());
- doAssertions(plan, rAllocation);
- checkAllocation(plan, alloc, start, period);
-
- // Now update it
- start = 110;
- int[] updatedAlloc = { 30, 40, 50 };
- rAllocation =
- createReservationAllocation(reservationID, start, updatedAlloc);
- rAllocation.getReservationDefinition()
- .setRecurrenceExpression(String.valueOf(period));
- rAllocation.setPeriodicity(period);
- try {
- plan.updateReservation(rAllocation);
- } catch (PlanningException e) {
- Assert.fail(e.getMessage());
+ Assert.assertEquals(
+ Resource.newInstance(1024 * (updatedAlloc[i] + i), updatedAlloc[i]
+ + i), plan.getTotalCommittedResources(start + i));
+ Assert.assertEquals(
+ Resource.newInstance(1024 * (updatedAlloc[i] + i), updatedAlloc[i]
+ + i), userCons.getCapacityAtTime(start + i));
}
- doAssertions(plan, rAllocation);
- checkAllocation(plan, updatedAlloc, start, period);
}
@Test
public void testUpdateNonExistingReservation() {
- Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ Plan plan =
+ new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
ReservationId reservationID =
ReservationSystemTestUtil.getNewReservationId();
// Try to update a reservation without adding
int[] alloc = { 10, 10, 10, 10, 10, 10 };
int start = 100;
ReservationAllocation rAllocation =
- createReservationAllocation(reservationID, start, alloc);
+ createReservationAllocation(reservationID, start, alloc);
Assert.assertNull(plan.getReservationById(reservationID));
try {
plan.updateReservation(rAllocation);
@@ -351,14 +260,15 @@ public class TestInMemoryPlan {
@Test
public void testDeleteReservation() {
// First add a reservation
- Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ Plan plan =
+ new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
ReservationId reservationID =
ReservationSystemTestUtil.getNewReservationId();
int[] alloc = { 10, 10, 10, 10, 10, 10 };
int start = 100;
ReservationAllocation rAllocation =
- createReservationAllocation(reservationID, start, alloc, true);
+ createReservationAllocation(reservationID, start, alloc, true);
Assert.assertNull(plan.getReservationById(reservationID));
try {
plan.addReservation(rAllocation, false);
@@ -397,46 +307,10 @@ public class TestInMemoryPlan {
}
@Test
- public void testDeletePeriodicReservation() {
- Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
- // First add a reservation
- ReservationId reservationID =
- ReservationSystemTestUtil.getNewReservationId();
- int[] alloc = { 10, 20 };
- int start = 100;
- ReservationAllocation rAllocation =
- createReservationAllocation(reservationID, start, alloc);
- // use periodicity of 1hr
- long period = 3600000;
- rAllocation.getReservationDefinition()
- .setRecurrenceExpression(String.valueOf(period));
- rAllocation.setPeriodicity(period);
- Assert.assertNull(plan.getReservationById(reservationID));
- try {
- plan.addReservation(rAllocation, false);
- } catch (PlanningException e) {
- Assert.fail(e.getMessage());
- }
- System.out.println(plan.toString());
- doAssertions(plan, rAllocation);
- checkAllocation(plan, alloc, start, period);
-
- // Now delete it
- try {
- plan.deleteReservation(reservationID);
- } catch (PlanningException e) {
- Assert.fail(e.getMessage());
- }
- Assert.assertNull(plan.getReservationById(reservationID));
- System.out.print(plan);
- checkAllocation(plan, new int[] { 0, 0 }, start, period);
- }
-
- @Test
public void testDeleteNonExistingReservation() {
- Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ Plan plan =
+ new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
ReservationId reservationID =
ReservationSystemTestUtil.getNewReservationId();
// Try to delete a reservation without adding
@@ -454,9 +328,8 @@ public class TestInMemoryPlan {
@Test
public void testArchiveCompletedReservations() {
- SharingPolicy sharingPolicy = mock(SharingPolicy.class);
Plan plan =
- new InMemoryPlan(queueMetrics, sharingPolicy, agent, totalCapacity, 1L,
+ new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
ReservationId reservationID1 =
ReservationSystemTestUtil.getNewReservationId();
@@ -464,7 +337,7 @@ public class TestInMemoryPlan {
int[] alloc1 = { 10, 10, 10, 10, 10, 10 };
int start = 100;
ReservationAllocation rAllocation =
- createReservationAllocation(reservationID1, start, alloc1);
+ createReservationAllocation(reservationID1, start, alloc1);
Assert.assertNull(plan.getReservationById(reservationID1));
try {
plan.addReservation(rAllocation, false);
@@ -472,14 +345,15 @@ public class TestInMemoryPlan {
Assert.fail(e.getMessage());
}
doAssertions(plan, rAllocation);
- checkAllocation(plan, alloc1, start, 0);
+ checkAllocation(plan, alloc1, start);
+
// Now add another one
ReservationId reservationID2 =
ReservationSystemTestUtil.getNewReservationId();
int[] alloc2 = { 0, 5, 10, 5, 0 };
rAllocation =
- createReservationAllocation(reservationID2, start, alloc2, true);
+ createReservationAllocation(reservationID2, start, alloc2, true);
Assert.assertNull(plan.getReservationById(reservationID2));
try {
plan.addReservation(rAllocation, false);
@@ -493,18 +367,16 @@ public class TestInMemoryPlan {
for (int i = 0; i < alloc2.length; i++) {
Assert.assertEquals(
- Resource.newInstance(1024 * (alloc1[i] + alloc2[i] + i),
- alloc1[i] + alloc2[i] + i),
- plan.getTotalCommittedResources(start + i));
+ Resource.newInstance(1024 * (alloc1[i] + alloc2[i] + i), alloc1[i]
+ + alloc2[i] + i), plan.getTotalCommittedResources(start + i));
Assert.assertEquals(
- Resource.newInstance(1024 * (alloc1[i] + alloc2[i] + i),
- alloc1[i] + alloc2[i] + i),
- userCons.getCapacityAtTime(start + i));
+ Resource.newInstance(1024 * (alloc1[i] + alloc2[i] + i), alloc1[i]
+ + alloc2[i] + i), userCons.getCapacityAtTime(start + i));
}
// Now archive completed reservations
when(clock.getTime()).thenReturn(106L);
- when(sharingPolicy.getValidWindow()).thenReturn(1L);
+ when(policy.getValidWindow()).thenReturn(1L);
try {
// will only remove 2nd reservation as only that has fallen out of the
// archival window
@@ -514,7 +386,7 @@ public class TestInMemoryPlan {
}
Assert.assertNotNull(plan.getReservationById(reservationID1));
Assert.assertNull(plan.getReservationById(reservationID2));
- checkAllocation(plan, alloc1, start, 0);
+ checkAllocation(plan, alloc1, start);
when(clock.getTime()).thenReturn(107L);
try {
@@ -539,14 +411,15 @@ public class TestInMemoryPlan {
@Test
public void testGetReservationsById() {
- Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ Plan plan =
+ new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
ReservationId reservationID =
- ReservationSystemTestUtil.getNewReservationId();
- int[] alloc = { 10, 10, 10, 10, 10, 10 };
+ ReservationSystemTestUtil.getNewReservationId();
+ int[] alloc = {10, 10, 10, 10, 10, 10};
int start = 100;
- ReservationAllocation rAllocation =
- createReservationAllocation(reservationID, start, alloc);
+ ReservationAllocation rAllocation = createReservationAllocation
+ (reservationID, start, alloc);
Assert.assertNull(plan.getReservationById(reservationID));
try {
plan.addReservation(rAllocation, false);
@@ -556,30 +429,31 @@ public class TestInMemoryPlan {
// Verify that get by reservation id works.
Set<ReservationAllocation> rAllocations =
- plan.getReservations(reservationID, null, "");
+ plan.getReservations(reservationID, null, "");
Assert.assertTrue(rAllocations.size() == 1);
- Assert.assertTrue(rAllocation
- .compareTo((ReservationAllocation) rAllocations.toArray()[0]) == 0);
+ Assert.assertTrue(rAllocation.compareTo(
+ (ReservationAllocation) rAllocations.toArray()[0]) == 0);
// Verify that get by reservation id works even when time range
// and user is invalid.
ReservationInterval interval = new ReservationInterval(0, 0);
rAllocations = plan.getReservations(reservationID, interval, "invalid");
Assert.assertTrue(rAllocations.size() == 1);
- Assert.assertTrue(rAllocation
- .compareTo((ReservationAllocation) rAllocations.toArray()[0]) == 0);
+ Assert.assertTrue(rAllocation.compareTo(
+ (ReservationAllocation) rAllocations.toArray()[0]) == 0);
}
@Test
public void testGetReservationsByInvalidId() {
- Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ Plan plan =
+ new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
ReservationId reservationID =
- ReservationSystemTestUtil.getNewReservationId();
- int[] alloc = { 10, 10, 10, 10, 10, 10 };
+ ReservationSystemTestUtil.getNewReservationId();
+ int[] alloc = {10, 10, 10, 10, 10, 10};
int start = 100;
- ReservationAllocation rAllocation =
- createReservationAllocation(reservationID, start, alloc);
+ ReservationAllocation rAllocation = createReservationAllocation
+ (reservationID, start, alloc);
Assert.assertNull(plan.getReservationById(reservationID));
try {
plan.addReservation(rAllocation, false);
@@ -589,22 +463,23 @@ public class TestInMemoryPlan {
// If reservationId is null, then nothing is returned.
ReservationId invalidReservationID =
- ReservationSystemTestUtil.getNewReservationId();
+ ReservationSystemTestUtil.getNewReservationId();
Set<ReservationAllocation> rAllocations =
- plan.getReservations(invalidReservationID, null, "");
+ plan.getReservations(invalidReservationID, null, "");
Assert.assertTrue(rAllocations.size() == 0);
}
@Test
public void testGetReservationsByTimeInterval() {
- Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ Plan plan =
+ new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
ReservationId reservationID =
- ReservationSystemTestUtil.getNewReservationId();
- int[] alloc = { 10, 10, 10, 10, 10, 10 };
+ ReservationSystemTestUtil.getNewReservationId();
+ int[] alloc = {10, 10, 10, 10, 10, 10};
int start = 100;
- ReservationAllocation rAllocation =
- createReservationAllocation(reservationID, start, alloc);
+ ReservationAllocation rAllocation = createReservationAllocation
+ (reservationID, start, alloc);
Assert.assertNull(plan.getReservationById(reservationID));
try {
plan.addReservation(rAllocation, false);
@@ -614,24 +489,23 @@ public class TestInMemoryPlan {
// Verify that get by time interval works if the selection interval
// completely overlaps with the allocation.
- ReservationInterval interval = new ReservationInterval(
- rAllocation.getStartTime(), rAllocation.getEndTime());
+ ReservationInterval interval = new ReservationInterval(rAllocation
+ .getStartTime(), rAllocation.getEndTime());
Set<ReservationAllocation> rAllocations =
- plan.getReservations(null, interval, "");
+ plan.getReservations(null, interval, "");
Assert.assertTrue(rAllocations.size() == 1);
- Assert.assertTrue(rAllocation
- .compareTo((ReservationAllocation) rAllocations.toArray()[0]) == 0);
+ Assert.assertTrue(rAllocation.compareTo(
+ (ReservationAllocation) rAllocations.toArray()[0]) == 0);
// Verify that get by time interval works if the selection interval
// falls within the allocation
long duration = rAllocation.getEndTime() - rAllocation.getStartTime();
- interval = new ReservationInterval(
- rAllocation.getStartTime() + duration * (long) 0.3,
- rAllocation.getEndTime() - duration * (long) 0.3);
+ interval = new ReservationInterval(rAllocation.getStartTime() + duration
+ * (long)0.3, rAllocation.getEndTime() - duration * (long)0.3);
rAllocations = plan.getReservations(null, interval, "");
Assert.assertTrue(rAllocations.size() == 1);
- Assert.assertTrue(rAllocation
- .compareTo((ReservationAllocation) rAllocations.toArray()[0]) == 0);
+ Assert.assertTrue(rAllocation.compareTo(
+ (ReservationAllocation) rAllocations.toArray()[0]) == 0);
// Verify that get by time interval selects 1 allocation if the end
// time of the selection interval falls right at the start of the
@@ -639,13 +513,13 @@ public class TestInMemoryPlan {
interval = new ReservationInterval(0, rAllocation.getStartTime());
rAllocations = plan.getReservations(null, interval, "");
Assert.assertTrue(rAllocations.size() == 1);
- Assert.assertTrue(rAllocation
- .compareTo((ReservationAllocation) rAllocations.toArray()[0]) == 0);
+ Assert.assertTrue(rAllocation.compareTo(
+ (ReservationAllocation) rAllocations.toArray()[0]) == 0);
// Verify that get by time interval selects no reservations if the start
// time of the selection interval falls right at the end of the allocation.
- interval =
- new ReservationInterval(rAllocation.getEndTime(), Long.MAX_VALUE);
+ interval = new ReservationInterval(rAllocation
+ .getEndTime(), Long.MAX_VALUE);
rAllocations = plan.getReservations(null, interval, "");
Assert.assertTrue(rAllocations.size() == 0);
@@ -658,14 +532,15 @@ public class TestInMemoryPlan {
@Test
public void testGetReservationsAtTime() {
- Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ Plan plan =
+ new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
ReservationId reservationID =
- ReservationSystemTestUtil.getNewReservationId();
- int[] alloc = { 10, 10, 10, 10, 10, 10 };
+ ReservationSystemTestUtil.getNewReservationId();
+ int[] alloc = {10, 10, 10, 10, 10, 10};
int start = 100;
- ReservationAllocation rAllocation =
- createReservationAllocation(reservationID, start, alloc);
+ ReservationAllocation rAllocation = createReservationAllocation
+ (reservationID, start, alloc);
Assert.assertNull(plan.getReservationById(reservationID));
try {
plan.addReservation(rAllocation, false);
@@ -674,22 +549,23 @@ public class TestInMemoryPlan {
}
Set<ReservationAllocation> rAllocations =
- plan.getReservationsAtTime(rAllocation.getStartTime());
+ plan.getReservationsAtTime(rAllocation.getStartTime());
Assert.assertTrue(rAllocations.size() == 1);
- Assert.assertTrue(rAllocation
- .compareTo((ReservationAllocation) rAllocations.toArray()[0]) == 0);
+ Assert.assertTrue(rAllocation.compareTo(
+ (ReservationAllocation) rAllocations.toArray()[0]) == 0);
}
@Test
public void testGetReservationsWithNoInput() {
- Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ Plan plan =
+ new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
ReservationId reservationID =
- ReservationSystemTestUtil.getNewReservationId();
- int[] alloc = { 10, 10, 10, 10, 10, 10 };
+ ReservationSystemTestUtil.getNewReservationId();
+ int[] alloc = {10, 10, 10, 10, 10, 10};
int start = 100;
- ReservationAllocation rAllocation =
- createReservationAllocation(reservationID, start, alloc);
+ ReservationAllocation rAllocation = createReservationAllocation
+ (reservationID, start, alloc);
Assert.assertNull(plan.getReservationById(reservationID));
try {
plan.addReservation(rAllocation, false);
@@ -700,21 +576,22 @@ public class TestInMemoryPlan {
// Verify that getReservations defaults to getting all reservations if no
// reservationID, time interval, and user is provided,
Set<ReservationAllocation> rAllocations =
- plan.getReservations(null, null, "");
+ plan.getReservations(null, null, "");
Assert.assertTrue(rAllocations.size() == 1);
- Assert.assertTrue(rAllocation
- .compareTo((ReservationAllocation) rAllocations.toArray()[0]) == 0);
+ Assert.assertTrue(rAllocation.compareTo(
+ (ReservationAllocation) rAllocations.toArray()[0]) == 0);
}
@Test
public void testGetReservationsWithNoReservation() {
- Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ Plan plan =
+ new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
// Verify that get reservation returns no entries if no queries are made.
ReservationInterval interval = new ReservationInterval(0, Long.MAX_VALUE);
Set<ReservationAllocation> rAllocations =
- plan.getReservations(null, interval, "");
+ plan.getReservations(null, interval, "");
Assert.assertTrue(rAllocations.size() == 0);
}
@@ -723,9 +600,7 @@ public class TestInMemoryPlan {
Assert.assertNotNull(plan.getReservationById(reservationID));
Assert.assertEquals(rAllocation, plan.getReservationById(reservationID));
Assert.assertTrue(((InMemoryPlan) plan).getAllReservations().size() == 1);
- if (rAllocation.getPeriodicity() <= 0) {
- Assert.assertEquals(rAllocation.getEndTime(), plan.getLastEndTime());
- }
+ Assert.assertEquals(rAllocation.getEndTime(), plan.getLastEndTime());
Assert.assertEquals(totalCapacity, plan.getTotalCapacity());
Assert.assertEquals(minAlloc, plan.getMinimumAllocation());
Assert.assertEquals(maxAlloc, plan.getMaximumAllocation());
@@ -735,8 +610,7 @@ public class TestInMemoryPlan {
}
private ReservationDefinition createSimpleReservationDefinition(long arrival,
- long deadline, long duration, Collection<ReservationRequest> resources,
- String recurrenceExpression) {
+ long deadline, long duration, Collection<ReservationRequest> resources) {
// create a request with a single atomic ask
ReservationDefinition rDef = new ReservationDefinitionPBImpl();
ReservationRequests reqs = new ReservationRequestsPBImpl();
@@ -745,7 +619,6 @@ public class TestInMemoryPlan {
rDef.setReservationRequests(reqs);
rDef.setArrival(arrival);
rDef.setDeadline(deadline);
- rDef.setRecurrenceExpression(recurrenceExpression);
return rDef;
}
@@ -760,43 +633,31 @@ public class TestInMemoryPlan {
} else {
numContainers = alloc[i];
}
- ReservationRequest rr = ReservationRequest
- .newInstance(Resource.newInstance(1024, 1), (numContainers));
+ ReservationRequest rr =
+ ReservationRequest.newInstance(Resource.newInstance(1024, 1),
+ (numContainers));
req.put(new ReservationInterval(startTime + i, startTime + i + 1), rr);
}
return req;
}
- private ReservationAllocation createReservationAllocation(
- ReservationId reservationID, int start, int[] alloc) {
- return createReservationAllocation(reservationID, start, alloc, false, "0");
- }
-
- private ReservationAllocation createReservationAllocation(
- ReservationId reservationID, int start, int[] alloc, boolean isStep) {
- return createReservationAllocation(reservationID, start, alloc, isStep,
- "0");
- }
-
- private ReservationAllocation createReservationAllocation(
- ReservationId reservationID, int start, int[] alloc,
- String recurrenceExp) {
- return createReservationAllocation(reservationID, start, alloc, false,
- recurrenceExp);
+ private ReservationAllocation createReservationAllocation(ReservationId
+ reservationID, int start, int[] alloc) {
+ return createReservationAllocation(reservationID, start, alloc, false);
}
- private ReservationAllocation createReservationAllocation(
- ReservationId reservationID, int start, int[] alloc, boolean isStep,
- String recurrenceExp) {
+ private ReservationAllocation createReservationAllocation(ReservationId
+ reservationID, int start, int[] alloc, boolean isStep) {
Map<ReservationInterval, ReservationRequest> allocations =
- generateAllocation(start, alloc, isStep);
+ generateAllocation(start, alloc, isStep);
ReservationDefinition rDef =
- createSimpleReservationDefinition(start, start + alloc.length,
- alloc.length, allocations.values(), recurrenceExp);
+ createSimpleReservationDefinition(start, start + alloc.length,
+ alloc.length, allocations.values());
Map<ReservationInterval, Resource> allocs =
- ReservationSystemUtil.toResources(allocations);
+ ReservationSystemUtil.toResources(allocations);
return new InMemoryReservationAllocation(reservationID, rDef, user,
- planName, start, start + alloc.length, allocs, resCalc, minAlloc);
+ planName,
+ start, start + alloc.length, allocs, resCalc, minAlloc);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3345e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestPeriodicRLESparseResourceAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestPeriodicRLESparseResourceAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestPeriodicRLESparseResourceAllocation.java
index 457e2ee..554eb58 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestPeriodicRLESparseResourceAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestPeriodicRLESparseResourceAllocation.java
@@ -19,27 +19,26 @@
package org.apache.hadoop.yarn.server.resourcemanager.reservation;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Testing the class {@link PeriodicRLESparseResourceAllocation}.
+ * Testing the class PeriodicRLESparseResourceAllocation.
*/
-@SuppressWarnings("checkstyle:nowhitespaceafter")
public class TestPeriodicRLESparseResourceAllocation {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestPeriodicRLESparseResourceAllocation.class);
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestPeriodicRLESparseResourceAllocation.class);
@Test
public void testPeriodicCapacity() {
- int[] alloc = { 10, 7, 5, 2, 0 };
- long[] timeSteps = { 0L, 5L, 10L, 15L, 19L };
- RLESparseResourceAllocation rleSparseVector = ReservationSystemTestUtil
- .generateRLESparseResourceAllocation(alloc, timeSteps);
+ int[] alloc = {10, 7, 5, 2, 0};
+ long[] timeSteps = {0L, 5L, 10L, 15L, 19L};
+ RLESparseResourceAllocation rleSparseVector =
+ ReservationSystemTestUtil.generateRLESparseResourceAllocation(
+ alloc, timeSteps);
PeriodicRLESparseResourceAllocation periodicVector =
new PeriodicRLESparseResourceAllocation(rleSparseVector, 20L);
LOG.info(periodicVector.toString());
@@ -55,74 +54,43 @@ public class TestPeriodicRLESparseResourceAllocation {
@Test
public void testMaxPeriodicCapacity() {
- int[] alloc = { 2, 5, 7, 10, 3, 4, 6, 8 };
- long[] timeSteps = { 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L };
- RLESparseResourceAllocation rleSparseVector = ReservationSystemTestUtil
- .generateRLESparseResourceAllocation(alloc, timeSteps);
+ int[] alloc = {2, 5, 7, 10, 3, 4, 6, 8};
+ long[] timeSteps = {0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L};
+ RLESparseResourceAllocation rleSparseVector =
+ ReservationSystemTestUtil.generateRLESparseResourceAllocation(
+ alloc, timeSteps);
PeriodicRLESparseResourceAllocation periodicVector =
new PeriodicRLESparseResourceAllocation(rleSparseVector, 8L);
LOG.info(periodicVector.toString());
- Assert.assertEquals(periodicVector.getMaximumPeriodicCapacity(0, 1),
+ Assert.assertEquals(
+ periodicVector.getMaximumPeriodicCapacity(0, 1),
Resource.newInstance(10, 10));
- Assert.assertEquals(periodicVector.getMaximumPeriodicCapacity(8, 2),
+ Assert.assertEquals(
+ periodicVector.getMaximumPeriodicCapacity(8, 2),
Resource.newInstance(7, 7));
- Assert.assertEquals(periodicVector.getMaximumPeriodicCapacity(16, 3),
+ Assert.assertEquals(
+ periodicVector.getMaximumPeriodicCapacity(16, 3),
Resource.newInstance(10, 10));
- Assert.assertEquals(periodicVector.getMaximumPeriodicCapacity(17, 4),
+ Assert.assertEquals(
+ periodicVector.getMaximumPeriodicCapacity(17, 4),
Resource.newInstance(5, 5));
- Assert.assertEquals(periodicVector.getMaximumPeriodicCapacity(32, 5),
+ Assert.assertEquals(
+ periodicVector.getMaximumPeriodicCapacity(32, 5),
Resource.newInstance(4, 4));
}
@Test
- public void testMixPeriodicAndNonPeriodic() throws PlanningException {
- int[] alloc = { 2, 5, 0 };
- long[] timeSteps = { 1L, 2L, 3L };
- RLESparseResourceAllocation tempPeriodic = ReservationSystemTestUtil
- .generateRLESparseResourceAllocation(alloc, timeSteps);
- PeriodicRLESparseResourceAllocation periodic =
- new PeriodicRLESparseResourceAllocation(tempPeriodic, 10L);
-
- int[] alloc2 = { 10, 10, 0 };
- long[] timeSteps2 = { 12L, 13L, 14L };
- RLESparseResourceAllocation nonPeriodic = ReservationSystemTestUtil
- .generateRLESparseResourceAllocation(alloc2, timeSteps2);
-
- RLESparseResourceAllocation merged =
- RLESparseResourceAllocation.merge(nonPeriodic.getResourceCalculator(),
- Resource.newInstance(100 * 1024, 100), periodic, nonPeriodic,
- RLESparseResourceAllocation.RLEOperator.add, 2, 25);
-
- Assert.assertEquals(Resource.newInstance(5, 5),
- merged.getCapacityAtTime(2L));
- Assert.assertEquals(Resource.newInstance(0, 0),
- merged.getCapacityAtTime(3L));
- Assert.assertEquals(Resource.newInstance(2, 2),
- merged.getCapacityAtTime(11L));
- Assert.assertEquals(Resource.newInstance(15, 15),
- merged.getCapacityAtTime(12L));
- Assert.assertEquals(Resource.newInstance(10, 10),
- merged.getCapacityAtTime(13L));
- Assert.assertEquals(Resource.newInstance(0, 0),
- merged.getCapacityAtTime(14L));
- Assert.assertEquals(Resource.newInstance(2, 2),
- merged.getCapacityAtTime(21L));
- Assert.assertEquals(Resource.newInstance(5, 5),
- merged.getCapacityAtTime(22L));
- Assert.assertEquals(Resource.newInstance(0, 0),
- merged.getCapacityAtTime(23L));
- }
-
- @Test
public void testSetCapacityInInterval() {
- int[] alloc = { 2, 5, 0 };
- long[] timeSteps = { 1L, 2L, 3L };
- RLESparseResourceAllocation rleSparseVector = ReservationSystemTestUtil
- .generateRLESparseResourceAllocation(alloc, timeSteps);
+ int[] alloc = {2, 5, 0};
+ long[] timeSteps = {1L, 2L, 3L};
+ RLESparseResourceAllocation rleSparseVector =
+ ReservationSystemTestUtil.generateRLESparseResourceAllocation(
+ alloc, timeSteps);
PeriodicRLESparseResourceAllocation periodicVector =
new PeriodicRLESparseResourceAllocation(rleSparseVector, 10L);
ReservationInterval interval = new ReservationInterval(5L, 10L);
- periodicVector.addInterval(interval, Resource.newInstance(8, 8));
+ periodicVector.addInterval(
+ interval, Resource.newInstance(8, 8));
Assert.assertEquals(Resource.newInstance(8, 8),
periodicVector.getCapacityAtTime(5L));
Assert.assertEquals(Resource.newInstance(8, 8),
@@ -131,20 +99,21 @@ public class TestPeriodicRLESparseResourceAllocation {
periodicVector.getCapacityAtTime(10L));
Assert.assertEquals(Resource.newInstance(0, 0),
periodicVector.getCapacityAtTime(0L));
- // Assert.assertFalse(periodicVector.addInterval(
- // new ReservationInterval(7L, 12L), Resource.newInstance(8, 8)));
+ Assert.assertFalse(periodicVector.addInterval(
+ new ReservationInterval(7L, 12L), Resource.newInstance(8, 8)));
}
public void testRemoveInterval() {
- int[] alloc = { 2, 5, 3, 4, 0 };
- long[] timeSteps = { 1L, 3L, 5L, 7L, 9L };
- RLESparseResourceAllocation rleSparseVector = ReservationSystemTestUtil
- .generateRLESparseResourceAllocation(alloc, timeSteps);
+ int[] alloc = {2, 5, 3, 4, 0};
+ long[] timeSteps = {1L, 3L, 5L, 7L, 9L};
+ RLESparseResourceAllocation rleSparseVector =
+ ReservationSystemTestUtil.generateRLESparseResourceAllocation(
+ alloc, timeSteps);
PeriodicRLESparseResourceAllocation periodicVector =
new PeriodicRLESparseResourceAllocation(rleSparseVector, 10L);
ReservationInterval interval = new ReservationInterval(3L, 7L);
- Assert.assertTrue(
- periodicVector.removeInterval(interval, Resource.newInstance(3, 3)));
+ Assert.assertTrue(periodicVector.removeInterval(
+ interval, Resource.newInstance(3, 3)));
Assert.assertEquals(Resource.newInstance(2, 2),
periodicVector.getCapacityAtTime(1L));
Assert.assertEquals(Resource.newInstance(2, 2),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3345e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestRLESparseResourceAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestRLESparseResourceAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestRLESparseResourceAllocation.java
index 0027ceb..bfe46e1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestRLESparseResourceAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestRLESparseResourceAllocation.java
@@ -40,14 +40,10 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-/**
- * Testing the class {@link RLESparseResourceAllocation}.
- */
-@SuppressWarnings("checkstyle:nowhitespaceafter")
public class TestRLESparseResourceAllocation {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestRLESparseResourceAllocation.class);
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestRLESparseResourceAllocation.class);
@Test
public void testMergeAdd() throws PlanningException {
@@ -200,8 +196,7 @@ public class TestRLESparseResourceAllocation {
// Expected!
}
- // Testing that the subtractTestNonNegative detects problems even if only
- // one
+ // Testing that the subtractTestNonNegative detects problems even if only one
// of the resource dimensions is "<0"
a.put(10L, Resource.newInstance(10, 5));
b.put(11L, Resource.newInstance(5, 6));
@@ -291,8 +286,9 @@ public class TestRLESparseResourceAllocation {
public void testRangeOverlapping() {
ResourceCalculator resCalc = new DefaultResourceCalculator();
- RLESparseResourceAllocation r = new RLESparseResourceAllocation(resCalc);
- int[] alloc = { 10, 10, 10, 10, 10, 10 };
+ RLESparseResourceAllocation r =
+ new RLESparseResourceAllocation(resCalc);
+ int[] alloc = {10, 10, 10, 10, 10, 10};
int start = 100;
Set<Entry<ReservationInterval, Resource>> inputs =
generateAllocation(start, alloc, false).entrySet();
@@ -303,9 +299,9 @@ public class TestRLESparseResourceAllocation {
long d = r.getLatestNonNullTime();
// tries to trigger "out-of-range" bug
- r = r.getRangeOverlapping(s, d);
- r = r.getRangeOverlapping(s - 1, d - 1);
- r = r.getRangeOverlapping(s + 1, d + 1);
+ r = r.getRangeOverlapping(s, d);
+ r = r.getRangeOverlapping(s-1, d-1);
+ r = r.getRangeOverlapping(s+1, d+1);
}
@Test
@@ -374,29 +370,25 @@ public class TestRLESparseResourceAllocation {
// Current bug prevents this to pass. The RLESparseResourceAllocation
// does not handle removal of "partial"
// allocations correctly.
- Assert.assertEquals(102400,
- rleSparseVector.getCapacityAtTime(10).getMemorySize());
- Assert.assertEquals(0,
- rleSparseVector.getCapacityAtTime(13).getMemorySize());
- Assert.assertEquals(0,
- rleSparseVector.getCapacityAtTime(19).getMemorySize());
- Assert.assertEquals(102400,
- rleSparseVector.getCapacityAtTime(21).getMemorySize());
- Assert.assertEquals(2 * 102400,
- rleSparseVector.getCapacityAtTime(26).getMemorySize());
+ Assert.assertEquals(102400, rleSparseVector.getCapacityAtTime(10)
+ .getMemorySize());
+ Assert.assertEquals(0, rleSparseVector.getCapacityAtTime(13).getMemorySize());
+ Assert.assertEquals(0, rleSparseVector.getCapacityAtTime(19).getMemorySize());
+ Assert.assertEquals(102400, rleSparseVector.getCapacityAtTime(21)
+ .getMemorySize());
+ Assert.assertEquals(2 * 102400, rleSparseVector.getCapacityAtTime(26)
+ .getMemorySize());
ReservationInterval riRemove2 = new ReservationInterval(9, 13);
rleSparseVector.removeInterval(riRemove2, rr);
LOG.info(rleSparseVector.toString());
- Assert.assertEquals(0,
- rleSparseVector.getCapacityAtTime(11).getMemorySize());
- Assert.assertEquals(-102400,
- rleSparseVector.getCapacityAtTime(9).getMemorySize());
- Assert.assertEquals(0,
- rleSparseVector.getCapacityAtTime(13).getMemorySize());
- Assert.assertEquals(102400,
- rleSparseVector.getCapacityAtTime(20).getMemorySize());
+ Assert.assertEquals(0, rleSparseVector.getCapacityAtTime(11).getMemorySize());
+ Assert.assertEquals(-102400, rleSparseVector.getCapacityAtTime(9)
+ .getMemorySize());
+ Assert.assertEquals(0, rleSparseVector.getCapacityAtTime(13).getMemorySize());
+ Assert.assertEquals(102400, rleSparseVector.getCapacityAtTime(20)
+ .getMemorySize());
}
@@ -508,8 +500,7 @@ public class TestRLESparseResourceAllocation {
}
mapAllocations = rleSparseVector.toIntervalMap();
Assert.assertTrue(mapAllocations.size() == 5);
- for (Entry<ReservationInterval, Resource> entry : mapAllocations
- .entrySet()) {
+ for (Entry<ReservationInterval, Resource> entry : mapAllocations.entrySet()) {
ReservationInterval interval = entry.getKey();
Resource resource = entry.getValue();
if (interval.getStartTime() == 101L) {
@@ -535,46 +526,59 @@ public class TestRLESparseResourceAllocation {
@Test
public void testMaxPeriodicCapacity() {
- long[] timeSteps = { 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L };
- int[] alloc = { 2, 5, 7, 10, 3, 4, 6, 8 };
- RLESparseResourceAllocation rleSparseVector = ReservationSystemTestUtil
- .generateRLESparseResourceAllocation(alloc, timeSteps);
+ long[] timeSteps = {0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L};
+ int[] alloc = {2, 5, 7, 10, 3, 4, 6, 8};
+ RLESparseResourceAllocation rleSparseVector =
+ ReservationSystemTestUtil.generateRLESparseResourceAllocation(
+ alloc, timeSteps);
LOG.info(rleSparseVector.toString());
- Assert.assertEquals(rleSparseVector.getMaximumPeriodicCapacity(0, 1),
+ Assert.assertEquals(
+ rleSparseVector.getMaximumPeriodicCapacity(0, 1),
Resource.newInstance(10, 10));
- Assert.assertEquals(rleSparseVector.getMaximumPeriodicCapacity(0, 2),
+ Assert.assertEquals(
+ rleSparseVector.getMaximumPeriodicCapacity(0, 2),
Resource.newInstance(7, 7));
- Assert.assertEquals(rleSparseVector.getMaximumPeriodicCapacity(0, 3),
+ Assert.assertEquals(
+ rleSparseVector.getMaximumPeriodicCapacity(0, 3),
Resource.newInstance(10, 10));
- Assert.assertEquals(rleSparseVector.getMaximumPeriodicCapacity(0, 4),
+ Assert.assertEquals(
+ rleSparseVector.getMaximumPeriodicCapacity(0, 4),
Resource.newInstance(3, 3));
- Assert.assertEquals(rleSparseVector.getMaximumPeriodicCapacity(0, 5),
+ Assert.assertEquals(
+ rleSparseVector.getMaximumPeriodicCapacity(0, 5),
Resource.newInstance(4, 4));
- Assert.assertEquals(rleSparseVector.getMaximumPeriodicCapacity(0, 5),
+ Assert.assertEquals(
+ rleSparseVector.getMaximumPeriodicCapacity(0, 5),
Resource.newInstance(4, 4));
- Assert.assertEquals(rleSparseVector.getMaximumPeriodicCapacity(7, 5),
+ Assert.assertEquals(
+ rleSparseVector.getMaximumPeriodicCapacity(7, 5),
Resource.newInstance(8, 8));
- Assert.assertEquals(rleSparseVector.getMaximumPeriodicCapacity(10, 3),
+ Assert.assertEquals(
+ rleSparseVector.getMaximumPeriodicCapacity(10, 3),
Resource.newInstance(0, 0));
}
@Test
public void testGetMinimumCapacityInInterval() {
- long[] timeSteps = { 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L };
- int[] alloc = { 2, 5, 7, 10, 3, 4, 0, 8 };
- RLESparseResourceAllocation rleSparseVector = ReservationSystemTestUtil
- .generateRLESparseResourceAllocation(alloc, timeSteps);
+ long[] timeSteps = {0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L};
+ int[] alloc = {2, 5, 7, 10, 3, 4, 0, 8};
+ RLESparseResourceAllocation rleSparseVector =
+ ReservationSystemTestUtil.generateRLESparseResourceAllocation(
+ alloc, timeSteps);
LOG.info(rleSparseVector.toString());
- Assert.assertEquals(rleSparseVector.getMinimumCapacityInInterval(
- new ReservationInterval(1L, 3L)), Resource.newInstance(5, 5));
- Assert.assertEquals(rleSparseVector.getMinimumCapacityInInterval(
- new ReservationInterval(2L, 5L)), Resource.newInstance(3, 3));
- Assert.assertEquals(rleSparseVector.getMinimumCapacityInInterval(
- new ReservationInterval(1L, 7L)), Resource.newInstance(0, 0));
+ Assert.assertEquals(
+ rleSparseVector.getMinimumCapacityInInterval(
+ new ReservationInterval(1L, 3L)), Resource.newInstance(5, 5));
+ Assert.assertEquals(
+ rleSparseVector.getMinimumCapacityInInterval(
+ new ReservationInterval(2L, 5L)), Resource.newInstance(3, 3));
+ Assert.assertEquals(
+ rleSparseVector.getMinimumCapacityInInterval(
+ new ReservationInterval(1L, 7L)), Resource.newInstance(0, 0));
}
- private void setupArrays(TreeMap<Long, Resource> a,
- TreeMap<Long, Resource> b) {
+ private void setupArrays(
+ TreeMap<Long, Resource> a, TreeMap<Long, Resource> b) {
a.put(10L, Resource.newInstance(5, 5));
a.put(20L, Resource.newInstance(10, 10));
a.put(30L, Resource.newInstance(15, 15));
@@ -616,8 +620,8 @@ public class TestRLESparseResourceAllocation {
numContainers = alloc[i];
}
req.put(new ReservationInterval(startTime + i, startTime + i + 1),
- ReservationSystemUtil.toResource(ReservationRequest
- .newInstance(Resource.newInstance(1024, 1), (numContainers))));
+ ReservationSystemUtil.toResource(ReservationRequest.newInstance(
+ Resource.newInstance(1024, 1), (numContainers))));
}
return req;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3345e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java
index ddd290d..c4f94c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java
@@ -31,7 +31,6 @@ import org.apache.hadoop.yarn.api.records.ReservationDefinition;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.ReservationRequest;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.InMemoryPlan;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.InMemoryReservationAllocation;
@@ -79,10 +78,9 @@ public class TestSimpleCapacityReplanner {
enf.init("blah", conf);
// Initialize the plan with more resources
- InMemoryPlan plan = new InMemoryPlan(queueMetrics, policy, agent,
- clusterCapacity, step, res, minAlloc, maxAlloc, "dedicated", enf, true,
- YarnConfiguration.DEFAULT_RM_RESERVATION_SYSTEM_MAX_PERIODICITY,
- context, clock);
+ InMemoryPlan plan =
+ new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step,
+ res, minAlloc, maxAlloc, "dedicated", enf, true, context, clock);
// add reservation filling the plan (separating them 1ms, so we are sure
// s2 follows s1 on acceptance
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/40] hadoop git commit: HDFS-10882. Federation State Store
Interface API. Contributed by Jason Kace and Inigo Goiri.
Posted by in...@apache.org.
HDFS-10882. Federation State Store Interface API. Contributed by Jason Kace and Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/951200d9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/951200d9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/951200d9
Branch: refs/heads/HDFS-10467
Commit: 951200d9cb5d9c22761970bf9637754c56b4c5f9
Parents: bc3303e
Author: Inigo <in...@apache.org>
Authored: Thu Apr 6 19:18:52 2017 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu Sep 7 13:19:50 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 11 ++
.../server/federation/store/RecordStore.java | 100 ++++++++++++++++
.../store/driver/StateStoreSerializer.java | 119 +++++++++++++++++++
.../driver/impl/StateStoreSerializerPBImpl.java | 115 ++++++++++++++++++
.../store/records/impl/pb/PBRecord.java | 47 ++++++++
.../store/records/impl/pb/package-info.java | 29 +++++
.../src/main/resources/hdfs-default.xml | 8 ++
7 files changed, 429 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/951200d9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 4fd9d41..6a984cb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRackFaultTolerant;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.RamDiskReplicaLruTracker;
+import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreSerializerPBImpl;
import org.apache.hadoop.http.HttpConfig;
/**
@@ -1126,6 +1127,16 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS_DEFAULT =
"org.apache.hadoop.hdfs.server.federation.MockResolver";
+ // HDFS Router-based federation State Store
+ public static final String FEDERATION_STORE_PREFIX =
+ FEDERATION_ROUTER_PREFIX + "store.";
+
+ public static final String FEDERATION_STORE_SERIALIZER_CLASS =
+ DFSConfigKeys.FEDERATION_STORE_PREFIX + "serializer";
+ public static final Class<StateStoreSerializerPBImpl>
+ FEDERATION_STORE_SERIALIZER_CLASS_DEFAULT =
+ StateStoreSerializerPBImpl.class;
+
// dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry
@Deprecated
public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY
http://git-wip-us.apache.org/repos/asf/hadoop/blob/951200d9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RecordStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RecordStore.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RecordStore.java
new file mode 100644
index 0000000..524f432
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RecordStore.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import java.lang.reflect.Constructor;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+
+/**
+ * Store records in the State Store. Subclasses provide interfaces to operate on
+ * those records.
+ *
+ * @param <R> Record to store by this interface.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public abstract class RecordStore<R extends BaseRecord> {
+
+ private static final Log LOG = LogFactory.getLog(RecordStore.class);
+
+
+ /** Class of the record stored in this State Store. */
+ private final Class<R> recordClass;
+
+ /** State store driver backed by persistent storage. */
+ private final StateStoreDriver driver;
+
+
+ /**
+ * Create a new store for records.
+ *
+ * @param clazz Class of the record to store.
+ * @param stateStoreDriver Driver for the State Store.
+ */
+ protected RecordStore(Class<R> clazz, StateStoreDriver stateStoreDriver) {
+ this.recordClass = clazz;
+ this.driver = stateStoreDriver;
+ }
+
+ /**
+ * Report a required record to the data store. The data store uses this to
+ * create/maintain storage for the record.
+ *
+ * @return The class of the required record or null if no record is required
+ * for this interface.
+ */
+ public Class<R> getRecordClass() {
+ return this.recordClass;
+ }
+
+ /**
+ * Get the State Store driver.
+ *
+ * @return State Store driver.
+ */
+ protected StateStoreDriver getDriver() {
+ return this.driver;
+ }
+
+ /**
+ * Build a state store API implementation interface.
+ *
+ * @param interfaceClass The specific interface implementation to create
+ * @param driver The {@link StateStoreDriver} implementation in use.
+ * @return An initialized instance of the specified state store API
+ * implementation.
+ */
+ public static <T extends RecordStore<?>> T newInstance(
+ final Class<T> clazz, final StateStoreDriver driver) {
+
+ try {
+ Constructor<T> constructor = clazz.getConstructor(StateStoreDriver.class);
+ T recordStore = constructor.newInstance(driver);
+ return recordStore;
+ } catch (Exception e) {
+ LOG.error("Cannot create new instance for " + clazz, e);
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/951200d9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreSerializer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreSerializer.java
new file mode 100644
index 0000000..8540405
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreSerializer.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.util.ReflectionUtils;
+
+/**
+ * Serializer to store and retrieve data in the State Store.
+ */
+public abstract class StateStoreSerializer {
+
+ /** Singleton for the serializer instance. */
+ private static StateStoreSerializer defaultSerializer;
+
+ /**
+ * Get the default serializer based.
+ * @return Singleton serializer.
+ */
+ public static StateStoreSerializer getSerializer() {
+ return getSerializer(null);
+ }
+
+ /**
+ * Get a serializer based on the provided configuration.
+ * @param conf Configuration. Default if null.
+ * @return Singleton serializer.
+ */
+ public static StateStoreSerializer getSerializer(Configuration conf) {
+ if (conf == null) {
+ synchronized (StateStoreSerializer.class) {
+ if (defaultSerializer == null) {
+ conf = new Configuration();
+ defaultSerializer = newSerializer(conf);
+ }
+ }
+ return defaultSerializer;
+ } else {
+ return newSerializer(conf);
+ }
+ }
+
+ private static StateStoreSerializer newSerializer(final Configuration conf) {
+ Class<? extends StateStoreSerializer> serializerName = conf.getClass(
+ DFSConfigKeys.FEDERATION_STORE_SERIALIZER_CLASS,
+ DFSConfigKeys.FEDERATION_STORE_SERIALIZER_CLASS_DEFAULT,
+ StateStoreSerializer.class);
+ return ReflectionUtils.newInstance(serializerName, conf);
+ }
+
+ /**
+ * Create a new record.
+ * @param clazz Class of the new record.
+ * @return New record.
+ */
+ public static <T> T newRecord(Class<T> clazz) {
+ return getSerializer(null).newRecordInstance(clazz);
+ }
+
+ /**
+ * Create a new record.
+ * @param clazz Class of the new record.
+ * @return New record.
+ */
+ public abstract <T> T newRecordInstance(Class<T> clazz);
+
+ /**
+ * Serialize a record into a byte array.
+ * @param record Record to serialize.
+ * @return Byte array with the serialized record.
+ */
+ public abstract byte[] serialize(BaseRecord record);
+
+ /**
+ * Serialize a record into a string.
+ * @param record Record to serialize.
+ * @return String with the serialized record.
+ */
+ public abstract String serializeString(BaseRecord record);
+
+ /**
+ * Deserialize a bytes array into a record.
+ * @param byteArray Byte array to deserialize.
+ * @param clazz Class of the record.
+ * @return New record.
+ * @throws IOException If it cannot deserialize the record.
+ */
+ public abstract <T extends BaseRecord> T deserialize(
+ byte[] byteArray, Class<T> clazz) throws IOException;
+
+ /**
+ * Deserialize a string into a record.
+ * @param data String with the data to deserialize.
+ * @param clazz Class of the record.
+ * @return New record.
+ * @throws IOException If it cannot deserialize the record.
+ */
+ public abstract <T extends BaseRecord> T deserialize(
+ String data, Class<T> clazz) throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/951200d9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializerPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializerPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializerPBImpl.java
new file mode 100644
index 0000000..45c5dd6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializerPBImpl.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
+
+import java.io.IOException;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.codec.binary.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+import org.apache.hadoop.util.ReflectionUtils;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the State Store serializer.
+ */
+public final class StateStoreSerializerPBImpl extends StateStoreSerializer {
+
+ private static final String PB_IMPL_PACKAGE_SUFFIX = "impl.pb";
+ private static final String PB_IMPL_CLASS_SUFFIX = "PBImpl";
+
+ private Configuration localConf = new Configuration();
+
+
+ private StateStoreSerializerPBImpl() {
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <T> T newRecordInstance(Class<T> clazz) {
+ try {
+ String clazzPBImpl = getPBImplClassName(clazz);
+ Class<?> pbClazz = localConf.getClassByName(clazzPBImpl);
+ Object retObject = ReflectionUtils.newInstance(pbClazz, localConf);
+ return (T)retObject;
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private String getPBImplClassName(Class<?> clazz) {
+ String srcPackagePart = getPackageName(clazz);
+ String srcClassName = getClassName(clazz);
+ String destPackagePart = srcPackagePart + "." + PB_IMPL_PACKAGE_SUFFIX;
+ String destClassPart = srcClassName + PB_IMPL_CLASS_SUFFIX;
+ return destPackagePart + "." + destClassPart;
+ }
+
+ private String getClassName(Class<?> clazz) {
+ String fqName = clazz.getName();
+ return (fqName.substring(fqName.lastIndexOf(".") + 1, fqName.length()));
+ }
+
+ private String getPackageName(Class<?> clazz) {
+ return clazz.getPackage().getName();
+ }
+
+ @Override
+ public byte[] serialize(BaseRecord record) {
+ byte[] byteArray64 = null;
+ if (record instanceof PBRecord) {
+ PBRecord recordPB = (PBRecord) record;
+ Message msg = recordPB.getProto();
+ byte[] byteArray = msg.toByteArray();
+ byteArray64 = Base64.encodeBase64(byteArray, false);
+ }
+ return byteArray64;
+ }
+
+ @Override
+ public String serializeString(BaseRecord record) {
+ byte[] byteArray64 = serialize(record);
+ String base64Encoded = StringUtils.newStringUtf8(byteArray64);
+ return base64Encoded;
+ }
+
+ @Override
+ public <T extends BaseRecord> T deserialize(
+ byte[] byteArray, Class<T> clazz) throws IOException {
+
+ T record = newRecord(clazz);
+ if (record instanceof PBRecord) {
+ PBRecord pbRecord = (PBRecord) record;
+ byte[] byteArray64 = Base64.encodeBase64(byteArray, false);
+ String base64Encoded = StringUtils.newStringUtf8(byteArray64);
+ pbRecord.readInstance(base64Encoded);
+ }
+ return record;
+ }
+
+ @Override
+ public <T extends BaseRecord> T deserialize(String data, Class<T> clazz)
+ throws IOException {
+ byte[] byteArray64 = Base64.decodeBase64(data);
+ return deserialize(byteArray64, clazz);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/951200d9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/PBRecord.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/PBRecord.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/PBRecord.java
new file mode 100644
index 0000000..c369275
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/PBRecord.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records.impl.pb;
+
+import java.io.IOException;
+
+import com.google.protobuf.Message;
+
+/**
+ * A record implementation using Protobuf.
+ */
+public interface PBRecord {
+
+ /**
+ * Get the protocol for the record.
+ * @return The protocol for this record.
+ */
+ Message getProto();
+
+ /**
+ * Set the protocol for the record.
+ * @param proto Protocol for this record.
+ */
+ void setProto(Message proto);
+
+ /**
+ * Populate this record with serialized data.
+ * @param base64String Serialized data in base64.
+ * @throws IOException If it cannot read the data.
+ */
+ void readInstance(String base64String) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/951200d9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/package-info.java
new file mode 100644
index 0000000..b329732
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/package-info.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The protobuf implementations of state store data records defined in the
+ * org.apache.hadoop.hdfs.server.federation.store.records package. Each
+ * implementation wraps an associated protobuf proto definition.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+package org.apache.hadoop.hdfs.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/951200d9/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 87150da..3b6146a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4647,4 +4647,12 @@
</description>
</property>
+ <property>
+ <name>dfs.federation.router.store.serializer</name>
+ <value>org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreSerializerPBImpl</value>
+ <description>
+ Class to serialize State Store records.
+ </description>
+ </property>
+
</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/40] hadoop git commit: HDFS-10646. Federation admin tool.
Contributed by Inigo Goiri.
Posted by in...@apache.org.
HDFS-10646. Federation admin tool. Contributed by Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2ba43eea
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2ba43eea
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2ba43eea
Branch: refs/heads/HDFS-10467
Commit: 2ba43eea3824a5d4e5db39e48e0e51484b3608c1
Parents: 83f2aa3
Author: Inigo Goiri <in...@apache.org>
Authored: Tue Aug 8 14:44:43 2017 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu Sep 7 13:19:51 2017 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 +
.../hadoop-hdfs/src/main/bin/hdfs | 5 +
.../hadoop-hdfs/src/main/bin/hdfs.cmd | 7 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 19 ++
.../hdfs/protocolPB/RouterAdminProtocolPB.java | 44 +++
...uterAdminProtocolServerSideTranslatorPB.java | 151 ++++++++
.../RouterAdminProtocolTranslatorPB.java | 150 ++++++++
.../resolver/MembershipNamenodeResolver.java | 34 +-
.../hdfs/server/federation/router/Router.java | 52 +++
.../federation/router/RouterAdminServer.java | 183 ++++++++++
.../server/federation/router/RouterClient.java | 76 +++++
.../hdfs/tools/federation/RouterAdmin.java | 341 +++++++++++++++++++
.../hdfs/tools/federation/package-info.java | 28 ++
.../src/main/proto/RouterProtocol.proto | 47 +++
.../src/main/resources/hdfs-default.xml | 46 +++
.../server/federation/RouterConfigBuilder.java | 26 ++
.../server/federation/RouterDFSCluster.java | 43 ++-
.../server/federation/StateStoreDFSCluster.java | 148 ++++++++
.../federation/router/TestRouterAdmin.java | 261 ++++++++++++++
19 files changed, 1644 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba43eea/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index cc7a975..93216db 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -332,6 +332,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<include>editlog.proto</include>
<include>fsimage.proto</include>
<include>FederationProtocol.proto</include>
+ <include>RouterProtocol.proto</include>
</includes>
</source>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba43eea/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index b1f44a4..d51a8e2 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -31,6 +31,7 @@ function hadoop_usage
hadoop_add_option "--hosts filename" "list of hosts to use in worker mode"
hadoop_add_option "--workers" "turn on worker mode"
+<<<<<<< HEAD
hadoop_add_subcommand "balancer" daemon "run a cluster balancing utility"
hadoop_add_subcommand "cacheadmin" admin "configure the HDFS cache"
hadoop_add_subcommand "classpath" client "prints the class path needed to get the hadoop jar and the required libraries"
@@ -42,6 +43,7 @@ function hadoop_usage
hadoop_add_subcommand "diskbalancer" daemon "Distributes data evenly among disks on a given node"
hadoop_add_subcommand "envvars" client "display computed Hadoop environment variables"
hadoop_add_subcommand "ec" admin "run a HDFS ErasureCoding CLI"
+ hadoop_add_subcommand "federation" admin "manage Router-based federation"
hadoop_add_subcommand "fetchdt" client "fetch a delegation token from the NameNode"
hadoop_add_subcommand "fsck" admin "run a DFS filesystem checking utility"
hadoop_add_subcommand "getconf" client "get config values from configuration"
@@ -181,6 +183,9 @@ function hdfscmd_case
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.federation.router.Router'
;;
+ federation)
+ HADOOP_CLASSNAME='org.apache.hadoop.hdfs.tools.federation.RouterAdmin'
+ ;;
secondarynamenode)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode'
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba43eea/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
index b9853d6..53bdf70 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
@@ -59,7 +59,7 @@ if "%1" == "--loglevel" (
)
)
- set hdfscommands=dfs namenode secondarynamenode journalnode zkfc datanode dfsadmin haadmin fsck balancer jmxget oiv oev fetchdt getconf groups snapshotDiff lsSnapshottableDir cacheadmin mover storagepolicies classpath crypto router debug
+ set hdfscommands=dfs namenode secondarynamenode journalnode zkfc datanode dfsadmin haadmin fsck balancer jmxget oiv oev fetchdt getconf groups snapshotDiff lsSnapshottableDir cacheadmin mover storagepolicies classpath crypto router federation debug
for %%i in ( %hdfscommands% ) do (
if %hdfs-command% == %%i set hdfscommand=true
)
@@ -184,6 +184,11 @@ goto :eof
set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_ROUTER_OPTS%
goto :eof
+:federation
+ set CLASS=org.apache.hadoop.hdfs.tools.federation.RouterAdmin
+ set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_ROUTER_OPTS%
+ goto :eof
+
:debug
set CLASS=org.apache.hadoop.hdfs.tools.DebugAdmin
goto :eof
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba43eea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 702729b..5fd0811 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -1193,6 +1193,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String FEDERATION_STORE_PREFIX =
FEDERATION_ROUTER_PREFIX + "store.";
+ public static final String DFS_ROUTER_STORE_ENABLE =
+ FEDERATION_STORE_PREFIX + "enable";
+ public static final boolean DFS_ROUTER_STORE_ENABLE_DEFAULT = true;
+
public static final String FEDERATION_STORE_SERIALIZER_CLASS =
DFSConfigKeys.FEDERATION_STORE_PREFIX + "serializer";
public static final Class<StateStoreSerializerPBImpl>
@@ -1219,6 +1223,21 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final long FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS_DEFAULT =
TimeUnit.MINUTES.toMillis(5);
+ // HDFS Router-based federation admin
+ public static final String DFS_ROUTER_ADMIN_HANDLER_COUNT_KEY =
+ FEDERATION_ROUTER_PREFIX + "admin.handler.count";
+ public static final int DFS_ROUTER_ADMIN_HANDLER_COUNT_DEFAULT = 1;
+ public static final int DFS_ROUTER_ADMIN_PORT_DEFAULT = 8111;
+ public static final String DFS_ROUTER_ADMIN_ADDRESS_KEY =
+ FEDERATION_ROUTER_PREFIX + "admin-address";
+ public static final String DFS_ROUTER_ADMIN_ADDRESS_DEFAULT =
+ "0.0.0.0:" + DFS_ROUTER_ADMIN_PORT_DEFAULT;
+ public static final String DFS_ROUTER_ADMIN_BIND_HOST_KEY =
+ FEDERATION_ROUTER_PREFIX + "admin-bind-host";
+ public static final String DFS_ROUTER_ADMIN_ENABLE =
+ FEDERATION_ROUTER_PREFIX + "admin.enable";
+ public static final boolean DFS_ROUTER_ADMIN_ENABLE_DEFAULT = true;
+
// dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry
@Deprecated
public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba43eea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolPB.java
new file mode 100644
index 0000000..96fa794
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolPB.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocolPB;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.proto.RouterProtocolProtos.RouterAdminProtocolService;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector;
+import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.security.KerberosInfo;
+import org.apache.hadoop.security.token.TokenInfo;
+
+/**
+ * Protocol that a clients use to communicate with the NameNode.
+ * Note: This extends the protocolbuffer service based interface to
+ * add annotations required for security.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Stable
+@KerberosInfo(
+ serverPrincipal = DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY)
+@TokenInfo(DelegationTokenSelector.class)
+@ProtocolInfo(protocolName = HdfsConstants.CLIENT_NAMENODE_PROTOCOL_NAME,
+ protocolVersion = 1)
+public interface RouterAdminProtocolPB extends
+ RouterAdminProtocolService.BlockingInterface {
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba43eea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java
new file mode 100644
index 0000000..415bbd9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java
@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocolPB;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryResponseProto;
+import org.apache.hadoop.hdfs.server.federation.router.RouterAdminServer;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.AddMountTableEntryRequestPBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.AddMountTableEntryResponsePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.GetMountTableEntriesRequestPBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.GetMountTableEntriesResponsePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryRequestPBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryResponsePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.UpdateMountTableEntryRequestPBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.UpdateMountTableEntryResponsePBImpl;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+
+/**
+ * This class is used on the server side. Calls come across the wire for the for
+ * protocol {@link RouterAdminProtocolPB}. This class translates the PB data
+ * types to the native data types used inside the HDFS Router as specified in
+ * the generic RouterAdminProtocol.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Stable
+public class RouterAdminProtocolServerSideTranslatorPB implements
+ RouterAdminProtocolPB {
+
+ private final RouterAdminServer server;
+
+ /**
+ * Constructor.
+ * @param server The NN server.
+ * @throws IOException
+ */
+ public RouterAdminProtocolServerSideTranslatorPB(RouterAdminServer server)
+ throws IOException {
+ this.server = server;
+ }
+
+ @Override
+ public AddMountTableEntryResponseProto addMountTableEntry(
+ RpcController controller, AddMountTableEntryRequestProto request)
+ throws ServiceException {
+
+ try {
+ AddMountTableEntryRequest req =
+ new AddMountTableEntryRequestPBImpl(request);
+ AddMountTableEntryResponse response = server.addMountTableEntry(req);
+ AddMountTableEntryResponsePBImpl responsePB =
+ (AddMountTableEntryResponsePBImpl)response;
+ return responsePB.getProto();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ /**
+ * Remove an entry from the mount table.
+ */
+ @Override
+ public RemoveMountTableEntryResponseProto removeMountTableEntry(
+ RpcController controller, RemoveMountTableEntryRequestProto request)
+ throws ServiceException {
+ try {
+ RemoveMountTableEntryRequest req =
+ new RemoveMountTableEntryRequestPBImpl(request);
+ RemoveMountTableEntryResponse response =
+ server.removeMountTableEntry(req);
+ RemoveMountTableEntryResponsePBImpl responsePB =
+ (RemoveMountTableEntryResponsePBImpl)response;
+ return responsePB.getProto();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ /**
+ * Get matching mount table entries.
+ */
+ @Override
+ public GetMountTableEntriesResponseProto getMountTableEntries(
+ RpcController controller, GetMountTableEntriesRequestProto request)
+ throws ServiceException {
+ try {
+ GetMountTableEntriesRequest req =
+ new GetMountTableEntriesRequestPBImpl(request);
+ GetMountTableEntriesResponse response = server.getMountTableEntries(req);
+ GetMountTableEntriesResponsePBImpl responsePB =
+ (GetMountTableEntriesResponsePBImpl)response;
+ return responsePB.getProto();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ /**
+ * Update a single mount table entry.
+ */
+ @Override
+ public UpdateMountTableEntryResponseProto updateMountTableEntry(
+ RpcController controller, UpdateMountTableEntryRequestProto request)
+ throws ServiceException {
+ try {
+ UpdateMountTableEntryRequest req =
+ new UpdateMountTableEntryRequestPBImpl(request);
+ UpdateMountTableEntryResponse response =
+ server.updateMountTableEntry(req);
+ UpdateMountTableEntryResponsePBImpl responsePB =
+ (UpdateMountTableEntryResponsePBImpl)response;
+ return responsePB.getProto();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba43eea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java
new file mode 100644
index 0000000..43663ac
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocolPB;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryResponseProto;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.AddMountTableEntryRequestPBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.AddMountTableEntryResponsePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.GetMountTableEntriesRequestPBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.GetMountTableEntriesResponsePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryRequestPBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryResponsePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.UpdateMountTableEntryRequestPBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.UpdateMountTableEntryResponsePBImpl;
+import org.apache.hadoop.ipc.ProtobufHelper;
+import org.apache.hadoop.ipc.ProtocolMetaInterface;
+import org.apache.hadoop.ipc.ProtocolTranslator;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.RpcClientUtil;
+
+import com.google.protobuf.ServiceException;
+
+/**
+ * This class forwards NN's ClientProtocol calls as RPC calls to the NN server
+ * while translating from the parameter types used in ClientProtocol to the
+ * new PB types.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Stable
+public class RouterAdminProtocolTranslatorPB
+ implements ProtocolMetaInterface, MountTableManager,
+ Closeable, ProtocolTranslator {
+ final private RouterAdminProtocolPB rpcProxy;
+
+ public RouterAdminProtocolTranslatorPB(RouterAdminProtocolPB proxy) {
+ rpcProxy = proxy;
+ }
+
+ @Override
+ public void close() {
+ RPC.stopProxy(rpcProxy);
+ }
+
+ @Override
+ public Object getUnderlyingProxyObject() {
+ return rpcProxy;
+ }
+
+ @Override
+ public boolean isMethodSupported(String methodName) throws IOException {
+ return RpcClientUtil.isMethodSupported(rpcProxy,
+ RouterAdminProtocolPB.class, RPC.RpcKind.RPC_PROTOCOL_BUFFER,
+ RPC.getProtocolVersion(RouterAdminProtocolPB.class), methodName);
+ }
+
+ @Override
+ public AddMountTableEntryResponse addMountTableEntry(
+ AddMountTableEntryRequest request) throws IOException {
+ AddMountTableEntryRequestPBImpl requestPB =
+ (AddMountTableEntryRequestPBImpl)request;
+ AddMountTableEntryRequestProto proto = requestPB.getProto();
+ try {
+ AddMountTableEntryResponseProto response =
+ rpcProxy.addMountTableEntry(null, proto);
+ return new AddMountTableEntryResponsePBImpl(response);
+ } catch (ServiceException e) {
+ throw new IOException(ProtobufHelper.getRemoteException(e).getMessage());
+ }
+ }
+
+ @Override
+ public UpdateMountTableEntryResponse updateMountTableEntry(
+ UpdateMountTableEntryRequest request) throws IOException {
+ UpdateMountTableEntryRequestPBImpl requestPB =
+ (UpdateMountTableEntryRequestPBImpl)request;
+ UpdateMountTableEntryRequestProto proto = requestPB.getProto();
+ try {
+ UpdateMountTableEntryResponseProto response =
+ rpcProxy.updateMountTableEntry(null, proto);
+ return new UpdateMountTableEntryResponsePBImpl(response);
+ } catch (ServiceException e) {
+ throw new IOException(ProtobufHelper.getRemoteException(e).getMessage());
+ }
+ }
+
+ @Override
+ public RemoveMountTableEntryResponse removeMountTableEntry(
+ RemoveMountTableEntryRequest request) throws IOException {
+ RemoveMountTableEntryRequestPBImpl requestPB =
+ (RemoveMountTableEntryRequestPBImpl)request;
+ RemoveMountTableEntryRequestProto proto = requestPB.getProto();
+ try {
+ RemoveMountTableEntryResponseProto responseProto =
+ rpcProxy.removeMountTableEntry(null, proto);
+ return new RemoveMountTableEntryResponsePBImpl(responseProto);
+ } catch (ServiceException e) {
+ throw new IOException(ProtobufHelper.getRemoteException(e).getMessage());
+ }
+ }
+
+ @Override
+ public GetMountTableEntriesResponse getMountTableEntries(
+ GetMountTableEntriesRequest request) throws IOException {
+ GetMountTableEntriesRequestPBImpl requestPB =
+ (GetMountTableEntriesRequestPBImpl)request;
+ GetMountTableEntriesRequestProto proto = requestPB.getProto();
+ try {
+ GetMountTableEntriesResponseProto response =
+ rpcProxy.getMountTableEntries(null, proto);
+ return new GetMountTableEntriesResponsePBImpl(response);
+ } catch (ServiceException e) {
+ throw new IOException(ProtobufHelper.getRemoteException(e).getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba43eea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java
index b0ced24..d974c78 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java
@@ -61,7 +61,7 @@ public class MembershipNamenodeResolver
/** Reference to the State Store. */
private final StateStoreService stateStore;
/** Membership State Store interface. */
- private final MembershipStore membershipInterface;
+ private MembershipStore membershipInterface;
/** Parent router ID. */
private String routerId;
@@ -82,25 +82,27 @@ public class MembershipNamenodeResolver
if (this.stateStore != null) {
// Request cache updates from the state store
this.stateStore.registerCacheExternal(this);
-
- // Initialize the interface to get the membership
- this.membershipInterface = this.stateStore.getRegisteredRecordStore(
- MembershipStore.class);
- } else {
- this.membershipInterface = null;
}
+ }
+ private synchronized MembershipStore getMembershipStore() throws IOException {
if (this.membershipInterface == null) {
- throw new IOException("State Store does not have an interface for " +
- MembershipStore.class.getSimpleName());
+ this.membershipInterface = this.stateStore.getRegisteredRecordStore(
+ MembershipStore.class);
+ if (this.membershipInterface == null) {
+ throw new IOException("State Store does not have an interface for " +
+ MembershipStore.class.getSimpleName());
+ }
}
+ return this.membershipInterface;
}
@Override
public boolean loadCache(boolean force) {
// Our cache depends on the store, update it first
try {
- this.membershipInterface.loadCache(force);
+ MembershipStore membership = getMembershipStore();
+ membership.loadCache(force);
} catch (IOException e) {
LOG.error("Cannot update membership from the State Store", e);
}
@@ -126,8 +128,9 @@ public class MembershipNamenodeResolver
GetNamenodeRegistrationsRequest request =
GetNamenodeRegistrationsRequest.newInstance(partial);
+ MembershipStore membership = getMembershipStore();
GetNamenodeRegistrationsResponse response =
- this.membershipInterface.getNamenodeRegistrations(request);
+ membership.getNamenodeRegistrations(request);
List<MembershipState> records = response.getNamenodeMemberships();
if (records != null && records.size() == 1) {
@@ -135,7 +138,7 @@ public class MembershipNamenodeResolver
UpdateNamenodeRegistrationRequest updateRequest =
UpdateNamenodeRegistrationRequest.newInstance(
record.getNameserviceId(), record.getNamenodeId(), ACTIVE);
- this.membershipInterface.updateNamenodeRegistration(updateRequest);
+ membership.updateNamenodeRegistration(updateRequest);
}
} catch (StateStoreUnavailableException e) {
LOG.error("Cannot update {} as active, State Store unavailable", address);
@@ -226,14 +229,14 @@ public class MembershipNamenodeResolver
NamenodeHeartbeatRequest request = NamenodeHeartbeatRequest.newInstance();
request.setNamenodeMembership(record);
- return this.membershipInterface.namenodeHeartbeat(request).getResult();
+ return getMembershipStore().namenodeHeartbeat(request).getResult();
}
@Override
public Set<FederationNamespaceInfo> getNamespaces() throws IOException {
GetNamespaceInfoRequest request = GetNamespaceInfoRequest.newInstance();
GetNamespaceInfoResponse response =
- this.membershipInterface.getNamespaceInfo(request);
+ getMembershipStore().getNamespaceInfo(request);
return response.getNamespaceInfo();
}
@@ -259,8 +262,9 @@ public class MembershipNamenodeResolver
// Retrieve a list of all registrations that match this query.
// This may include all NN records for a namespace/blockpool, including
// duplicate records for the same NN from different routers.
+ MembershipStore membershipStore = getMembershipStore();
GetNamenodeRegistrationsResponse response =
- this.membershipInterface.getNamenodeRegistrations(request);
+ membershipStore.getNamenodeRegistrations(request);
List<MembershipState> memberships = response.getNamenodeMemberships();
if (!addExpired || !addUnavailable) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba43eea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
index 213a58f..fcbd2eb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
@@ -81,6 +81,10 @@ public class Router extends CompositeService {
private RouterRpcServer rpcServer;
private InetSocketAddress rpcAddress;
+ /** RPC interface for the admin. */
+ private RouterAdminServer adminServer;
+ private InetSocketAddress adminAddress;
+
/** Interface with the State Store. */
private StateStoreService stateStore;
@@ -116,6 +120,14 @@ public class Router extends CompositeService {
protected void serviceInit(Configuration configuration) throws Exception {
this.conf = configuration;
+ if (conf.getBoolean(
+ DFSConfigKeys.DFS_ROUTER_STORE_ENABLE,
+ DFSConfigKeys.DFS_ROUTER_STORE_ENABLE_DEFAULT)) {
+ // Service that maintains the State Store connection
+ this.stateStore = new StateStoreService();
+ addService(this.stateStore);
+ }
+
// Resolver to track active NNs
this.namenodeResolver = newActiveNamenodeResolver(
this.conf, this.stateStore);
@@ -139,6 +151,14 @@ public class Router extends CompositeService {
}
if (conf.getBoolean(
+ DFSConfigKeys.DFS_ROUTER_ADMIN_ENABLE,
+ DFSConfigKeys.DFS_ROUTER_ADMIN_ENABLE_DEFAULT)) {
+ // Create admin server
+ this.adminServer = createAdminServer();
+ addService(this.adminServer);
+ }
+
+ if (conf.getBoolean(
DFSConfigKeys.DFS_ROUTER_HEARTBEAT_ENABLE,
DFSConfigKeys.DFS_ROUTER_HEARTBEAT_ENABLE_DEFAULT)) {
@@ -264,6 +284,38 @@ public class Router extends CompositeService {
}
/////////////////////////////////////////////////////////
+ // Admin server
+ /////////////////////////////////////////////////////////
+
+ /**
+ * Create a new router admin server to handle the router admin interface.
+ *
+ * @return RouterAdminServer
+ * @throws IOException If the admin server was not successfully started.
+ */
+ protected RouterAdminServer createAdminServer() throws IOException {
+ return new RouterAdminServer(this.conf, this);
+ }
+
+ /**
+ * Set the current Admin socket for the router.
+ *
+ * @param adminAddress Admin RPC address.
+ */
+ protected void setAdminServerAddress(InetSocketAddress address) {
+ this.adminAddress = address;
+ }
+
+ /**
+ * Get the current Admin socket address for the router.
+ *
+ * @return InetSocketAddress Admin address.
+ */
+ public InetSocketAddress getAdminServerAddress() {
+ return adminAddress;
+ }
+
+ /////////////////////////////////////////////////////////
// Namenode heartbeat monitors
/////////////////////////////////////////////////////////
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba43eea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
new file mode 100644
index 0000000..7687216
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
@@ -0,0 +1,183 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.protocol.proto.RouterProtocolProtos.RouterAdminProtocolService;
+import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolPB;
+import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolServerSideTranslatorPB;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
+import org.apache.hadoop.hdfs.server.federation.store.MountTableStore;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.RPC.Server;
+import org.apache.hadoop.service.AbstractService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.protobuf.BlockingService;
+
+/**
+ * This class is responsible for handling all of the Admin calls to the HDFS
+ * router. It is created, started, and stopped by {@link Router}.
+ */
+public class RouterAdminServer extends AbstractService
+ implements MountTableManager {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RouterAdminServer.class);
+
+ private Configuration conf;
+
+ private final Router router;
+
+ private MountTableStore mountTableStore;
+
+ /** The Admin server that listens to requests from clients. */
+ private final Server adminServer;
+ private final InetSocketAddress adminAddress;
+
+ public RouterAdminServer(Configuration conf, Router router)
+ throws IOException {
+ super(RouterAdminServer.class.getName());
+
+ this.conf = conf;
+ this.router = router;
+
+ int handlerCount = this.conf.getInt(
+ DFSConfigKeys.DFS_ROUTER_ADMIN_HANDLER_COUNT_KEY,
+ DFSConfigKeys.DFS_ROUTER_ADMIN_HANDLER_COUNT_DEFAULT);
+
+ RPC.setProtocolEngine(this.conf, RouterAdminProtocolPB.class,
+ ProtobufRpcEngine.class);
+
+ RouterAdminProtocolServerSideTranslatorPB routerAdminProtocolTranslator =
+ new RouterAdminProtocolServerSideTranslatorPB(this);
+ BlockingService clientNNPbService = RouterAdminProtocolService.
+ newReflectiveBlockingService(routerAdminProtocolTranslator);
+
+ InetSocketAddress confRpcAddress = conf.getSocketAddr(
+ DFSConfigKeys.DFS_ROUTER_ADMIN_BIND_HOST_KEY,
+ DFSConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_KEY,
+ DFSConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_DEFAULT,
+ DFSConfigKeys.DFS_ROUTER_ADMIN_PORT_DEFAULT);
+
+ String bindHost = conf.get(
+ DFSConfigKeys.DFS_ROUTER_ADMIN_BIND_HOST_KEY,
+ confRpcAddress.getHostName());
+ LOG.info("Admin server binding to {}:{}",
+ bindHost, confRpcAddress.getPort());
+
+ this.adminServer = new RPC.Builder(this.conf)
+ .setProtocol(RouterAdminProtocolPB.class)
+ .setInstance(clientNNPbService)
+ .setBindAddress(bindHost)
+ .setPort(confRpcAddress.getPort())
+ .setNumHandlers(handlerCount)
+ .setVerbose(false)
+ .build();
+
+ // The RPC-server port can be ephemeral... ensure we have the correct info
+ InetSocketAddress listenAddress = this.adminServer.getListenerAddress();
+ this.adminAddress = new InetSocketAddress(
+ confRpcAddress.getHostName(), listenAddress.getPort());
+ router.setAdminServerAddress(this.adminAddress);
+ }
+
+ /** Allow access to the client RPC server for testing. */
+ @VisibleForTesting
+ Server getAdminServer() {
+ return this.adminServer;
+ }
+
+ private MountTableStore getMountTableStore() throws IOException {
+ if (this.mountTableStore == null) {
+ this.mountTableStore = router.getStateStore().getRegisteredRecordStore(
+ MountTableStore.class);
+ if (this.mountTableStore == null) {
+ throw new IOException("Mount table state store is not available.");
+ }
+ }
+ return this.mountTableStore;
+ }
+
+ /**
+ * Get the RPC address of the admin service.
+ * @return Administration service RPC address.
+ */
+ public InetSocketAddress getRpcAddress() {
+ return this.adminAddress;
+ }
+
+ @Override
+ protected void serviceInit(Configuration configuration) throws Exception {
+ this.conf = configuration;
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ this.adminServer.start();
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (this.adminServer != null) {
+ this.adminServer.stop();
+ }
+ super.serviceStop();
+ }
+
+ @Override
+ public AddMountTableEntryResponse addMountTableEntry(
+ AddMountTableEntryRequest request) throws IOException {
+ return getMountTableStore().addMountTableEntry(request);
+ }
+
+ @Override
+ public UpdateMountTableEntryResponse updateMountTableEntry(
+ UpdateMountTableEntryRequest request) throws IOException {
+ return getMountTableStore().updateMountTableEntry(request);
+ }
+
+ @Override
+ public RemoveMountTableEntryResponse removeMountTableEntry(
+ RemoveMountTableEntryRequest request) throws IOException {
+ return getMountTableStore().removeMountTableEntry(request);
+ }
+
+ @Override
+ public GetMountTableEntriesResponse getMountTableEntries(
+ GetMountTableEntriesRequest request) throws IOException {
+ return getMountTableStore().getMountTableEntries(request);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba43eea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClient.java
new file mode 100644
index 0000000..1f76b98
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClient.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolPB;
+import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolTranslatorPB;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+
+/**
+ * Client to connect to the {@link Router} via the admin protocol.
+ */
+@Private
+public class RouterClient implements Closeable {
+
+ private final RouterAdminProtocolTranslatorPB proxy;
+ private final UserGroupInformation ugi;
+
+ private static RouterAdminProtocolTranslatorPB createRouterProxy(
+ InetSocketAddress address, Configuration conf, UserGroupInformation ugi)
+ throws IOException {
+
+ RPC.setProtocolEngine(
+ conf, RouterAdminProtocolPB.class, ProtobufRpcEngine.class);
+
+ AtomicBoolean fallbackToSimpleAuth = new AtomicBoolean(false);
+ final long version = RPC.getProtocolVersion(RouterAdminProtocolPB.class);
+ RouterAdminProtocolPB proxy = RPC.getProtocolProxy(
+ RouterAdminProtocolPB.class, version, address, ugi, conf,
+ NetUtils.getDefaultSocketFactory(conf),
+ RPC.getRpcTimeout(conf), null,
+ fallbackToSimpleAuth).getProxy();
+
+ return new RouterAdminProtocolTranslatorPB(proxy);
+ }
+
+ public RouterClient(InetSocketAddress address, Configuration conf)
+ throws IOException {
+ this.ugi = UserGroupInformation.getCurrentUser();
+ this.proxy = createRouterProxy(address, conf, ugi);
+ }
+
+ public MountTableManager getMountTableManager() {
+ return proxy;
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ RPC.stopProxy(proxy);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba43eea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
new file mode 100644
index 0000000..0786419
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
@@ -0,0 +1,341 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.federation;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
+import org.apache.hadoop.hdfs.server.federation.router.RouterClient;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class provides some Federation administrative access shell commands.
+ */
+@Private
+public class RouterAdmin extends Configured implements Tool {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RouterAdmin.class);
+
+ private RouterClient client;
+
+ public static void main(String[] argv) throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ RouterAdmin admin = new RouterAdmin(conf);
+
+ int res = ToolRunner.run(admin, argv);
+ System.exit(res);
+ }
+
+ public RouterAdmin(Configuration conf) {
+ super(conf);
+ }
+
+ /**
+ * Print the usage message.
+ */
+ public void printUsage() {
+ String usage = "Federation Admin Tools:\n"
+ + "\t[-add <source> <nameservice> <destination> "
+ + "[-readonly] [-order HASH|LOCAL|RANDOM|HASH_ALL]]\n"
+ + "\t[-rm <source>]\n"
+ + "\t[-ls <path>]\n";
+ System.out.println(usage);
+ }
+
+ @Override
+ public int run(String[] argv) throws Exception {
+ if (argv.length < 1) {
+ System.err.println("Not enough parameters specificed");
+ printUsage();
+ return -1;
+ }
+
+ int exitCode = -1;
+ int i = 0;
+ String cmd = argv[i++];
+
+ // Verify that we have enough command line parameters
+ if ("-add".equals(cmd)) {
+ if (argv.length < 4) {
+ System.err.println("Not enough parameters specificed for cmd " + cmd);
+ printUsage();
+ return exitCode;
+ }
+ } else if ("-rm".equalsIgnoreCase(cmd)) {
+ if (argv.length < 2) {
+ System.err.println("Not enough parameters specificed for cmd " + cmd);
+ printUsage();
+ return exitCode;
+ }
+ }
+
+ // Initialize RouterClient
+ try {
+ String address = getConf().getTrimmed(
+ DFSConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_KEY,
+ DFSConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_DEFAULT);
+ InetSocketAddress routerSocket = NetUtils.createSocketAddr(address);
+ client = new RouterClient(routerSocket, getConf());
+ } catch (RPC.VersionMismatch v) {
+ System.err.println(
+ "Version mismatch between client and server... command aborted");
+ return exitCode;
+ } catch (IOException e) {
+ System.err.println("Bad connection to Router... command aborted");
+ return exitCode;
+ }
+
+ Exception debugException = null;
+ exitCode = 0;
+ try {
+ if ("-add".equals(cmd)) {
+ if (addMount(argv, i)) {
+ System.err.println("Successfuly added mount point " + argv[i]);
+ }
+ } else if ("-rm".equals(cmd)) {
+ if (removeMount(argv[i])) {
+ System.err.println("Successfully removed mount point " + argv[i]);
+ }
+ } else if ("-ls".equals(cmd)) {
+ if (argv.length > 1) {
+ listMounts(argv[i]);
+ } else {
+ listMounts("/");
+ }
+ } else {
+ printUsage();
+ return exitCode;
+ }
+ } catch (IllegalArgumentException arge) {
+ debugException = arge;
+ exitCode = -1;
+ System.err.println(cmd.substring(1) + ": " + arge.getLocalizedMessage());
+ printUsage();
+ } catch (RemoteException e) {
+ // This is a error returned by the server.
+ // Print out the first line of the error message, ignore the stack trace.
+ exitCode = -1;
+ debugException = e;
+ try {
+ String[] content;
+ content = e.getLocalizedMessage().split("\n");
+ System.err.println(cmd.substring(1) + ": " + content[0]);
+ e.printStackTrace();
+ } catch (Exception ex) {
+ System.err.println(cmd.substring(1) + ": " + ex.getLocalizedMessage());
+ e.printStackTrace();
+ debugException = ex;
+ }
+ } catch (Exception e) {
+ exitCode = -1;
+ debugException = e;
+ System.err.println(cmd.substring(1) + ": " + e.getLocalizedMessage());
+ e.printStackTrace();
+ }
+ if (debugException != null) {
+ LOG.debug("Exception encountered", debugException);
+ }
+ return exitCode;
+ }
+
+ /**
+ * Add a mount table entry or update if it exists.
+ *
+ * @param parameters Parameters for the mount point.
+ * @param i Index in the parameters.
+ */
+ public boolean addMount(String[] parameters, int i) throws IOException {
+ // Mandatory parameters
+ String mount = parameters[i++];
+ String[] nss = parameters[i++].split(",");
+ String dest = parameters[i++];
+
+ // Optional parameters
+ boolean readOnly = false;
+ DestinationOrder order = DestinationOrder.HASH;
+ while (i < parameters.length) {
+ if (parameters[i].equals("-readonly")) {
+ readOnly = true;
+ } else if (parameters[i].equals("-order")) {
+ i++;
+ try {
+ order = DestinationOrder.valueOf(parameters[i]);
+ } catch(Exception e) {
+ System.err.println("Cannot parse order: " + parameters[i]);
+ }
+ }
+ i++;
+ }
+
+ return addMount(mount, nss, dest, readOnly, order);
+ }
+
+ /**
+ * Add a mount table entry or update if it exists.
+ *
+ * @param mount Mount point.
+ * @param nss Namespaces where this is mounted to.
+ * @param dest Destination path.
+ * @param readonly If the mount point is read only.
+ * @param order Order of the destination locations.
+ * @return If the mount point was added.
+ * @throws IOException Error adding the mount point.
+ */
+ public boolean addMount(String mount, String[] nss, String dest,
+ boolean readonly, DestinationOrder order) throws IOException {
+ // Get the existing entry
+ MountTableManager mountTable = client.getMountTableManager();
+ GetMountTableEntriesRequest getRequest =
+ GetMountTableEntriesRequest.newInstance(mount);
+ GetMountTableEntriesResponse getResponse =
+ mountTable.getMountTableEntries(getRequest);
+ List<MountTable> results = getResponse.getEntries();
+ MountTable existingEntry = null;
+ for (MountTable result : results) {
+ if (mount.equals(result.getSourcePath())) {
+ existingEntry = result;
+ }
+ }
+
+ if (existingEntry == null) {
+ // Create and add the entry if it doesn't exist
+ Map<String, String> destMap = new LinkedHashMap<>();
+ for (String ns : nss) {
+ destMap.put(ns, dest);
+ }
+ MountTable newEntry = MountTable.newInstance(mount, destMap);
+ if (readonly) {
+ newEntry.setReadOnly(true);
+ }
+ if (order != null) {
+ newEntry.setDestOrder(order);
+ }
+ AddMountTableEntryRequest request =
+ AddMountTableEntryRequest.newInstance(newEntry);
+ AddMountTableEntryResponse addResponse =
+ mountTable.addMountTableEntry(request);
+ boolean added = addResponse.getStatus();
+ if (!added) {
+ System.err.println("Cannot add mount point " + mount);
+ }
+ return added;
+ } else {
+ // Update the existing entry if it exists
+ for (String nsId : nss) {
+ if (!existingEntry.addDestination(nsId, dest)) {
+ System.err.println("Cannot add destination at " + nsId + " " + dest);
+ }
+ }
+ if (readonly) {
+ existingEntry.setReadOnly(true);
+ }
+ if (order != null) {
+ existingEntry.setDestOrder(order);
+ }
+ UpdateMountTableEntryRequest updateRequest =
+ UpdateMountTableEntryRequest.newInstance(existingEntry);
+ UpdateMountTableEntryResponse updateResponse =
+ mountTable.updateMountTableEntry(updateRequest);
+ boolean updated = updateResponse.getStatus();
+ if (!updated) {
+ System.err.println("Cannot update mount point " + mount);
+ }
+ return updated;
+ }
+ }
+
+ /**
+ * Remove mount point.
+ *
+ * @param path Path to remove.
+ * @throws IOException If it cannot be removed.
+ */
+ public boolean removeMount(String path) throws IOException {
+ MountTableManager mountTable = client.getMountTableManager();
+ RemoveMountTableEntryRequest request =
+ RemoveMountTableEntryRequest.newInstance(path);
+ RemoveMountTableEntryResponse response =
+ mountTable.removeMountTableEntry(request);
+ boolean removed = response.getStatus();
+ if (!removed) {
+ System.out.println("Cannot remove mount point " + path);
+ }
+ return removed;
+ }
+
+ /**
+ * List mount points.
+ *
+ * @param path Path to list.
+ * @throws IOException If it cannot be listed.
+ */
+ public void listMounts(String path) throws IOException {
+ MountTableManager mountTable = client.getMountTableManager();
+ GetMountTableEntriesRequest request =
+ GetMountTableEntriesRequest.newInstance(path);
+ GetMountTableEntriesResponse response =
+ mountTable.getMountTableEntries(request);
+ List<MountTable> entries = response.getEntries();
+ printMounts(entries);
+ }
+
+ private static void printMounts(List<MountTable> entries) {
+ System.out.println("Mount Table Entries:");
+ System.out.println(String.format(
+ "%-25s %-25s",
+ "Source", "Destinations"));
+ for (MountTable entry : entries) {
+ StringBuilder destBuilder = new StringBuilder();
+ for (RemoteLocation location : entry.getDestinations()) {
+ if (destBuilder.length() > 0) {
+ destBuilder.append(",");
+ }
+ destBuilder.append(String.format("%s->%s", location.getNameserviceId(),
+ location.getDest()));
+ }
+ System.out.println(String.format("%-25s %-25s", entry.getSourcePath(),
+ destBuilder.toString()));
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba43eea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/package-info.java
new file mode 100644
index 0000000..466c3d3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/package-info.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * It includes the tools to manage the Router-based federation. Includes the
+ * utilities to add and remove mount table entries.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+package org.apache.hadoop.hdfs.tools.federation;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba43eea/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/RouterProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/RouterProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/RouterProtocol.proto
new file mode 100644
index 0000000..3f43040
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/RouterProtocol.proto
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.hadoop.hdfs.protocol.proto";
+option java_outer_classname = "RouterProtocolProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+package hadoop.hdfs.router;
+
+import "FederationProtocol.proto";
+
+service RouterAdminProtocolService {
+ /**
+ * Add a mount table entry.
+ */
+ rpc addMountTableEntry(AddMountTableEntryRequestProto) returns(AddMountTableEntryResponseProto);
+
+ /**
+ * Update an existing mount table entry without copying files.
+ */
+ rpc updateMountTableEntry(UpdateMountTableEntryRequestProto) returns(UpdateMountTableEntryResponseProto);
+
+ /**
+ * Remove a mount table entry.
+ */
+ rpc removeMountTableEntry(RemoveMountTableEntryRequestProto) returns(RemoveMountTableEntryResponseProto);
+
+ /**
+ * Get matching mount entries
+ */
+ rpc getMountTableEntries(GetMountTableEntriesRequestProto) returns(GetMountTableEntriesResponseProto);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba43eea/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index b7c0f5d..17989f2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4726,6 +4726,44 @@
</description>
</property>
+ <property>
+ <name>dfs.federation.router.admin.enable</name>
+ <value>true</value>
+ <description>
+ If the RPC admin service to handle client requests in the router is
+ enabled.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.admin-address</name>
+ <value>0.0.0.0:8111</value>
+ <description>
+ RPC address that handles the admin requests.
+ The value of this property will take the form of router-host1:rpc-port.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.admin-bind-host</name>
+ <value></value>
+ <description>
+ The actual address the RPC admin server will bind to. If this optional
+ address is set, it overrides only the hostname portion of
+ dfs.federation.router.admin-address. This is useful for making the name
+ node listen on all interfaces by setting it to 0.0.0.0.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.admin.handler.count</name>
+ <value>1</value>
+ <description>
+ The number of server threads for the router to handle RPC requests from
+ admin.
+ </description>
+ </property>
+
<property>
<name>dfs.federation.router.file.resolver.client.class</name>
<value>org.apache.hadoop.hdfs.server.federation.MockResolver</value>
@@ -4743,6 +4781,14 @@
</property>
<property>
+ <name>dfs.federation.router.store.enable</name>
+ <value>true</value>
+ <description>
+ If the Router connects to the State Store.
+ </description>
+ </property>
+
+ <property>
<name>dfs.federation.router.store.serializer</name>
<value>org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreSerializerPBImpl</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba43eea/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
index 21555c5..cac5e6b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
@@ -28,8 +28,10 @@ public class RouterConfigBuilder {
private Configuration conf;
private boolean enableRpcServer = false;
+ private boolean enableAdminServer = false;
private boolean enableHeartbeat = false;
private boolean enableLocalHeartbeat = false;
+ private boolean enableStateStore = false;
public RouterConfigBuilder(Configuration configuration) {
this.conf = configuration;
@@ -41,8 +43,10 @@ public class RouterConfigBuilder {
public RouterConfigBuilder all() {
this.enableRpcServer = true;
+ this.enableAdminServer = true;
this.enableHeartbeat = true;
this.enableLocalHeartbeat = true;
+ this.enableStateStore = true;
return this;
}
@@ -56,21 +60,43 @@ public class RouterConfigBuilder {
return this;
}
+ public RouterConfigBuilder admin(boolean enable) {
+ this.enableAdminServer = enable;
+ return this;
+ }
+
public RouterConfigBuilder heartbeat(boolean enable) {
this.enableHeartbeat = enable;
return this;
}
+ public RouterConfigBuilder stateStore(boolean enable) {
+ this.enableStateStore = enable;
+ return this;
+ }
+
public RouterConfigBuilder rpc() {
return this.rpc(true);
}
+ public RouterConfigBuilder admin() {
+ return this.admin(true);
+ }
+
public RouterConfigBuilder heartbeat() {
return this.heartbeat(true);
}
+ public RouterConfigBuilder stateStore() {
+ return this.stateStore(true);
+ }
+
public Configuration build() {
+ conf.setBoolean(DFSConfigKeys.DFS_ROUTER_STORE_ENABLE,
+ this.enableStateStore);
conf.setBoolean(DFSConfigKeys.DFS_ROUTER_RPC_ENABLE, this.enableRpcServer);
+ conf.setBoolean(DFSConfigKeys.DFS_ROUTER_ADMIN_ENABLE,
+ this.enableAdminServer);
conf.setBoolean(DFSConfigKeys.DFS_ROUTER_HEARTBEAT_ENABLE,
this.enableHeartbeat);
conf.setBoolean(DFSConfigKeys.DFS_ROUTER_MONITOR_LOCAL_NAMENODE,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba43eea/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
index 0830c19..1ee49d5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
@@ -25,8 +25,12 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_BIND_HOST_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_ADMIN_BIND_HOST_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_CACHE_TIME_TO_LIVE_MS;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HANDLER_COUNT_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HEARTBEAT_INTERVAL_MS;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_RPC_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_RPC_BIND_HOST_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS;
@@ -46,6 +50,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map.Entry;
import java.util.Random;
+import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
@@ -67,6 +72,7 @@ import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServi
import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport;
import org.apache.hadoop.hdfs.server.federation.router.Router;
+import org.apache.hadoop.hdfs.server.federation.router.RouterClient;
import org.apache.hadoop.hdfs.server.namenode.FSImage;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
@@ -101,6 +107,15 @@ public class RouterDFSCluster {
/** Mini cluster. */
private MiniDFSCluster cluster;
+ protected static final long DEFAULT_HEARTBEAT_INTERVAL_MS =
+ TimeUnit.SECONDS.toMillis(5);
+ protected static final long DEFAULT_CACHE_INTERVAL_MS =
+ TimeUnit.SECONDS.toMillis(5);
+ /** Heartbeat interval in milliseconds. */
+ private long heartbeatInterval;
+ /** Cache flush interval in milliseconds. */
+ private long cacheFlushInterval;
+
/** Router configuration overrides. */
private Configuration routerOverrides;
/** Namenode configuration overrides. */
@@ -118,6 +133,7 @@ public class RouterDFSCluster {
private int rpcPort;
private DFSClient client;
private Configuration conf;
+ private RouterClient adminClient;
private URI fileSystemUri;
public RouterContext(Configuration conf, String nsId, String nnId)
@@ -183,6 +199,15 @@ public class RouterDFSCluster {
});
}
+ public RouterClient getAdminClient() throws IOException {
+ if (adminClient == null) {
+ InetSocketAddress routerSocket = router.getAdminServerAddress();
+ LOG.info("Connecting to router admin at {}", routerSocket);
+ adminClient = new RouterClient(routerSocket, conf);
+ }
+ return adminClient;
+ }
+
public DFSClient getClient() throws IOException, URISyntaxException {
if (client == null) {
LOG.info("Connecting to router at {}", fileSystemUri);
@@ -304,13 +329,22 @@ public class RouterDFSCluster {
}
}
- public RouterDFSCluster(boolean ha, int numNameservices, int numNamenodes) {
+ public RouterDFSCluster(boolean ha, int numNameservices, int numNamenodes,
+ long heartbeatInterval, long cacheFlushInterval) {
this.highAvailability = ha;
+ this.heartbeatInterval = heartbeatInterval;
+ this.cacheFlushInterval = cacheFlushInterval;
configureNameservices(numNameservices, numNamenodes);
}
public RouterDFSCluster(boolean ha, int numNameservices) {
- this(ha, numNameservices, 2);
+ this(ha, numNameservices, 2,
+ DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_CACHE_INTERVAL_MS);
+ }
+
+ public RouterDFSCluster(boolean ha, int numNameservices, int numNamnodes) {
+ this(ha, numNameservices, numNamnodes,
+ DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_CACHE_INTERVAL_MS);
}
/**
@@ -404,7 +438,12 @@ public class RouterDFSCluster {
conf.set(DFS_ROUTER_RPC_ADDRESS_KEY, "127.0.0.1:0");
conf.set(DFS_ROUTER_RPC_BIND_HOST_KEY, "0.0.0.0");
+ conf.set(DFS_ROUTER_ADMIN_ADDRESS_KEY, "127.0.0.1:0");
+ conf.set(DFS_ROUTER_ADMIN_BIND_HOST_KEY, "0.0.0.0");
+
conf.set(DFS_ROUTER_DEFAULT_NAMESERVICE, nameservices.get(0));
+ conf.setLong(DFS_ROUTER_HEARTBEAT_INTERVAL_MS, heartbeatInterval);
+ conf.setLong(DFS_ROUTER_CACHE_TIME_TO_LIVE_MS, cacheFlushInterval);
// Use mock resolver classes
conf.setClass(FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba43eea/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/StateStoreDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/StateStoreDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/StateStoreDFSCluster.java
new file mode 100644
index 0000000..e42ab50
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/StateStoreDFSCluster.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation;
+
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.createMockRegistrationForNamenode;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.getStateStoreConfiguration;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.synchronizeRecords;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+
+/**
+ * Test utility to mimic a federated HDFS cluster with a router and a state
+ * store.
+ */
+public class StateStoreDFSCluster extends RouterDFSCluster {
+
+ private static final Class<?> DEFAULT_FILE_RESOLVER =
+ MountTableResolver.class;
+ private static final Class<?> DEFAULT_NAMENODE_RESOLVER =
+ MembershipNamenodeResolver.class;
+
+ public StateStoreDFSCluster(boolean ha, int numNameservices, int numNamenodes,
+ long heartbeatInterval, long cacheFlushInterval)
+ throws IOException, InterruptedException {
+ this(ha, numNameservices, numNamenodes, heartbeatInterval,
+ cacheFlushInterval, DEFAULT_FILE_RESOLVER);
+ }
+
+ public StateStoreDFSCluster(boolean ha, int numNameservices, int numNamenodes,
+ long heartbeatInterval, long cacheFlushInterval, Class<?> fileResolver)
+ throws IOException, InterruptedException {
+ super(ha, numNameservices, numNamenodes, heartbeatInterval,
+ cacheFlushInterval);
+
+ // Attach state store and resolvers to router
+ Configuration stateStoreConfig = getStateStoreConfiguration();
+ // Use state store backed resolvers
+ stateStoreConfig.setClass(
+ DFSConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS,
+ DEFAULT_NAMENODE_RESOLVER, ActiveNamenodeResolver.class);
+ stateStoreConfig.setClass(
+ DFSConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
+ fileResolver, FileSubclusterResolver.class);
+ this.addRouterOverrides(stateStoreConfig);
+ }
+
+ public StateStoreDFSCluster(boolean ha, int numNameservices,
+ Class<?> fileResolver) throws IOException, InterruptedException {
+ this(ha, numNameservices, 2,
+ DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_CACHE_INTERVAL_MS, fileResolver);
+ }
+
+ public StateStoreDFSCluster(boolean ha, int numNameservices)
+ throws IOException, InterruptedException {
+ this(ha, numNameservices, 2,
+ DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_CACHE_INTERVAL_MS);
+ }
+
+ public StateStoreDFSCluster(boolean ha, int numNameservices,
+ int numNamnodes) throws IOException, InterruptedException {
+ this(ha, numNameservices, numNamnodes,
+ DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_CACHE_INTERVAL_MS);
+ }
+
+ /////////////////////////////////////////////////////////////////////////////
+ // State Store Test Fixtures
+ /////////////////////////////////////////////////////////////////////////////
+
+ /**
+ * Adds test fixtures for NN registation for each NN nameservice -> NS
+ * namenode -> NN rpcAddress -> 0.0.0.0:0 webAddress -> 0.0.0.0:0 state ->
+ * STANDBY safeMode -> false blockPool -> test.
+ *
+ * @param stateStore State Store.
+ * @throws IOException If it cannot register.
+ */
+ public void createTestRegistration(StateStoreService stateStore)
+ throws IOException {
+ List<MembershipState> entries = new ArrayList<MembershipState>();
+ for (NamenodeContext nn : this.getNamenodes()) {
+ MembershipState entry = createMockRegistrationForNamenode(
+ nn.getNameserviceId(), nn.getNamenodeId(),
+ FederationNamenodeServiceState.STANDBY);
+ entries.add(entry);
+ }
+ synchronizeRecords(
+ stateStore, entries, MembershipState.class);
+ }
+
+ public void createTestMountTable(StateStoreService stateStore)
+ throws IOException {
+ List<MountTable> mounts = generateMockMountTable();
+ synchronizeRecords(stateStore, mounts, MountTable.class);
+ stateStore.refreshCaches();
+ }
+
+ public List<MountTable> generateMockMountTable() throws IOException {
+ // create table entries
+ List<MountTable> entries = new ArrayList<>();
+ for (String ns : this.getNameservices()) {
+ Map<String, String> destMap = new HashMap<>();
+ destMap.put(ns, getNamenodePathForNS(ns));
+
+ // Direct path
+ String fedPath = getFederatedPathForNS(ns);
+ MountTable entry = MountTable.newInstance(fedPath, destMap);
+ entries.add(entry);
+ }
+
+ // Root path goes to nameservice 1
+ Map<String, String> destMap = new HashMap<>();
+ String ns0 = this.getNameservices().get(0);
+ destMap.put(ns0, "/");
+ MountTable entry = MountTable.newInstance("/", destMap);
+ entries.add(entry);
+ return entries;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/40] hadoop git commit: YARN-6992. Kill application button is
visible even if the application is FINISHED in RM UI. Contributed by Suma
Shivaprasad.
Posted by in...@apache.org.
YARN-6992. Kill application button is visible even if the application is FINISHED in RM UI. Contributed by Suma Shivaprasad.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f101e7d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f101e7d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f101e7d
Branch: refs/heads/HDFS-10467
Commit: 6f101e7df10d477cf7469f6979d0cb8269315a3e
Parents: b6e7d13
Author: Sunil G <su...@apache.org>
Authored: Thu Sep 7 13:16:20 2017 +0530
Committer: Sunil G <su...@apache.org>
Committed: Thu Sep 7 13:16:20 2017 +0530
----------------------------------------------------------------------
.../java/org/apache/hadoop/yarn/server/webapp/AppBlock.java | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f101e7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
index 693aa04..95bc0aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
@@ -144,7 +144,8 @@ public class AppBlock extends HtmlBlock {
&& webUiType.equals(YarnWebParams.RM_WEB_UI)
&& conf.getBoolean(YarnConfiguration.RM_WEBAPP_UI_ACTIONS_ENABLED,
YarnConfiguration.DEFAULT_RM_WEBAPP_UI_ACTIONS_ENABLED)
- && !unsecuredUIForSecuredCluster) {
+ && !unsecuredUIForSecuredCluster
+ && !isAppInFinalState(app)) {
// Application Kill
html.div()
.button()
@@ -419,4 +420,10 @@ public class AppBlock extends HtmlBlock {
}
return ret;
}
+
+ private boolean isAppInFinalState(AppInfo app) {
+ return app.getAppState() == YarnApplicationState.FINISHED
+ || app.getAppState() == YarnApplicationState.FAILED
+ || app.getAppState() == YarnApplicationState.KILLED;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/40] hadoop git commit: HADOOP-14520. WASB: Block compaction for
Azure Block Blobs. Contributed by Georgi Chalakov
Posted by in...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13eda500/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockCompaction.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockCompaction.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockCompaction.java
new file mode 100644
index 0000000..820ce4f
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockCompaction.java
@@ -0,0 +1,266 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import com.microsoft.azure.storage.blob.BlockEntry;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.util.List;
+
+/**
+ * Test class that runs WASB block compaction process for block blobs.
+ */
+
+public class TestNativeAzureFileSystemBlockCompaction extends AbstractWasbTestBase {
+
+ private static final String TEST_FILE = "/user/active/test.dat";
+ private static final Path TEST_PATH = new Path(TEST_FILE);
+
+ private static final String TEST_FILE_NORMAL = "/user/normal/test.dat";
+ private static final Path TEST_PATH_NORMAL = new Path(TEST_FILE_NORMAL);
+
+ private AzureBlobStorageTestAccount testAccount = null;
+
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ testAccount = createTestAccount();
+ fs = testAccount.getFileSystem();
+ Configuration conf = fs.getConf();
+ conf.setBoolean(NativeAzureFileSystem.APPEND_SUPPORT_ENABLE_PROPERTY_NAME, true);
+ conf.set(AzureNativeFileSystemStore.KEY_BLOCK_BLOB_WITH_COMPACTION_DIRECTORIES, "/user/active");
+ URI uri = fs.getUri();
+ fs.initialize(uri, conf);
+ }
+
+ /*
+ * Helper method that creates test data of size provided by the
+ * "size" parameter.
+ */
+ private static byte[] getTestData(int size) {
+ byte[] testData = new byte[size];
+ System.arraycopy(RandomStringUtils.randomAlphabetic(size).getBytes(), 0, testData, 0, size);
+ return testData;
+ }
+
+ @Override
+ protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
+ return AzureBlobStorageTestAccount.create();
+ }
+
+ private BlockBlobAppendStream getBlockBlobAppendStream(FSDataOutputStream appendStream) {
+ SyncableDataOutputStream dataOutputStream = null;
+
+ if (appendStream.getWrappedStream() instanceof NativeAzureFileSystem.NativeAzureFsOutputStream) {
+ NativeAzureFileSystem.NativeAzureFsOutputStream fsOutputStream =
+ (NativeAzureFileSystem.NativeAzureFsOutputStream) appendStream.getWrappedStream();
+
+ dataOutputStream = (SyncableDataOutputStream) fsOutputStream.getOutStream();
+ }
+
+ if (appendStream.getWrappedStream() instanceof SyncableDataOutputStream) {
+ dataOutputStream = (SyncableDataOutputStream) appendStream.getWrappedStream();
+ }
+
+ Assert.assertNotNull("Did not recognize " + dataOutputStream,
+ dataOutputStream);
+
+ return (BlockBlobAppendStream) dataOutputStream.getOutStream();
+ }
+
+ private void verifyBlockList(BlockBlobAppendStream blockBlobStream,
+ int[] testData) throws Throwable {
+ List<BlockEntry> blockList = blockBlobStream.getBlockList();
+ Assert.assertEquals("Block list length", testData.length, blockList.size());
+
+ int i = 0;
+ for (BlockEntry block: blockList) {
+ Assert.assertTrue(block.getSize() == testData[i++]);
+ }
+ }
+
+ private void appendBlockList(FSDataOutputStream fsStream,
+ ByteArrayOutputStream memStream,
+ int[] testData) throws Throwable {
+
+ for (int d: testData) {
+ byte[] data = getTestData(d);
+ memStream.write(data);
+ fsStream.write(data);
+ }
+ fsStream.hflush();
+ }
+
+ @Test
+ public void testCompactionDisabled() throws Throwable {
+
+ try (FSDataOutputStream appendStream = fs.create(TEST_PATH_NORMAL)) {
+
+ // testing new file
+
+ SyncableDataOutputStream dataOutputStream = null;
+
+ OutputStream wrappedStream = appendStream.getWrappedStream();
+ if (wrappedStream instanceof NativeAzureFileSystem.NativeAzureFsOutputStream) {
+ NativeAzureFileSystem.NativeAzureFsOutputStream fsOutputStream =
+ (NativeAzureFileSystem.NativeAzureFsOutputStream) wrappedStream;
+
+ dataOutputStream = (SyncableDataOutputStream) fsOutputStream.getOutStream();
+ } else if (wrappedStream instanceof SyncableDataOutputStream) {
+ dataOutputStream = (SyncableDataOutputStream) wrappedStream;
+ } else {
+ Assert.fail("Unable to determine type of " + wrappedStream
+ + " class of " + wrappedStream.getClass());
+ }
+
+ Assert.assertFalse("Data output stream is a BlockBlobAppendStream: "
+ + dataOutputStream,
+ dataOutputStream.getOutStream() instanceof BlockBlobAppendStream);
+
+ }
+ }
+
+ @Test
+ public void testCompaction() throws Throwable {
+
+ final int n2 = 2;
+ final int n4 = 4;
+ final int n10 = 10;
+ final int n12 = 12;
+ final int n14 = 14;
+ final int n16 = 16;
+
+ final int maxBlockSize = 16;
+ final int compactionBlockCount = 4;
+
+ ByteArrayOutputStream memStream = new ByteArrayOutputStream();
+
+ try (FSDataOutputStream appendStream = fs.create(TEST_PATH)) {
+
+ // test new file
+
+ BlockBlobAppendStream blockBlobStream = getBlockBlobAppendStream(appendStream);
+ blockBlobStream.setMaxBlockSize(maxBlockSize);
+ blockBlobStream.setCompactionBlockCount(compactionBlockCount);
+
+ appendBlockList(appendStream, memStream, new int[]{n2});
+ verifyBlockList(blockBlobStream, new int[]{n2});
+
+ appendStream.hflush();
+ verifyBlockList(blockBlobStream, new int[]{n2});
+
+ appendBlockList(appendStream, memStream, new int[]{n4});
+ verifyBlockList(blockBlobStream, new int[]{n2, n4});
+
+ appendStream.hsync();
+ verifyBlockList(blockBlobStream, new int[]{n2, n4});
+
+ appendBlockList(appendStream, memStream, new int[]{n4});
+ verifyBlockList(blockBlobStream, new int[]{n2, n4, n4});
+
+ appendBlockList(appendStream, memStream, new int[]{n4});
+ verifyBlockList(blockBlobStream, new int[]{n2, n4, n4, n4});
+
+ appendBlockList(appendStream, memStream, new int[]{n4});
+ verifyBlockList(blockBlobStream, new int[]{n14, n4});
+
+ appendBlockList(appendStream, memStream, new int[]{n4});
+ verifyBlockList(blockBlobStream, new int[]{n14, n4, n4});
+
+ appendBlockList(appendStream, memStream, new int[]{n4});
+ verifyBlockList(blockBlobStream, new int[]{n14, n4, n4, n4});
+
+ appendBlockList(appendStream, memStream, new int[]{n2, n4, n4});
+ verifyBlockList(blockBlobStream, new int[]{n14, n12, n10});
+
+ appendBlockList(appendStream, memStream, new int[]{n4});
+ verifyBlockList(blockBlobStream, new int[]{n14, n12, n10, n4});
+
+ appendBlockList(appendStream, memStream,
+ new int[]{n4, n4, n4, n4});
+ verifyBlockList(blockBlobStream,
+ new int[]{n14, n12, n14, n16});
+
+ appendBlockList(appendStream, memStream,
+ new int[]{n4, n4, n4, n4, n4});
+ verifyBlockList(blockBlobStream,
+ new int[]{n14, n12, n14, n16, n16, n4});
+
+ appendBlockList(appendStream, memStream,
+ new int[]{n4});
+ verifyBlockList(blockBlobStream,
+ new int[]{n14, n12, n14, n16, n16, n4, n4});
+
+ appendBlockList(appendStream, memStream,
+ new int[]{n4});
+ verifyBlockList(blockBlobStream,
+ new int[]{n14, n12, n14, n16, n16, n4, n4, n4});
+
+ appendBlockList(appendStream, memStream,
+ new int[]{n4});
+ verifyBlockList(blockBlobStream,
+ new int[]{n14, n12, n14, n16, n16, n4, n4, n4, n4});
+
+ appendBlockList(appendStream, memStream, new int[]{n4});
+
+ appendStream.close();
+
+ ContractTestUtils.verifyFileContents(fs, TEST_PATH, memStream.toByteArray());
+ }
+
+ try (FSDataOutputStream appendStream = fs.append(TEST_PATH)) {
+
+ // test existing file
+
+ BlockBlobAppendStream blockBlobStream = getBlockBlobAppendStream(appendStream);
+ blockBlobStream.setMaxBlockSize(maxBlockSize);
+ blockBlobStream.setCompactionBlockCount(compactionBlockCount);
+
+ appendBlockList(appendStream, memStream, new int[]{n4});
+ verifyBlockList(blockBlobStream,
+ new int[]{n14, n12, n14, n16, n16, n16, n4, n4});
+
+ appendBlockList(appendStream, memStream, new int[]{n4});
+ verifyBlockList(blockBlobStream,
+ new int[]{n14, n12, n14, n16, n16, n16, n4, n4, n4});
+
+ appendBlockList(appendStream, memStream, new int[]{n4});
+ verifyBlockList(blockBlobStream,
+ new int[]{n14, n12, n14, n16, n16, n16, n4, n4, n4, n4});
+
+ appendBlockList(appendStream, memStream, new int[]{n4});
+ verifyBlockList(blockBlobStream,
+ new int[]{n14, n12, n14, n16, n16, n16, n16, n4});
+
+ appendStream.close();
+
+ ContractTestUtils.verifyFileContents(fs, TEST_PATH, memStream.toByteArray());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13eda500/hadoop-tools/hadoop-azure/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/resources/log4j.properties b/hadoop-tools/hadoop-azure/src/test/resources/log4j.properties
index 73ee3f9..a5e0c4f 100644
--- a/hadoop-tools/hadoop-azure/src/test/resources/log4j.properties
+++ b/hadoop-tools/hadoop-azure/src/test/resources/log4j.properties
@@ -23,3 +23,4 @@ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n
log4j.logger.org.apache.hadoop.fs.azure.AzureFileSystemThreadPoolExecutor=DEBUG
+log4j.logger.org.apache.hadoop.fs.azure.BlockBlobAppendStream=DEBUG
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/40] hadoop git commit: YARN-5328. Plan/ResourceAllocation data
structure enhancements required to support recurring reservations in
ReservationSystem.
Posted by in...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e7d136/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java
index bc98e2f..03569d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.planning.Planne
import org.apache.hadoop.yarn.server.resourcemanager.reservation.planning.ReservationAgent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.UTCClock;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.junit.After;
@@ -47,6 +48,10 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+/**
+ * Testing the class {@link InMemoryPlan}.
+ */
+@SuppressWarnings("checkstyle:nowhitespaceafter")
public class TestInMemoryPlan {
private String user = "yarn";
@@ -62,6 +67,7 @@ public class TestInMemoryPlan {
private ReservationAgent agent;
private Planner replanner;
private RMContext context;
+ private long maxPeriodicity;
@Before
public void setUp() throws PlanningException {
@@ -72,7 +78,7 @@ public class TestInMemoryPlan {
clock = mock(Clock.class);
queueMetrics = mock(QueueMetrics.class);
- policy = mock(SharingPolicy.class);
+ policy = new NoOverCommitPolicy();
replanner = mock(Planner.class);
when(clock.getTime()).thenReturn(1L);
@@ -95,15 +101,41 @@ public class TestInMemoryPlan {
@Test
public void testAddReservation() {
- Plan plan =
- new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
ReservationId reservationID =
- ReservationSystemTestUtil.getNewReservationId();
+ ReservationSystemTestUtil.getNewReservationId();
int[] alloc = { 10, 10, 10, 10, 10, 10 };
int start = 100;
- ReservationAllocation rAllocation = createReservationAllocation
- (reservationID, start, alloc);
+ ReservationAllocation rAllocation =
+ createReservationAllocation(reservationID, start, alloc);
+ Assert.assertNull(plan.getReservationById(reservationID));
+ try {
+ plan.addReservation(rAllocation, false);
+ } catch (PlanningException e) {
+ Assert.fail(e.getMessage());
+ }
+ doAssertions(plan, rAllocation);
+ checkAllocation(plan, alloc, start, 0);
+ }
+
+ @Test
+ public void testAddPeriodicReservation() throws PlanningException {
+
+ maxPeriodicity = 100;
+ Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, maxPeriodicity,
+ context, new UTCClock());
+
+ ReservationId reservationID =
+ ReservationSystemTestUtil.getNewReservationId();
+ int[] alloc = { 10, 50 };
+ int start = 10;
+ long period = 20;
+ ReservationAllocation rAllocation = createReservationAllocation(
+ reservationID, start, alloc, String.valueOf(period));
+ // use periodicity of 1hr
+ rAllocation.setPeriodicity(period);
Assert.assertNull(plan.getReservationById(reservationID));
try {
plan.addReservation(rAllocation, false);
@@ -111,32 +143,54 @@ public class TestInMemoryPlan {
Assert.fail(e.getMessage());
}
doAssertions(plan, rAllocation);
- checkAllocation(plan, alloc, start);
+ checkAllocation(plan, alloc, start, period);
+
+ RLESparseResourceAllocation available =
+ plan.getAvailableResourceOverTime(user, reservationID, 150, 330, 50);
+ System.out.println(available);
}
- private void checkAllocation(Plan plan, int[] alloc, int start) {
+ private void checkAllocation(Plan plan, int[] alloc, int start,
+ long periodicity) {
+ long end = start + alloc.length;
+ if (periodicity > 0) {
+ end = end + maxPeriodicity;
+ }
RLESparseResourceAllocation userCons =
- plan.getConsumptionForUserOverTime(user, start, start + alloc.length);
+ plan.getConsumptionForUserOverTime(user, start, end * 3);
for (int i = 0; i < alloc.length; i++) {
- Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])),
- plan.getTotalCommittedResources(start + i));
- Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])),
- userCons.getCapacityAtTime(start + i));
+ // only one instance for non-periodic reservation
+ if (periodicity <= 0) {
+ Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])),
+ plan.getTotalCommittedResources(start + i));
+ Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])),
+ userCons.getCapacityAtTime(start + i));
+ } else {
+ // periodic reservations should repeat
+ long y = 0;
+ Resource res = Resource.newInstance(1024 * (alloc[i]), (alloc[i]));
+ while (y <= end * 2) {
+ Assert.assertEquals("At time: " + start + i + y, res,
+ plan.getTotalCommittedResources(start + i + y));
+ Assert.assertEquals(" At time: " + (start + i + y), res,
+ userCons.getCapacityAtTime(start + i + y));
+ y = y + periodicity;
+ }
+ }
}
}
@Test
public void testAddEmptyReservation() {
- Plan plan =
- new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
ReservationId reservationID =
- ReservationSystemTestUtil.getNewReservationId();
+ ReservationSystemTestUtil.getNewReservationId();
int[] alloc = {};
int start = 100;
- ReservationAllocation rAllocation = createReservationAllocation
- (reservationID, start, alloc);
+ ReservationAllocation rAllocation =
+ createReservationAllocation(reservationID, start, alloc);
Assert.assertNull(plan.getReservationById(reservationID));
try {
plan.addReservation(rAllocation, false);
@@ -148,15 +202,14 @@ public class TestInMemoryPlan {
@Test
public void testAddReservationAlreadyExists() {
// First add a reservation
- Plan plan =
- new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
ReservationId reservationID =
- ReservationSystemTestUtil.getNewReservationId();
+ ReservationSystemTestUtil.getNewReservationId();
int[] alloc = { 10, 10, 10, 10, 10, 10 };
int start = 100;
- ReservationAllocation rAllocation = createReservationAllocation
- (reservationID, start, alloc);
+ ReservationAllocation rAllocation =
+ createReservationAllocation(reservationID, start, alloc);
Assert.assertNull(plan.getReservationById(reservationID));
try {
plan.addReservation(rAllocation, false);
@@ -164,7 +217,7 @@ public class TestInMemoryPlan {
Assert.fail(e.getMessage());
}
doAssertions(plan, rAllocation);
- checkAllocation(plan, alloc, start);
+ checkAllocation(plan, alloc, start, 0);
// Try to add it again
try {
@@ -180,16 +233,15 @@ public class TestInMemoryPlan {
@Test
public void testUpdateReservation() {
- Plan plan =
- new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
ReservationId reservationID =
ReservationSystemTestUtil.getNewReservationId();
// First add a reservation
int[] alloc = { 10, 10, 10, 10, 10, 10 };
int start = 100;
- ReservationAllocation rAllocation = createReservationAllocation
- (reservationID, start, alloc);
+ ReservationAllocation rAllocation =
+ createReservationAllocation(reservationID, start, alloc);
Assert.assertNull(plan.getReservationById(reservationID));
try {
plan.addReservation(rAllocation, false);
@@ -210,8 +262,8 @@ public class TestInMemoryPlan {
// Now update it
start = 110;
int[] updatedAlloc = { 0, 5, 10, 10, 5, 0 };
- rAllocation = createReservationAllocation(reservationID, start,
- updatedAlloc, true);
+ rAllocation =
+ createReservationAllocation(reservationID, start, updatedAlloc, true);
try {
plan.updateReservation(rAllocation);
} catch (PlanningException e) {
@@ -219,32 +271,71 @@ public class TestInMemoryPlan {
}
doAssertions(plan, rAllocation);
- userCons =
- plan.getConsumptionForUserOverTime(user, start, start
- + updatedAlloc.length);
+ userCons = plan.getConsumptionForUserOverTime(user, start,
+ start + updatedAlloc.length);
for (int i = 0; i < updatedAlloc.length; i++) {
- Assert.assertEquals(
- Resource.newInstance(1024 * (updatedAlloc[i] + i), updatedAlloc[i]
- + i), plan.getTotalCommittedResources(start + i));
- Assert.assertEquals(
- Resource.newInstance(1024 * (updatedAlloc[i] + i), updatedAlloc[i]
- + i), userCons.getCapacityAtTime(start + i));
+ Assert.assertEquals(Resource.newInstance(1024 * (updatedAlloc[i] + i),
+ updatedAlloc[i] + i), plan.getTotalCommittedResources(start + i));
+ Assert.assertEquals(Resource.newInstance(1024 * (updatedAlloc[i] + i),
+ updatedAlloc[i] + i), userCons.getCapacityAtTime(start + i));
+ }
+ }
+
+ @Test
+ public void testUpdatePeriodicReservation() {
+ Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ // First add a reservation
+ ReservationId reservationID =
+ ReservationSystemTestUtil.getNewReservationId();
+ int[] alloc = { 10, 20 };
+ int start = 100;
+ ReservationAllocation rAllocation =
+ createReservationAllocation(reservationID, start, alloc);
+ // use periodicity of 1hr
+ long period = 3600000;
+ rAllocation.getReservationDefinition()
+ .setRecurrenceExpression(String.valueOf(period));
+ rAllocation.setPeriodicity(period);
+ Assert.assertNull(plan.getReservationById(reservationID));
+ try {
+ plan.addReservation(rAllocation, false);
+ } catch (PlanningException e) {
+ Assert.fail(e.getMessage());
+ }
+ System.out.println(plan.toString());
+ doAssertions(plan, rAllocation);
+ checkAllocation(plan, alloc, start, period);
+
+ // Now update it
+ start = 110;
+ int[] updatedAlloc = { 30, 40, 50 };
+ rAllocation =
+ createReservationAllocation(reservationID, start, updatedAlloc);
+ rAllocation.getReservationDefinition()
+ .setRecurrenceExpression(String.valueOf(period));
+ rAllocation.setPeriodicity(period);
+ try {
+ plan.updateReservation(rAllocation);
+ } catch (PlanningException e) {
+ Assert.fail(e.getMessage());
}
+ doAssertions(plan, rAllocation);
+ checkAllocation(plan, updatedAlloc, start, period);
}
@Test
public void testUpdateNonExistingReservation() {
- Plan plan =
- new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
ReservationId reservationID =
ReservationSystemTestUtil.getNewReservationId();
// Try to update a reservation without adding
int[] alloc = { 10, 10, 10, 10, 10, 10 };
int start = 100;
ReservationAllocation rAllocation =
- createReservationAllocation(reservationID, start, alloc);
+ createReservationAllocation(reservationID, start, alloc);
Assert.assertNull(plan.getReservationById(reservationID));
try {
plan.updateReservation(rAllocation);
@@ -260,15 +351,14 @@ public class TestInMemoryPlan {
@Test
public void testDeleteReservation() {
// First add a reservation
- Plan plan =
- new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
ReservationId reservationID =
ReservationSystemTestUtil.getNewReservationId();
int[] alloc = { 10, 10, 10, 10, 10, 10 };
int start = 100;
ReservationAllocation rAllocation =
- createReservationAllocation(reservationID, start, alloc, true);
+ createReservationAllocation(reservationID, start, alloc, true);
Assert.assertNull(plan.getReservationById(reservationID));
try {
plan.addReservation(rAllocation, false);
@@ -307,10 +397,46 @@ public class TestInMemoryPlan {
}
@Test
+ public void testDeletePeriodicReservation() {
+ Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ // First add a reservation
+ ReservationId reservationID =
+ ReservationSystemTestUtil.getNewReservationId();
+ int[] alloc = { 10, 20 };
+ int start = 100;
+ ReservationAllocation rAllocation =
+ createReservationAllocation(reservationID, start, alloc);
+ // use periodicity of 1hr
+ long period = 3600000;
+ rAllocation.getReservationDefinition()
+ .setRecurrenceExpression(String.valueOf(period));
+ rAllocation.setPeriodicity(period);
+ Assert.assertNull(plan.getReservationById(reservationID));
+ try {
+ plan.addReservation(rAllocation, false);
+ } catch (PlanningException e) {
+ Assert.fail(e.getMessage());
+ }
+ System.out.println(plan.toString());
+ doAssertions(plan, rAllocation);
+ checkAllocation(plan, alloc, start, period);
+
+ // Now delete it
+ try {
+ plan.deleteReservation(reservationID);
+ } catch (PlanningException e) {
+ Assert.fail(e.getMessage());
+ }
+ Assert.assertNull(plan.getReservationById(reservationID));
+ System.out.print(plan);
+ checkAllocation(plan, new int[] { 0, 0 }, start, period);
+ }
+
+ @Test
public void testDeleteNonExistingReservation() {
- Plan plan =
- new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
ReservationId reservationID =
ReservationSystemTestUtil.getNewReservationId();
// Try to delete a reservation without adding
@@ -328,8 +454,9 @@ public class TestInMemoryPlan {
@Test
public void testArchiveCompletedReservations() {
+ SharingPolicy sharingPolicy = mock(SharingPolicy.class);
Plan plan =
- new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ new InMemoryPlan(queueMetrics, sharingPolicy, agent, totalCapacity, 1L,
resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
ReservationId reservationID1 =
ReservationSystemTestUtil.getNewReservationId();
@@ -337,7 +464,7 @@ public class TestInMemoryPlan {
int[] alloc1 = { 10, 10, 10, 10, 10, 10 };
int start = 100;
ReservationAllocation rAllocation =
- createReservationAllocation(reservationID1, start, alloc1);
+ createReservationAllocation(reservationID1, start, alloc1);
Assert.assertNull(plan.getReservationById(reservationID1));
try {
plan.addReservation(rAllocation, false);
@@ -345,15 +472,14 @@ public class TestInMemoryPlan {
Assert.fail(e.getMessage());
}
doAssertions(plan, rAllocation);
- checkAllocation(plan, alloc1, start);
-
+ checkAllocation(plan, alloc1, start, 0);
// Now add another one
ReservationId reservationID2 =
ReservationSystemTestUtil.getNewReservationId();
int[] alloc2 = { 0, 5, 10, 5, 0 };
rAllocation =
- createReservationAllocation(reservationID2, start, alloc2, true);
+ createReservationAllocation(reservationID2, start, alloc2, true);
Assert.assertNull(plan.getReservationById(reservationID2));
try {
plan.addReservation(rAllocation, false);
@@ -367,16 +493,18 @@ public class TestInMemoryPlan {
for (int i = 0; i < alloc2.length; i++) {
Assert.assertEquals(
- Resource.newInstance(1024 * (alloc1[i] + alloc2[i] + i), alloc1[i]
- + alloc2[i] + i), plan.getTotalCommittedResources(start + i));
+ Resource.newInstance(1024 * (alloc1[i] + alloc2[i] + i),
+ alloc1[i] + alloc2[i] + i),
+ plan.getTotalCommittedResources(start + i));
Assert.assertEquals(
- Resource.newInstance(1024 * (alloc1[i] + alloc2[i] + i), alloc1[i]
- + alloc2[i] + i), userCons.getCapacityAtTime(start + i));
+ Resource.newInstance(1024 * (alloc1[i] + alloc2[i] + i),
+ alloc1[i] + alloc2[i] + i),
+ userCons.getCapacityAtTime(start + i));
}
// Now archive completed reservations
when(clock.getTime()).thenReturn(106L);
- when(policy.getValidWindow()).thenReturn(1L);
+ when(sharingPolicy.getValidWindow()).thenReturn(1L);
try {
// will only remove 2nd reservation as only that has fallen out of the
// archival window
@@ -386,7 +514,7 @@ public class TestInMemoryPlan {
}
Assert.assertNotNull(plan.getReservationById(reservationID1));
Assert.assertNull(plan.getReservationById(reservationID2));
- checkAllocation(plan, alloc1, start);
+ checkAllocation(plan, alloc1, start, 0);
when(clock.getTime()).thenReturn(107L);
try {
@@ -411,15 +539,14 @@ public class TestInMemoryPlan {
@Test
public void testGetReservationsById() {
- Plan plan =
- new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
ReservationId reservationID =
- ReservationSystemTestUtil.getNewReservationId();
- int[] alloc = {10, 10, 10, 10, 10, 10};
+ ReservationSystemTestUtil.getNewReservationId();
+ int[] alloc = { 10, 10, 10, 10, 10, 10 };
int start = 100;
- ReservationAllocation rAllocation = createReservationAllocation
- (reservationID, start, alloc);
+ ReservationAllocation rAllocation =
+ createReservationAllocation(reservationID, start, alloc);
Assert.assertNull(plan.getReservationById(reservationID));
try {
plan.addReservation(rAllocation, false);
@@ -429,31 +556,30 @@ public class TestInMemoryPlan {
// Verify that get by reservation id works.
Set<ReservationAllocation> rAllocations =
- plan.getReservations(reservationID, null, "");
+ plan.getReservations(reservationID, null, "");
Assert.assertTrue(rAllocations.size() == 1);
- Assert.assertTrue(rAllocation.compareTo(
- (ReservationAllocation) rAllocations.toArray()[0]) == 0);
+ Assert.assertTrue(rAllocation
+ .compareTo((ReservationAllocation) rAllocations.toArray()[0]) == 0);
// Verify that get by reservation id works even when time range
// and user is invalid.
ReservationInterval interval = new ReservationInterval(0, 0);
rAllocations = plan.getReservations(reservationID, interval, "invalid");
Assert.assertTrue(rAllocations.size() == 1);
- Assert.assertTrue(rAllocation.compareTo(
- (ReservationAllocation) rAllocations.toArray()[0]) == 0);
+ Assert.assertTrue(rAllocation
+ .compareTo((ReservationAllocation) rAllocations.toArray()[0]) == 0);
}
@Test
public void testGetReservationsByInvalidId() {
- Plan plan =
- new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
ReservationId reservationID =
- ReservationSystemTestUtil.getNewReservationId();
- int[] alloc = {10, 10, 10, 10, 10, 10};
+ ReservationSystemTestUtil.getNewReservationId();
+ int[] alloc = { 10, 10, 10, 10, 10, 10 };
int start = 100;
- ReservationAllocation rAllocation = createReservationAllocation
- (reservationID, start, alloc);
+ ReservationAllocation rAllocation =
+ createReservationAllocation(reservationID, start, alloc);
Assert.assertNull(plan.getReservationById(reservationID));
try {
plan.addReservation(rAllocation, false);
@@ -463,23 +589,22 @@ public class TestInMemoryPlan {
// If reservationId is null, then nothing is returned.
ReservationId invalidReservationID =
- ReservationSystemTestUtil.getNewReservationId();
+ ReservationSystemTestUtil.getNewReservationId();
Set<ReservationAllocation> rAllocations =
- plan.getReservations(invalidReservationID, null, "");
+ plan.getReservations(invalidReservationID, null, "");
Assert.assertTrue(rAllocations.size() == 0);
}
@Test
public void testGetReservationsByTimeInterval() {
- Plan plan =
- new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
ReservationId reservationID =
- ReservationSystemTestUtil.getNewReservationId();
- int[] alloc = {10, 10, 10, 10, 10, 10};
+ ReservationSystemTestUtil.getNewReservationId();
+ int[] alloc = { 10, 10, 10, 10, 10, 10 };
int start = 100;
- ReservationAllocation rAllocation = createReservationAllocation
- (reservationID, start, alloc);
+ ReservationAllocation rAllocation =
+ createReservationAllocation(reservationID, start, alloc);
Assert.assertNull(plan.getReservationById(reservationID));
try {
plan.addReservation(rAllocation, false);
@@ -489,23 +614,24 @@ public class TestInMemoryPlan {
// Verify that get by time interval works if the selection interval
// completely overlaps with the allocation.
- ReservationInterval interval = new ReservationInterval(rAllocation
- .getStartTime(), rAllocation.getEndTime());
+ ReservationInterval interval = new ReservationInterval(
+ rAllocation.getStartTime(), rAllocation.getEndTime());
Set<ReservationAllocation> rAllocations =
- plan.getReservations(null, interval, "");
+ plan.getReservations(null, interval, "");
Assert.assertTrue(rAllocations.size() == 1);
- Assert.assertTrue(rAllocation.compareTo(
- (ReservationAllocation) rAllocations.toArray()[0]) == 0);
+ Assert.assertTrue(rAllocation
+ .compareTo((ReservationAllocation) rAllocations.toArray()[0]) == 0);
// Verify that get by time interval works if the selection interval
// falls within the allocation
long duration = rAllocation.getEndTime() - rAllocation.getStartTime();
- interval = new ReservationInterval(rAllocation.getStartTime() + duration
- * (long)0.3, rAllocation.getEndTime() - duration * (long)0.3);
+ interval = new ReservationInterval(
+ rAllocation.getStartTime() + duration * (long) 0.3,
+ rAllocation.getEndTime() - duration * (long) 0.3);
rAllocations = plan.getReservations(null, interval, "");
Assert.assertTrue(rAllocations.size() == 1);
- Assert.assertTrue(rAllocation.compareTo(
- (ReservationAllocation) rAllocations.toArray()[0]) == 0);
+ Assert.assertTrue(rAllocation
+ .compareTo((ReservationAllocation) rAllocations.toArray()[0]) == 0);
// Verify that get by time interval selects 1 allocation if the end
// time of the selection interval falls right at the start of the
@@ -513,13 +639,13 @@ public class TestInMemoryPlan {
interval = new ReservationInterval(0, rAllocation.getStartTime());
rAllocations = plan.getReservations(null, interval, "");
Assert.assertTrue(rAllocations.size() == 1);
- Assert.assertTrue(rAllocation.compareTo(
- (ReservationAllocation) rAllocations.toArray()[0]) == 0);
+ Assert.assertTrue(rAllocation
+ .compareTo((ReservationAllocation) rAllocations.toArray()[0]) == 0);
// Verify that get by time interval selects no reservations if the start
// time of the selection interval falls right at the end of the allocation.
- interval = new ReservationInterval(rAllocation
- .getEndTime(), Long.MAX_VALUE);
+ interval =
+ new ReservationInterval(rAllocation.getEndTime(), Long.MAX_VALUE);
rAllocations = plan.getReservations(null, interval, "");
Assert.assertTrue(rAllocations.size() == 0);
@@ -532,15 +658,14 @@ public class TestInMemoryPlan {
@Test
public void testGetReservationsAtTime() {
- Plan plan =
- new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
ReservationId reservationID =
- ReservationSystemTestUtil.getNewReservationId();
- int[] alloc = {10, 10, 10, 10, 10, 10};
+ ReservationSystemTestUtil.getNewReservationId();
+ int[] alloc = { 10, 10, 10, 10, 10, 10 };
int start = 100;
- ReservationAllocation rAllocation = createReservationAllocation
- (reservationID, start, alloc);
+ ReservationAllocation rAllocation =
+ createReservationAllocation(reservationID, start, alloc);
Assert.assertNull(plan.getReservationById(reservationID));
try {
plan.addReservation(rAllocation, false);
@@ -549,23 +674,22 @@ public class TestInMemoryPlan {
}
Set<ReservationAllocation> rAllocations =
- plan.getReservationsAtTime(rAllocation.getStartTime());
+ plan.getReservationsAtTime(rAllocation.getStartTime());
Assert.assertTrue(rAllocations.size() == 1);
- Assert.assertTrue(rAllocation.compareTo(
- (ReservationAllocation) rAllocations.toArray()[0]) == 0);
+ Assert.assertTrue(rAllocation
+ .compareTo((ReservationAllocation) rAllocations.toArray()[0]) == 0);
}
@Test
public void testGetReservationsWithNoInput() {
- Plan plan =
- new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
ReservationId reservationID =
- ReservationSystemTestUtil.getNewReservationId();
- int[] alloc = {10, 10, 10, 10, 10, 10};
+ ReservationSystemTestUtil.getNewReservationId();
+ int[] alloc = { 10, 10, 10, 10, 10, 10 };
int start = 100;
- ReservationAllocation rAllocation = createReservationAllocation
- (reservationID, start, alloc);
+ ReservationAllocation rAllocation =
+ createReservationAllocation(reservationID, start, alloc);
Assert.assertNull(plan.getReservationById(reservationID));
try {
plan.addReservation(rAllocation, false);
@@ -576,22 +700,21 @@ public class TestInMemoryPlan {
// Verify that getReservations defaults to getting all reservations if no
// reservationID, time interval, and user is provided,
Set<ReservationAllocation> rAllocations =
- plan.getReservations(null, null, "");
+ plan.getReservations(null, null, "");
Assert.assertTrue(rAllocations.size() == 1);
- Assert.assertTrue(rAllocation.compareTo(
- (ReservationAllocation) rAllocations.toArray()[0]) == 0);
+ Assert.assertTrue(rAllocation
+ .compareTo((ReservationAllocation) rAllocations.toArray()[0]) == 0);
}
@Test
public void testGetReservationsWithNoReservation() {
- Plan plan =
- new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
- resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
+ Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
// Verify that get reservation returns no entries if no queries are made.
ReservationInterval interval = new ReservationInterval(0, Long.MAX_VALUE);
Set<ReservationAllocation> rAllocations =
- plan.getReservations(null, interval, "");
+ plan.getReservations(null, interval, "");
Assert.assertTrue(rAllocations.size() == 0);
}
@@ -600,7 +723,9 @@ public class TestInMemoryPlan {
Assert.assertNotNull(plan.getReservationById(reservationID));
Assert.assertEquals(rAllocation, plan.getReservationById(reservationID));
Assert.assertTrue(((InMemoryPlan) plan).getAllReservations().size() == 1);
- Assert.assertEquals(rAllocation.getEndTime(), plan.getLastEndTime());
+ if (rAllocation.getPeriodicity() <= 0) {
+ Assert.assertEquals(rAllocation.getEndTime(), plan.getLastEndTime());
+ }
Assert.assertEquals(totalCapacity, plan.getTotalCapacity());
Assert.assertEquals(minAlloc, plan.getMinimumAllocation());
Assert.assertEquals(maxAlloc, plan.getMaximumAllocation());
@@ -610,7 +735,8 @@ public class TestInMemoryPlan {
}
private ReservationDefinition createSimpleReservationDefinition(long arrival,
- long deadline, long duration, Collection<ReservationRequest> resources) {
+ long deadline, long duration, Collection<ReservationRequest> resources,
+ String recurrenceExpression) {
// create a request with a single atomic ask
ReservationDefinition rDef = new ReservationDefinitionPBImpl();
ReservationRequests reqs = new ReservationRequestsPBImpl();
@@ -619,6 +745,7 @@ public class TestInMemoryPlan {
rDef.setReservationRequests(reqs);
rDef.setArrival(arrival);
rDef.setDeadline(deadline);
+ rDef.setRecurrenceExpression(recurrenceExpression);
return rDef;
}
@@ -633,31 +760,43 @@ public class TestInMemoryPlan {
} else {
numContainers = alloc[i];
}
- ReservationRequest rr =
- ReservationRequest.newInstance(Resource.newInstance(1024, 1),
- (numContainers));
+ ReservationRequest rr = ReservationRequest
+ .newInstance(Resource.newInstance(1024, 1), (numContainers));
req.put(new ReservationInterval(startTime + i, startTime + i + 1), rr);
}
return req;
}
- private ReservationAllocation createReservationAllocation(ReservationId
- reservationID, int start, int[] alloc) {
- return createReservationAllocation(reservationID, start, alloc, false);
+ private ReservationAllocation createReservationAllocation(
+ ReservationId reservationID, int start, int[] alloc) {
+ return createReservationAllocation(reservationID, start, alloc, false, "0");
+ }
+
+ private ReservationAllocation createReservationAllocation(
+ ReservationId reservationID, int start, int[] alloc, boolean isStep) {
+ return createReservationAllocation(reservationID, start, alloc, isStep,
+ "0");
+ }
+
+ private ReservationAllocation createReservationAllocation(
+ ReservationId reservationID, int start, int[] alloc,
+ String recurrenceExp) {
+ return createReservationAllocation(reservationID, start, alloc, false,
+ recurrenceExp);
}
- private ReservationAllocation createReservationAllocation(ReservationId
- reservationID, int start, int[] alloc, boolean isStep) {
+ private ReservationAllocation createReservationAllocation(
+ ReservationId reservationID, int start, int[] alloc, boolean isStep,
+ String recurrenceExp) {
Map<ReservationInterval, ReservationRequest> allocations =
- generateAllocation(start, alloc, isStep);
+ generateAllocation(start, alloc, isStep);
ReservationDefinition rDef =
- createSimpleReservationDefinition(start, start + alloc.length,
- alloc.length, allocations.values());
+ createSimpleReservationDefinition(start, start + alloc.length,
+ alloc.length, allocations.values(), recurrenceExp);
Map<ReservationInterval, Resource> allocs =
- ReservationSystemUtil.toResources(allocations);
+ ReservationSystemUtil.toResources(allocations);
return new InMemoryReservationAllocation(reservationID, rDef, user,
- planName,
- start, start + alloc.length, allocs, resCalc, minAlloc);
+ planName, start, start + alloc.length, allocs, resCalc, minAlloc);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e7d136/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestPeriodicRLESparseResourceAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestPeriodicRLESparseResourceAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestPeriodicRLESparseResourceAllocation.java
index 554eb58..457e2ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestPeriodicRLESparseResourceAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestPeriodicRLESparseResourceAllocation.java
@@ -19,26 +19,27 @@
package org.apache.hadoop.yarn.server.resourcemanager.reservation;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Testing the class PeriodicRLESparseResourceAllocation.
+ * Testing the class {@link PeriodicRLESparseResourceAllocation}.
*/
+@SuppressWarnings("checkstyle:nowhitespaceafter")
public class TestPeriodicRLESparseResourceAllocation {
- private static final Logger LOG = LoggerFactory
- .getLogger(TestPeriodicRLESparseResourceAllocation.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestPeriodicRLESparseResourceAllocation.class);
@Test
public void testPeriodicCapacity() {
- int[] alloc = {10, 7, 5, 2, 0};
- long[] timeSteps = {0L, 5L, 10L, 15L, 19L};
- RLESparseResourceAllocation rleSparseVector =
- ReservationSystemTestUtil.generateRLESparseResourceAllocation(
- alloc, timeSteps);
+ int[] alloc = { 10, 7, 5, 2, 0 };
+ long[] timeSteps = { 0L, 5L, 10L, 15L, 19L };
+ RLESparseResourceAllocation rleSparseVector = ReservationSystemTestUtil
+ .generateRLESparseResourceAllocation(alloc, timeSteps);
PeriodicRLESparseResourceAllocation periodicVector =
new PeriodicRLESparseResourceAllocation(rleSparseVector, 20L);
LOG.info(periodicVector.toString());
@@ -54,43 +55,74 @@ public class TestPeriodicRLESparseResourceAllocation {
@Test
public void testMaxPeriodicCapacity() {
- int[] alloc = {2, 5, 7, 10, 3, 4, 6, 8};
- long[] timeSteps = {0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L};
- RLESparseResourceAllocation rleSparseVector =
- ReservationSystemTestUtil.generateRLESparseResourceAllocation(
- alloc, timeSteps);
+ int[] alloc = { 2, 5, 7, 10, 3, 4, 6, 8 };
+ long[] timeSteps = { 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L };
+ RLESparseResourceAllocation rleSparseVector = ReservationSystemTestUtil
+ .generateRLESparseResourceAllocation(alloc, timeSteps);
PeriodicRLESparseResourceAllocation periodicVector =
new PeriodicRLESparseResourceAllocation(rleSparseVector, 8L);
LOG.info(periodicVector.toString());
- Assert.assertEquals(
- periodicVector.getMaximumPeriodicCapacity(0, 1),
+ Assert.assertEquals(periodicVector.getMaximumPeriodicCapacity(0, 1),
Resource.newInstance(10, 10));
- Assert.assertEquals(
- periodicVector.getMaximumPeriodicCapacity(8, 2),
+ Assert.assertEquals(periodicVector.getMaximumPeriodicCapacity(8, 2),
Resource.newInstance(7, 7));
- Assert.assertEquals(
- periodicVector.getMaximumPeriodicCapacity(16, 3),
+ Assert.assertEquals(periodicVector.getMaximumPeriodicCapacity(16, 3),
Resource.newInstance(10, 10));
- Assert.assertEquals(
- periodicVector.getMaximumPeriodicCapacity(17, 4),
+ Assert.assertEquals(periodicVector.getMaximumPeriodicCapacity(17, 4),
Resource.newInstance(5, 5));
- Assert.assertEquals(
- periodicVector.getMaximumPeriodicCapacity(32, 5),
+ Assert.assertEquals(periodicVector.getMaximumPeriodicCapacity(32, 5),
Resource.newInstance(4, 4));
}
@Test
+ public void testMixPeriodicAndNonPeriodic() throws PlanningException {
+ int[] alloc = { 2, 5, 0 };
+ long[] timeSteps = { 1L, 2L, 3L };
+ RLESparseResourceAllocation tempPeriodic = ReservationSystemTestUtil
+ .generateRLESparseResourceAllocation(alloc, timeSteps);
+ PeriodicRLESparseResourceAllocation periodic =
+ new PeriodicRLESparseResourceAllocation(tempPeriodic, 10L);
+
+ int[] alloc2 = { 10, 10, 0 };
+ long[] timeSteps2 = { 12L, 13L, 14L };
+ RLESparseResourceAllocation nonPeriodic = ReservationSystemTestUtil
+ .generateRLESparseResourceAllocation(alloc2, timeSteps2);
+
+ RLESparseResourceAllocation merged =
+ RLESparseResourceAllocation.merge(nonPeriodic.getResourceCalculator(),
+ Resource.newInstance(100 * 1024, 100), periodic, nonPeriodic,
+ RLESparseResourceAllocation.RLEOperator.add, 2, 25);
+
+ Assert.assertEquals(Resource.newInstance(5, 5),
+ merged.getCapacityAtTime(2L));
+ Assert.assertEquals(Resource.newInstance(0, 0),
+ merged.getCapacityAtTime(3L));
+ Assert.assertEquals(Resource.newInstance(2, 2),
+ merged.getCapacityAtTime(11L));
+ Assert.assertEquals(Resource.newInstance(15, 15),
+ merged.getCapacityAtTime(12L));
+ Assert.assertEquals(Resource.newInstance(10, 10),
+ merged.getCapacityAtTime(13L));
+ Assert.assertEquals(Resource.newInstance(0, 0),
+ merged.getCapacityAtTime(14L));
+ Assert.assertEquals(Resource.newInstance(2, 2),
+ merged.getCapacityAtTime(21L));
+ Assert.assertEquals(Resource.newInstance(5, 5),
+ merged.getCapacityAtTime(22L));
+ Assert.assertEquals(Resource.newInstance(0, 0),
+ merged.getCapacityAtTime(23L));
+ }
+
+ @Test
public void testSetCapacityInInterval() {
- int[] alloc = {2, 5, 0};
- long[] timeSteps = {1L, 2L, 3L};
- RLESparseResourceAllocation rleSparseVector =
- ReservationSystemTestUtil.generateRLESparseResourceAllocation(
- alloc, timeSteps);
+ int[] alloc = { 2, 5, 0 };
+ long[] timeSteps = { 1L, 2L, 3L };
+ RLESparseResourceAllocation rleSparseVector = ReservationSystemTestUtil
+ .generateRLESparseResourceAllocation(alloc, timeSteps);
PeriodicRLESparseResourceAllocation periodicVector =
new PeriodicRLESparseResourceAllocation(rleSparseVector, 10L);
ReservationInterval interval = new ReservationInterval(5L, 10L);
- periodicVector.addInterval(
- interval, Resource.newInstance(8, 8));
+ periodicVector.addInterval(interval, Resource.newInstance(8, 8));
Assert.assertEquals(Resource.newInstance(8, 8),
periodicVector.getCapacityAtTime(5L));
Assert.assertEquals(Resource.newInstance(8, 8),
@@ -99,21 +131,20 @@ public class TestPeriodicRLESparseResourceAllocation {
periodicVector.getCapacityAtTime(10L));
Assert.assertEquals(Resource.newInstance(0, 0),
periodicVector.getCapacityAtTime(0L));
- Assert.assertFalse(periodicVector.addInterval(
- new ReservationInterval(7L, 12L), Resource.newInstance(8, 8)));
+ // Assert.assertFalse(periodicVector.addInterval(
+ // new ReservationInterval(7L, 12L), Resource.newInstance(8, 8)));
}
public void testRemoveInterval() {
- int[] alloc = {2, 5, 3, 4, 0};
- long[] timeSteps = {1L, 3L, 5L, 7L, 9L};
- RLESparseResourceAllocation rleSparseVector =
- ReservationSystemTestUtil.generateRLESparseResourceAllocation(
- alloc, timeSteps);
+ int[] alloc = { 2, 5, 3, 4, 0 };
+ long[] timeSteps = { 1L, 3L, 5L, 7L, 9L };
+ RLESparseResourceAllocation rleSparseVector = ReservationSystemTestUtil
+ .generateRLESparseResourceAllocation(alloc, timeSteps);
PeriodicRLESparseResourceAllocation periodicVector =
new PeriodicRLESparseResourceAllocation(rleSparseVector, 10L);
ReservationInterval interval = new ReservationInterval(3L, 7L);
- Assert.assertTrue(periodicVector.removeInterval(
- interval, Resource.newInstance(3, 3)));
+ Assert.assertTrue(
+ periodicVector.removeInterval(interval, Resource.newInstance(3, 3)));
Assert.assertEquals(Resource.newInstance(2, 2),
periodicVector.getCapacityAtTime(1L));
Assert.assertEquals(Resource.newInstance(2, 2),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e7d136/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestRLESparseResourceAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestRLESparseResourceAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestRLESparseResourceAllocation.java
index bfe46e1..0027ceb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestRLESparseResourceAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestRLESparseResourceAllocation.java
@@ -40,10 +40,14 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+/**
+ * Testing the class {@link RLESparseResourceAllocation}.
+ */
+@SuppressWarnings("checkstyle:nowhitespaceafter")
public class TestRLESparseResourceAllocation {
- private static final Logger LOG = LoggerFactory
- .getLogger(TestRLESparseResourceAllocation.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestRLESparseResourceAllocation.class);
@Test
public void testMergeAdd() throws PlanningException {
@@ -196,7 +200,8 @@ public class TestRLESparseResourceAllocation {
// Expected!
}
- // Testing that the subtractTestNonNegative detects problems even if only one
+ // Testing that the subtractTestNonNegative detects problems even if only
+ // one
// of the resource dimensions is "<0"
a.put(10L, Resource.newInstance(10, 5));
b.put(11L, Resource.newInstance(5, 6));
@@ -286,9 +291,8 @@ public class TestRLESparseResourceAllocation {
public void testRangeOverlapping() {
ResourceCalculator resCalc = new DefaultResourceCalculator();
- RLESparseResourceAllocation r =
- new RLESparseResourceAllocation(resCalc);
- int[] alloc = {10, 10, 10, 10, 10, 10};
+ RLESparseResourceAllocation r = new RLESparseResourceAllocation(resCalc);
+ int[] alloc = { 10, 10, 10, 10, 10, 10 };
int start = 100;
Set<Entry<ReservationInterval, Resource>> inputs =
generateAllocation(start, alloc, false).entrySet();
@@ -299,9 +303,9 @@ public class TestRLESparseResourceAllocation {
long d = r.getLatestNonNullTime();
// tries to trigger "out-of-range" bug
- r = r.getRangeOverlapping(s, d);
- r = r.getRangeOverlapping(s-1, d-1);
- r = r.getRangeOverlapping(s+1, d+1);
+ r = r.getRangeOverlapping(s, d);
+ r = r.getRangeOverlapping(s - 1, d - 1);
+ r = r.getRangeOverlapping(s + 1, d + 1);
}
@Test
@@ -370,25 +374,29 @@ public class TestRLESparseResourceAllocation {
// Current bug prevents this to pass. The RLESparseResourceAllocation
// does not handle removal of "partial"
// allocations correctly.
- Assert.assertEquals(102400, rleSparseVector.getCapacityAtTime(10)
- .getMemorySize());
- Assert.assertEquals(0, rleSparseVector.getCapacityAtTime(13).getMemorySize());
- Assert.assertEquals(0, rleSparseVector.getCapacityAtTime(19).getMemorySize());
- Assert.assertEquals(102400, rleSparseVector.getCapacityAtTime(21)
- .getMemorySize());
- Assert.assertEquals(2 * 102400, rleSparseVector.getCapacityAtTime(26)
- .getMemorySize());
+ Assert.assertEquals(102400,
+ rleSparseVector.getCapacityAtTime(10).getMemorySize());
+ Assert.assertEquals(0,
+ rleSparseVector.getCapacityAtTime(13).getMemorySize());
+ Assert.assertEquals(0,
+ rleSparseVector.getCapacityAtTime(19).getMemorySize());
+ Assert.assertEquals(102400,
+ rleSparseVector.getCapacityAtTime(21).getMemorySize());
+ Assert.assertEquals(2 * 102400,
+ rleSparseVector.getCapacityAtTime(26).getMemorySize());
ReservationInterval riRemove2 = new ReservationInterval(9, 13);
rleSparseVector.removeInterval(riRemove2, rr);
LOG.info(rleSparseVector.toString());
- Assert.assertEquals(0, rleSparseVector.getCapacityAtTime(11).getMemorySize());
- Assert.assertEquals(-102400, rleSparseVector.getCapacityAtTime(9)
- .getMemorySize());
- Assert.assertEquals(0, rleSparseVector.getCapacityAtTime(13).getMemorySize());
- Assert.assertEquals(102400, rleSparseVector.getCapacityAtTime(20)
- .getMemorySize());
+ Assert.assertEquals(0,
+ rleSparseVector.getCapacityAtTime(11).getMemorySize());
+ Assert.assertEquals(-102400,
+ rleSparseVector.getCapacityAtTime(9).getMemorySize());
+ Assert.assertEquals(0,
+ rleSparseVector.getCapacityAtTime(13).getMemorySize());
+ Assert.assertEquals(102400,
+ rleSparseVector.getCapacityAtTime(20).getMemorySize());
}
@@ -500,7 +508,8 @@ public class TestRLESparseResourceAllocation {
}
mapAllocations = rleSparseVector.toIntervalMap();
Assert.assertTrue(mapAllocations.size() == 5);
- for (Entry<ReservationInterval, Resource> entry : mapAllocations.entrySet()) {
+ for (Entry<ReservationInterval, Resource> entry : mapAllocations
+ .entrySet()) {
ReservationInterval interval = entry.getKey();
Resource resource = entry.getValue();
if (interval.getStartTime() == 101L) {
@@ -526,59 +535,46 @@ public class TestRLESparseResourceAllocation {
@Test
public void testMaxPeriodicCapacity() {
- long[] timeSteps = {0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L};
- int[] alloc = {2, 5, 7, 10, 3, 4, 6, 8};
- RLESparseResourceAllocation rleSparseVector =
- ReservationSystemTestUtil.generateRLESparseResourceAllocation(
- alloc, timeSteps);
+ long[] timeSteps = { 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L };
+ int[] alloc = { 2, 5, 7, 10, 3, 4, 6, 8 };
+ RLESparseResourceAllocation rleSparseVector = ReservationSystemTestUtil
+ .generateRLESparseResourceAllocation(alloc, timeSteps);
LOG.info(rleSparseVector.toString());
- Assert.assertEquals(
- rleSparseVector.getMaximumPeriodicCapacity(0, 1),
+ Assert.assertEquals(rleSparseVector.getMaximumPeriodicCapacity(0, 1),
Resource.newInstance(10, 10));
- Assert.assertEquals(
- rleSparseVector.getMaximumPeriodicCapacity(0, 2),
+ Assert.assertEquals(rleSparseVector.getMaximumPeriodicCapacity(0, 2),
Resource.newInstance(7, 7));
- Assert.assertEquals(
- rleSparseVector.getMaximumPeriodicCapacity(0, 3),
+ Assert.assertEquals(rleSparseVector.getMaximumPeriodicCapacity(0, 3),
Resource.newInstance(10, 10));
- Assert.assertEquals(
- rleSparseVector.getMaximumPeriodicCapacity(0, 4),
+ Assert.assertEquals(rleSparseVector.getMaximumPeriodicCapacity(0, 4),
Resource.newInstance(3, 3));
- Assert.assertEquals(
- rleSparseVector.getMaximumPeriodicCapacity(0, 5),
+ Assert.assertEquals(rleSparseVector.getMaximumPeriodicCapacity(0, 5),
Resource.newInstance(4, 4));
- Assert.assertEquals(
- rleSparseVector.getMaximumPeriodicCapacity(0, 5),
+ Assert.assertEquals(rleSparseVector.getMaximumPeriodicCapacity(0, 5),
Resource.newInstance(4, 4));
- Assert.assertEquals(
- rleSparseVector.getMaximumPeriodicCapacity(7, 5),
+ Assert.assertEquals(rleSparseVector.getMaximumPeriodicCapacity(7, 5),
Resource.newInstance(8, 8));
- Assert.assertEquals(
- rleSparseVector.getMaximumPeriodicCapacity(10, 3),
+ Assert.assertEquals(rleSparseVector.getMaximumPeriodicCapacity(10, 3),
Resource.newInstance(0, 0));
}
@Test
public void testGetMinimumCapacityInInterval() {
- long[] timeSteps = {0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L};
- int[] alloc = {2, 5, 7, 10, 3, 4, 0, 8};
- RLESparseResourceAllocation rleSparseVector =
- ReservationSystemTestUtil.generateRLESparseResourceAllocation(
- alloc, timeSteps);
+ long[] timeSteps = { 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L };
+ int[] alloc = { 2, 5, 7, 10, 3, 4, 0, 8 };
+ RLESparseResourceAllocation rleSparseVector = ReservationSystemTestUtil
+ .generateRLESparseResourceAllocation(alloc, timeSteps);
LOG.info(rleSparseVector.toString());
- Assert.assertEquals(
- rleSparseVector.getMinimumCapacityInInterval(
- new ReservationInterval(1L, 3L)), Resource.newInstance(5, 5));
- Assert.assertEquals(
- rleSparseVector.getMinimumCapacityInInterval(
- new ReservationInterval(2L, 5L)), Resource.newInstance(3, 3));
- Assert.assertEquals(
- rleSparseVector.getMinimumCapacityInInterval(
- new ReservationInterval(1L, 7L)), Resource.newInstance(0, 0));
+ Assert.assertEquals(rleSparseVector.getMinimumCapacityInInterval(
+ new ReservationInterval(1L, 3L)), Resource.newInstance(5, 5));
+ Assert.assertEquals(rleSparseVector.getMinimumCapacityInInterval(
+ new ReservationInterval(2L, 5L)), Resource.newInstance(3, 3));
+ Assert.assertEquals(rleSparseVector.getMinimumCapacityInInterval(
+ new ReservationInterval(1L, 7L)), Resource.newInstance(0, 0));
}
- private void setupArrays(
- TreeMap<Long, Resource> a, TreeMap<Long, Resource> b) {
+ private void setupArrays(TreeMap<Long, Resource> a,
+ TreeMap<Long, Resource> b) {
a.put(10L, Resource.newInstance(5, 5));
a.put(20L, Resource.newInstance(10, 10));
a.put(30L, Resource.newInstance(15, 15));
@@ -620,8 +616,8 @@ public class TestRLESparseResourceAllocation {
numContainers = alloc[i];
}
req.put(new ReservationInterval(startTime + i, startTime + i + 1),
- ReservationSystemUtil.toResource(ReservationRequest.newInstance(
- Resource.newInstance(1024, 1), (numContainers))));
+ ReservationSystemUtil.toResource(ReservationRequest
+ .newInstance(Resource.newInstance(1024, 1), (numContainers))));
}
return req;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e7d136/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java
index c4f94c2..ddd290d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.ReservationDefinition;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.ReservationRequest;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.InMemoryPlan;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.InMemoryReservationAllocation;
@@ -78,9 +79,10 @@ public class TestSimpleCapacityReplanner {
enf.init("blah", conf);
// Initialize the plan with more resources
- InMemoryPlan plan =
- new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step,
- res, minAlloc, maxAlloc, "dedicated", enf, true, context, clock);
+ InMemoryPlan plan = new InMemoryPlan(queueMetrics, policy, agent,
+ clusterCapacity, step, res, minAlloc, maxAlloc, "dedicated", enf, true,
+ YarnConfiguration.DEFAULT_RM_RESERVATION_SYSTEM_MAX_PERIODICITY,
+ context, clock);
// add reservation filling the plan (separating them 1ms, so we are sure
// s2 follows s1 on acceptance
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/40] hadoop git commit: YARN-7164. TestAMRMClientOnRMRestart fails
sporadically with bind address in use. Contributed by Jason Lowe
Posted by in...@apache.org.
YARN-7164. TestAMRMClientOnRMRestart fails sporadically with bind address in use. Contributed by Jason Lowe
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/704267cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/704267cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/704267cb
Branch: refs/heads/HDFS-10467
Commit: 704267cb4909bf9968dd30067617f74543e07ffb
Parents: 63231a2
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Sep 6 15:23:49 2017 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Sep 6 15:23:49 2017 -0500
----------------------------------------------------------------------
.../hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/704267cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
index fa3c6af..337d7d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
@@ -29,7 +29,6 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.net.NetworkTopology;
-import org.apache.hadoop.net.ServerSocketUtil;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
@@ -93,6 +92,7 @@ public class TestAMRMClientOnRMRestart {
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, true);
conf.setLong(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS, 0);
+ conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, "0.0.0.0:0");
}
@AfterClass
@@ -496,8 +496,6 @@ public class TestAMRMClientOnRMRestart {
.getMasterKey().getKeyId() == newToken.decodeIdentifier().getKeyId());
// start 2nd RM
- conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, "0.0.0.0:"
- + ServerSocketUtil.getPort(45020, 10));
final MyResourceManager2 rm2 = new MyResourceManager2(conf, memStore);
rm2.start();
nm1.setResourceTrackerService(rm2.getResourceTrackerService());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/40] hadoop git commit: HDFS-12402. Refactor
ErasureCodingPolicyManager and related codes. Contributed by Sammi Chen
Posted by in...@apache.org.
HDFS-12402. Refactor ErasureCodingPolicyManager and related codes. Contributed by Sammi Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2adf8bed
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2adf8bed
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2adf8bed
Branch: refs/heads/HDFS-10467
Commit: 2adf8bed712e6d770a0d53eea198d8911ae1a258
Parents: 6f101e7
Author: Kai Zheng <ka...@intel.com>
Authored: Thu Sep 7 20:38:23 2017 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Thu Sep 7 20:38:23 2017 +0800
----------------------------------------------------------------------
.../hdfs/protocol/AddECPolicyResponse.java | 4 +-
.../hdfs/protocol/IllegalECPolicyException.java | 34 -----------------
.../namenode/ErasureCodingPolicyManager.java | 39 ++++++++++----------
.../server/namenode/FSDirErasureCodingOp.java | 3 +-
.../hdfs/server/namenode/FSNamesystem.java | 3 +-
.../hadoop/hdfs/TestDistributedFileSystem.java | 4 +-
6 files changed, 27 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2adf8bed/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/AddECPolicyResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/AddECPolicyResponse.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/AddECPolicyResponse.java
index e7a8435..4e55680 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/AddECPolicyResponse.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/AddECPolicyResponse.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hdfs.protocol;
+import org.apache.hadoop.HadoopIllegalArgumentException;
+
/**
* A response of add an ErasureCoding policy.
*/
@@ -38,7 +40,7 @@ public class AddECPolicyResponse {
}
public AddECPolicyResponse(ErasureCodingPolicy policy,
- IllegalECPolicyException e) {
+ HadoopIllegalArgumentException e) {
this(policy, e.getMessage());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2adf8bed/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/IllegalECPolicyException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/IllegalECPolicyException.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/IllegalECPolicyException.java
deleted file mode 100644
index 03ce2a5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/IllegalECPolicyException.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.protocol;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * An Exception indicates the error when adding an ErasureCoding policy.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class IllegalECPolicyException extends Exception {
- static final long serialVersionUID = 1L;
-
- public IllegalECPolicyException(String msg) {
- super(msg);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2adf8bed/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
index 74b5ebf..4c75709 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
@@ -19,11 +19,11 @@ package org.apache.hadoop.hdfs.server.namenode;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang.ArrayUtils;
+import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState;
-import org.apache.hadoop.hdfs.protocol.IllegalECPolicyException;
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -144,7 +144,7 @@ public final class ErasureCodingPolicyManager {
policyName,
DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
names);
- throw new IllegalArgumentException(msg);
+ throw new HadoopIllegalArgumentException(msg);
}
enabledPoliciesByName.put(ecPolicy.getName(), ecPolicy);
}
@@ -230,33 +230,34 @@ public final class ErasureCodingPolicyManager {
* Add an erasure coding policy.
* @return the added policy
*/
- public synchronized ErasureCodingPolicy addPolicy(ErasureCodingPolicy policy)
- throws IllegalECPolicyException {
+ public synchronized ErasureCodingPolicy addPolicy(
+ ErasureCodingPolicy policy) {
// Set policy state into DISABLED when adding into Hadoop.
policy.setState(ErasureCodingPolicyState.DISABLED);
if (!CodecUtil.hasCodec(policy.getCodecName())) {
- throw new IllegalECPolicyException("Codec name "
+ throw new HadoopIllegalArgumentException("Codec name "
+ policy.getCodecName() + " is not supported");
}
if (policy.getCellSize() > maxCellSize) {
- throw new IllegalECPolicyException("Cell size " + policy.getCellSize()
- + " should not exceed maximum " + maxCellSize + " byte");
+ throw new HadoopIllegalArgumentException("Cell size " +
+ policy.getCellSize() + " should not exceed maximum " +
+ maxCellSize + " bytes");
}
String assignedNewName = ErasureCodingPolicy.composePolicyName(
policy.getSchema(), policy.getCellSize());
for (ErasureCodingPolicy p : getPolicies()) {
if (p.getName().equals(assignedNewName)) {
- throw new IllegalECPolicyException("The policy name " + assignedNewName
- + " already exists");
+ throw new HadoopIllegalArgumentException("The policy name " +
+ assignedNewName + " already exists");
}
if (p.getSchema().equals(policy.getSchema()) &&
p.getCellSize() == policy.getCellSize()) {
- throw new IllegalECPolicyException("A policy with same schema "
+ throw new HadoopIllegalArgumentException("A policy with same schema "
+ policy.getSchema().toString() + " and cell size "
- + p.getCellSize() + " is already exists");
+ + p.getCellSize() + " already exists");
}
}
policy.setName(assignedNewName);
@@ -281,12 +282,12 @@ public final class ErasureCodingPolicyManager {
public synchronized void removePolicy(String name) {
ErasureCodingPolicy ecPolicy = policiesByName.get(name);
if (ecPolicy == null) {
- throw new IllegalArgumentException("The policy name " +
- name + " does not exists");
+ throw new HadoopIllegalArgumentException("The policy name " +
+ name + " does not exist");
}
if (ecPolicy.isSystemPolicy()) {
- throw new IllegalArgumentException("System erasure coding policy " +
+ throw new HadoopIllegalArgumentException("System erasure coding policy " +
name + " cannot be removed");
}
@@ -317,8 +318,8 @@ public final class ErasureCodingPolicyManager {
public synchronized void disablePolicy(String name) {
ErasureCodingPolicy ecPolicy = policiesByName.get(name);
if (ecPolicy == null) {
- throw new IllegalArgumentException("The policy name " +
- name + " does not exists");
+ throw new HadoopIllegalArgumentException("The policy name " +
+ name + " does not exist");
}
if (enabledPoliciesByName.containsKey(name)) {
@@ -336,8 +337,8 @@ public final class ErasureCodingPolicyManager {
public synchronized void enablePolicy(String name) {
ErasureCodingPolicy ecPolicy = policiesByName.get(name);
if (ecPolicy == null) {
- throw new IllegalArgumentException("The policy name " +
- name + " does not exists");
+ throw new HadoopIllegalArgumentException("The policy name " +
+ name + " does not exist");
}
enabledPoliciesByName.put(name, ecPolicy);
@@ -346,4 +347,4 @@ public final class ErasureCodingPolicyManager {
enabledPoliciesByName.values().toArray(new ErasureCodingPolicy[0]);
LOG.info("Enable the erasure coding policy " + name);
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2adf8bed/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
index 4f4befe..deb03af 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
-import org.apache.hadoop.hdfs.protocol.IllegalECPolicyException;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.WritableUtils;
@@ -212,7 +211,7 @@ final class FSDirErasureCodingOp {
}
static ErasureCodingPolicy addErasureCodePolicy(final FSNamesystem fsn,
- ErasureCodingPolicy policy) throws IllegalECPolicyException {
+ ErasureCodingPolicy policy) {
Preconditions.checkNotNull(policy);
return fsn.getErasureCodingPolicyManager().addPolicy(policy);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2adf8bed/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index e5604c4..c30999b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -201,7 +201,6 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
-import org.apache.hadoop.hdfs.protocol.IllegalECPolicyException;
import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
@@ -7207,7 +7206,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
FSDirErasureCodingOp.addErasureCodePolicy(this, policy);
addECPolicyName = newPolicy.getName();
responses.add(new AddECPolicyResponse(newPolicy));
- } catch (IllegalECPolicyException e) {
+ } catch (HadoopIllegalArgumentException e) {
responses.add(new AddECPolicyResponse(policy, e));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2adf8bed/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
index bf2b002..8e54e5f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
@@ -1617,7 +1617,7 @@ public class TestDistributedFileSystem {
fs.enableErasureCodingPolicy("notExistECName");
Assert.fail("enable the policy that doesn't exist should fail");
} catch (Exception e) {
- GenericTestUtils.assertExceptionContains("does not exists", e);
+ GenericTestUtils.assertExceptionContains("does not exist", e);
// pass
}
@@ -1626,7 +1626,7 @@ public class TestDistributedFileSystem {
fs.disableErasureCodingPolicy("notExistECName");
Assert.fail("disable the policy that doesn't exist should fail");
} catch (Exception e) {
- GenericTestUtils.assertExceptionContains("does not exists", e);
+ GenericTestUtils.assertExceptionContains("does not exist", e);
// pass
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/40] hadoop git commit: YARN-7144. Log Aggregation controller
should not swallow the exceptions when it calls closeWriter and closeReader.
Contributed by Xuan Gong.
Posted by in...@apache.org.
YARN-7144. Log Aggregation controller should not swallow the exceptions when it calls closeWriter and closeReader. Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/22de9449
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/22de9449
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/22de9449
Branch: refs/heads/HDFS-10467
Commit: 22de9449f8aa72c5b0bb586b8253390773502189
Parents: dd81494
Author: Junping Du <ju...@apache.org>
Authored: Wed Sep 6 14:53:31 2017 -0700
Committer: Junping Du <ju...@apache.org>
Committed: Wed Sep 6 14:53:31 2017 -0700
----------------------------------------------------------------------
.../logaggregation/AggregatedLogFormat.java | 27 +++++++++-----------
.../LogAggregationFileController.java | 10 ++++----
2 files changed, 17 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22de9449/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
index 9bec147..af3066e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
@@ -44,11 +44,8 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.regex.Pattern;
-import org.apache.commons.io.IOUtils;
import org.apache.commons.io.input.BoundedInputStream;
import org.apache.commons.io.output.WriterOutputStream;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.commons.math3.util.Pair;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
@@ -61,6 +58,7 @@ import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SecureIOUtils;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.file.tfile.TFile;
@@ -71,7 +69,8 @@ import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.util.Times;
-
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
@@ -81,7 +80,8 @@ import com.google.common.collect.Sets;
@Evolving
public class AggregatedLogFormat {
- private static final Log LOG = LogFactory.getLog(AggregatedLogFormat.class);
+ private final static Logger LOG = LoggerFactory.getLogger(
+ AggregatedLogFormat.class);
private static final LogKey APPLICATION_ACL_KEY = new LogKey("APPLICATION_ACL");
private static final LogKey APPLICATION_OWNER_KEY = new LogKey("APPLICATION_OWNER");
private static final LogKey VERSION_KEY = new LogKey("VERSION");
@@ -247,7 +247,7 @@ public class AggregatedLogFormat {
in = secureOpenFile(logFile);
} catch (IOException e) {
logErrorMessage(logFile, e);
- IOUtils.closeQuietly(in);
+ IOUtils.cleanupWithLogger(LOG, in);
continue;
}
@@ -285,7 +285,7 @@ public class AggregatedLogFormat {
String message = logErrorMessage(logFile, e);
out.write(message.getBytes(Charset.forName("UTF-8")));
} finally {
- IOUtils.closeQuietly(in);
+ IOUtils.cleanupWithLogger(LOG, in);
}
}
}
@@ -555,7 +555,7 @@ public class AggregatedLogFormat {
} catch (Exception e) {
LOG.warn("Exception closing writer", e);
} finally {
- IOUtils.closeQuietly(this.fsDataOStream);
+ IOUtils.cleanupWithLogger(LOG, this.fsDataOStream);
}
}
}
@@ -603,7 +603,7 @@ public class AggregatedLogFormat {
}
return null;
} finally {
- IOUtils.closeQuietly(ownerScanner);
+ IOUtils.cleanupWithLogger(LOG, ownerScanner);
}
}
@@ -649,7 +649,7 @@ public class AggregatedLogFormat {
}
return acls;
} finally {
- IOUtils.closeQuietly(aclScanner);
+ IOUtils.cleanupWithLogger(LOG, aclScanner);
}
}
@@ -774,8 +774,7 @@ public class AggregatedLogFormat {
}
}
} finally {
- IOUtils.closeQuietly(ps);
- IOUtils.closeQuietly(os);
+ IOUtils.cleanupWithLogger(LOG, ps, os);
}
}
@@ -1001,9 +1000,7 @@ public class AggregatedLogFormat {
}
public void close() {
- IOUtils.closeQuietly(scanner);
- IOUtils.closeQuietly(reader);
- IOUtils.closeQuietly(fsDataIStream);
+ IOUtils.cleanupWithLogger(LOG, scanner, reader, fsDataIStream);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22de9449/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
index aafdb66..39f3dc3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
@@ -37,9 +37,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
@@ -47,6 +44,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -56,6 +54,8 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils;
import org.apache.hadoop.yarn.webapp.View.ViewContext;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock.Block;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey;
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogValue;
import org.apache.hadoop.yarn.logaggregation.ContainerLogMeta;
@@ -68,7 +68,7 @@ import org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest;
@Unstable
public abstract class LogAggregationFileController {
- private static final Log LOG = LogFactory.getLog(
+ private static final Logger LOG = LoggerFactory.getLogger(
LogAggregationFileController.class);
/*
@@ -193,7 +193,7 @@ public abstract class LogAggregationFileController {
protected void closePrintStream(OutputStream out) {
if (out != System.out) {
- IOUtils.closeQuietly(out);
+ IOUtils.cleanupWithLogger(LOG, out);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/40] hadoop git commit: HDFS-10881. Federation State Store Driver
API. Contributed by Jason Kace and Inigo Goiri.
Posted by in...@apache.org.
HDFS-10881. Federation State Store Driver API. Contributed by Jason Kace and Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bc3303e7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bc3303e7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bc3303e7
Branch: refs/heads/HDFS-10467
Commit: bc3303e77f1643a9beb885a100d92cfd327ad5ae
Parents: ffdfd2e
Author: Inigo <in...@apache.org>
Authored: Wed Mar 29 19:35:06 2017 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu Sep 7 13:19:50 2017 -0700
----------------------------------------------------------------------
.../store/StateStoreUnavailableException.java | 33 ++++
.../federation/store/StateStoreUtils.java | 72 +++++++
.../store/driver/StateStoreDriver.java | 172 +++++++++++++++++
.../driver/StateStoreRecordOperations.java | 164 ++++++++++++++++
.../store/driver/impl/StateStoreBaseImpl.java | 69 +++++++
.../store/driver/impl/package-info.java | 39 ++++
.../federation/store/driver/package-info.java | 37 ++++
.../federation/store/protocol/package-info.java | 31 +++
.../federation/store/records/BaseRecord.java | 189 +++++++++++++++++++
.../federation/store/records/QueryResult.java | 56 ++++++
.../federation/store/records/package-info.java | 36 ++++
11 files changed, 898 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc3303e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUnavailableException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUnavailableException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUnavailableException.java
new file mode 100644
index 0000000..4e6f8c8
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUnavailableException.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import java.io.IOException;
+
+/**
+ * Thrown when the state store is not reachable or available. Cached APIs and
+ * queries may succeed. Client should retry again later.
+ */
+public class StateStoreUnavailableException extends IOException {
+
+ private static final long serialVersionUID = 1L;
+
+ public StateStoreUnavailableException(String msg) {
+ super(msg);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc3303e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUtils.java
new file mode 100644
index 0000000..8c681df
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUtils.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+
+/**
+ * Set of utility functions used to query, create, update and delete data
+ * records in the state store.
+ */
+public final class StateStoreUtils {
+
+ private static final Log LOG = LogFactory.getLog(StateStoreUtils.class);
+
+ private StateStoreUtils() {
+ // Utility class
+ }
+
+ /**
+ * Get the base class for a record class. If we get an implementation of a
+ * record we will return the real parent record class.
+ *
+ * @param clazz Class of the data record to check.
+ * @return Base class for the record.
+ */
+ @SuppressWarnings("unchecked")
+ public static <T extends BaseRecord>
+ Class<? extends BaseRecord> getRecordClass(final Class<T> clazz) {
+
+ // We ignore the Impl classes and go to the super class
+ Class<? extends BaseRecord> actualClazz = clazz;
+ while (actualClazz.getSimpleName().endsWith("Impl")) {
+ actualClazz = (Class<? extends BaseRecord>) actualClazz.getSuperclass();
+ }
+
+ // Check if we went too far
+ if (actualClazz.equals(BaseRecord.class)) {
+ LOG.error("We went too far (" + actualClazz + ") with " + clazz);
+ actualClazz = clazz;
+ }
+ return actualClazz;
+ }
+
+ /**
+ * Get the base class for a record. If we get an implementation of a record we
+ * will return the real parent record class.
+ *
+ * @param record Record to check its main class.
+ * @return Base class for the record.
+ */
+ public static <T extends BaseRecord>
+ Class<? extends BaseRecord> getRecordClass(final T record) {
+ return getRecordClass(record.getClass());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc3303e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java
new file mode 100644
index 0000000..a1527df
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java
@@ -0,0 +1,172 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver;
+
+import java.net.InetAddress;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreUnavailableException;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.util.Time;
+
+/**
+ * Driver class for an implementation of a {@link StateStoreService}
+ * provider. Driver implementations will extend this class and implement some of
+ * the default methods.
+ */
+public abstract class StateStoreDriver implements StateStoreRecordOperations {
+
+ private static final Log LOG = LogFactory.getLog(StateStoreDriver.class);
+
+
+ /** State Store configuration. */
+ private Configuration conf;
+
+ /** Identifier for the driver. */
+ private String identifier;
+
+
+ /**
+ * Initialize the state store connection.
+ * @param config Configuration for the driver.
+ * @param id Identifier for the driver.
+ * @param records Records that are supported.
+ * @return If initialized and ready, false if failed to initialize driver.
+ */
+ public boolean init(final Configuration config, final String id,
+ final List<Class<? extends BaseRecord>> records) {
+
+ this.conf = config;
+ this.identifier = id;
+
+ if (this.identifier == null) {
+ LOG.warn("The identifier for the State Store connection is not set");
+ }
+
+ // TODO stub
+ return false;
+ }
+
+ /**
+ * Get the State Store configuration.
+ *
+ * @return Configuration for the State Store.
+ */
+ protected Configuration getConf() {
+ return this.conf;
+ }
+
+ /**
+ * Gets a unique identifier for the running task/process. Typically the
+ * router address.
+ *
+ * @return Unique identifier for the running task.
+ */
+ public String getIdentifier() {
+ return this.identifier;
+ }
+
+ /**
+ * Prepare the driver to access data storage.
+ *
+ * @return True if the driver was successfully initialized. If false is
+ * returned, the state store will periodically attempt to
+ * re-initialize the driver and the router will remain in safe mode
+ * until the driver is initialized.
+ */
+ public abstract boolean initDriver();
+
+ /**
+ * Initialize storage for a single record class.
+ *
+ * @param name String reference of the record class to initialize, used to
+ * construct paths and file names for the record. Determined by
+ * configuration settings for the specific driver.
+ * @param clazz Record type corresponding to the provided name.
+ * @return True if successful, false otherwise.
+ */
+ public abstract <T extends BaseRecord> boolean initRecordStorage(
+ String className, Class<T> clazz);
+
+ /**
+ * Check if the driver is currently running and the data store connection is
+ * valid.
+ *
+ * @return True if the driver is initialized and the data store is ready.
+ */
+ public abstract boolean isDriverReady();
+
+ /**
+ * Check if the driver is ready to be used and throw an exception otherwise.
+ *
+ * @throws StateStoreUnavailableException If the driver is not ready.
+ */
+ public void verifyDriverReady() throws StateStoreUnavailableException {
+ if (!isDriverReady()) {
+ String driverName = getDriverName();
+ String hostname = getHostname();
+ throw new StateStoreUnavailableException("State Store driver " +
+ driverName + " in " + hostname + " is not ready.");
+ }
+ }
+
+ /**
+ * Close the State Store driver connection.
+ */
+ public abstract void close() throws Exception;
+
+ /**
+ * Returns the current time synchronization from the underlying store.
+ * Override for stores that supply a current date. The data store driver is
+ * responsible for maintaining the official synchronization time/date for all
+ * distributed components.
+ *
+ * @return Current time stamp, used for all synchronization dates.
+ */
+ public long getTime() {
+ return Time.now();
+ }
+
+ /**
+ * Get the name of the driver implementation for debugging.
+ *
+ * @return Name of the driver implementation.
+ */
+ private String getDriverName() {
+ return this.getClass().getSimpleName();
+ }
+
+ /**
+ * Get the host name of the machine running the driver for debugging.
+ *
+ * @return Host name of the machine running the driver.
+ */
+ private String getHostname() {
+ String hostname = "Unknown";
+ try {
+ hostname = InetAddress.getLocalHost().getHostName();
+ } catch (Exception e) {
+ LOG.error("Cannot get local address", e);
+ }
+ return hostname;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc3303e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
new file mode 100644
index 0000000..739eeba
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
+import org.apache.hadoop.io.retry.AtMostOnce;
+import org.apache.hadoop.io.retry.Idempotent;
+
+/**
+ * Operations for a driver to manage records in the State Store.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface StateStoreRecordOperations {
+
+ /**
+ * Get all records of the requested record class from the data store. To use
+ * the default implementations in this class, getAll must return new instances
+ * of the records on each call. It is recommended to override the default
+ * implementations for better performance.
+ *
+ * @param clazz Class of record to fetch.
+ * @return List of all records that match the clazz.
+ * @throws IOException Throws exception if unable to query the data store.
+ */
+ @Idempotent
+ <T extends BaseRecord> QueryResult<T> get(Class<T> clazz) throws IOException;
+
+ /**
+ * Get all records of the requested record class from the data store. To use
+ * the default implementations in this class, getAll must return new instances
+ * of the records on each call. It is recommended to override the default
+ * implementations for better performance.
+ *
+ * @param clazz Class of record to fetch.
+ * @param sub Sub path.
+ * @return List of all records that match the clazz and the sub path.
+ * @throws IOException
+ */
+ @Idempotent
+ <T extends BaseRecord> QueryResult<T> get(Class<T> clazz, String sub)
+ throws IOException;
+
+ /**
+ * Get a single record from the store that matches the query.
+ *
+ * @param clazz Class of record to fetch.
+ * @param query Map of field names and objects to filter results.
+ * @return A single record matching the query. Null if there are no matching
+ * records or more than one matching record in the store.
+ * @throws IOException If multiple records match or if the data store cannot
+ * be queried.
+ */
+ @Idempotent
+ <T extends BaseRecord> T get(Class<T> clazz, Map<String, String> query)
+ throws IOException;
+
+ /**
+ * Get multiple records from the store that match a query. This method
+ * assumes the underlying driver does not support filtering. If the driver
+ * supports filtering it should overwrite this method.
+ *
+ * @param clazz Class of record to fetch.
+ * @param query Map of field names and objects to filter results.
+ * @return Records of type clazz that match the query or empty list if none
+ * are found.
+ * @throws IOException Throws exception if unable to query the data store.
+ */
+ @Idempotent
+ <T extends BaseRecord> List<T> getMultiple(
+ Class<T> clazz, Map<String, String> query) throws IOException;
+
+ /**
+ * Creates a single record. Optionally updates an existing record with same
+ * primary key.
+ *
+ * @param record The record to insert or update.
+ * @param allowUpdate True if update of exiting record is allowed.
+ * @param errorIfExists True if an error should be returned when inserting
+ * an existing record. Only used if allowUpdate = false.
+ * @return True if the operation was successful.
+ *
+ * @throws IOException Throws exception if unable to query the data store.
+ */
+ @AtMostOnce
+ <T extends BaseRecord> boolean put(
+ T record, boolean allowUpdate, boolean errorIfExists) throws IOException;
+
+ /**
+ * Creates multiple records. Optionally updates existing records that have
+ * the same primary key.
+ *
+ * @param records List of data records to update or create. All records must
+ * be of class clazz.
+ * @param clazz Record class of records.
+ * @param allowUpdate True if update of exiting record is allowed.
+ * @param errorIfExists True if an error should be returned when inserting
+ * an existing record. Only used if allowUpdate = false.
+ * @return true if all operations were successful.
+ *
+ * @throws IOException Throws exception if unable to query the data store.
+ */
+ @AtMostOnce
+ <T extends BaseRecord> boolean putAll(
+ List<T> records, boolean allowUpdate, boolean errorIfExists)
+ throws IOException;
+
+ /**
+ * Remove a single record.
+ *
+ * @param record Record to be removed.
+ * @return true If the record was successfully removed. False if the record
+ * could not be removed or not stored.
+ * @throws IOException Throws exception if unable to query the data store.
+ */
+ @AtMostOnce
+ <T extends BaseRecord> boolean remove(T record) throws IOException;
+
+ /**
+ * Remove all records of this class from the store.
+ *
+ * @param clazz Class of records to remove.
+ * @return True if successful.
+ * @throws IOException Throws exception if unable to query the data store.
+ */
+ @AtMostOnce
+ <T extends BaseRecord> boolean removeAll(Class<T> clazz) throws IOException;
+
+ /**
+ * Remove multiple records of a specific class that match a query. Requires
+ * the getAll implementation to fetch fresh records on each call.
+ *
+ * @param clazz Class of record to remove.
+ * @param filter matching filter to remove.
+ * @return The number of records removed.
+ * @throws IOException Throws exception if unable to query the data store.
+ */
+ @AtMostOnce
+ <T extends BaseRecord> int remove(Class<T> clazz, Map<String, String> filter)
+ throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc3303e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreBaseImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreBaseImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreBaseImpl.java
new file mode 100644
index 0000000..b711fa9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreBaseImpl.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+
+/**
+ * Base implementation of a State Store driver. It contains default
+ * implementations for the optional functions. These implementations use an
+ * uncached read/write all algorithm for all changes. In most cases it is
+ * recommended to override the optional functions.
+ * <p>
+ * Drivers may optionally override additional routines for performance
+ * optimization, such as custom get/put/remove queries, depending on the
+ * capabilities of the data store.
+ * <p>
+ */
+public abstract class StateStoreBaseImpl extends StateStoreDriver {
+
+ @Override
+ public <T extends BaseRecord> T get(
+ Class<T> clazz, Map<String, String> query) throws IOException {
+ List<T> records = getMultiple(clazz, query);
+ if (records.size() > 1) {
+ throw new IOException("Found more than one object in collection");
+ } else if (records.size() == 1) {
+ return records.get(0);
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public <T extends BaseRecord> boolean put(
+ T record, boolean allowUpdate, boolean errorIfExists) throws IOException {
+ List<T> singletonList = new ArrayList<T>();
+ singletonList.add(record);
+ return putAll(singletonList, allowUpdate, errorIfExists);
+ }
+
+ @Override
+ public <T extends BaseRecord> boolean remove(T record) throws IOException {
+ Map<String, String> primaryKeys = record.getPrimaryKeys();
+ Class<? extends BaseRecord> clazz = StateStoreUtils.getRecordClass(record);
+ return remove(clazz, primaryKeys) == 1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc3303e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/package-info.java
new file mode 100644
index 0000000..a18433e
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/package-info.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Implementations of state store data providers/drivers. Each driver is
+ * responsible for maintaining, querying, updating and deleting persistent data
+ * records. Data records are defined as subclasses of
+ * {@link org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord}.
+ * Each driver implements the
+ * {@link org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver
+ * StateStoreDriver} interface.
+ * <p>
+ * Currently supported drivers:
+ * <ul>
+ * <li>{@link StateStoreFileImpl} A file-based data storage backend.
+ * <li>{@link StateStoreZooKeeperImpl} A zookeeper based data storage backend.
+ * </ul>
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc3303e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/package-info.java
new file mode 100644
index 0000000..da998b5
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/package-info.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The state store uses modular data storage classes derived from
+ * StateStoreDriver to handle querying, updating and deleting data records. The
+ * data storage driver is initialized and maintained by the StateStoreService.
+ * The state store supports fetching all records of a type, filtering by column
+ * values or fetching a single record by its primary key.
+ * <p>
+ * Each data storage backend is required to implement the methods contained in
+ * the StateStoreDriver interface. These methods allow the querying, updating,
+ * inserting and deleting of data records into the state store.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+
+package org.apache.hadoop.hdfs.server.federation.store.driver;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc3303e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/package-info.java
new file mode 100644
index 0000000..0249d2c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/package-info.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Contains the abstract definitions of the API request and response objects for
+ * the various state store APIs. The state store supports multiple interface
+ * APIs and multiple data records. Each protocol object requires a serialization
+ * implementation, the default is protobuf.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc3303e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java
new file mode 100644
index 0000000..4192a3d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java
@@ -0,0 +1,189 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+import java.util.Map;
+
+import org.apache.hadoop.util.Time;
+
+/**
+ * Abstract base of a data record in the StateStore. All StateStore records are
+ * derived from this class. Data records are persisted in the data store and
+ * are identified by their primary key. Each data record contains:
+ * <ul>
+ * <li>A primary key consisting of a combination of record data fields.
+ * <li>A modification date.
+ * <li>A creation date.
+ * </ul>
+ */
+public abstract class BaseRecord implements Comparable<BaseRecord> {
+
+ /**
+ * Set the modification time for the record.
+ *
+ * @param time Modification time of the record.
+ */
+ public abstract void setDateModified(long time);
+
+ /**
+ * Get the modification time for the record.
+ *
+ * @return Modification time of the record.
+ */
+ public abstract long getDateModified();
+
+ /**
+ * Set the creation time for the record.
+ *
+ * @param time Creation time of the record.
+ */
+ public abstract void setDateCreated(long time);
+
+ /**
+ * Get the creation time for the record.
+ *
+ * @return Creation time of the record
+ */
+ public abstract long getDateCreated();
+
+ /**
+ * Get the expiration time for the record.
+ *
+ * @return Expiration time for the record.
+ */
+ public abstract long getExpirationMs();
+
+ /**
+ * Map of primary key names->values for the record. The primary key can be a
+ * combination of 1-n different State Store serialized values.
+ *
+ * @return Map of key/value pairs that constitute this object's primary key.
+ */
+ public abstract Map<String, String> getPrimaryKeys();
+
+ /**
+ * Initialize the object.
+ */
+ public void init() {
+ // Call this after the object has been constructed
+ initDefaultTimes();
+ }
+
+ /**
+ * Initialize default times. The driver may update these timestamps on insert
+ * and/or update. This should only be called when initializing an object that
+ * is not backed by a data store.
+ */
+ private void initDefaultTimes() {
+ long now = Time.now();
+ this.setDateCreated(now);
+ this.setDateModified(now);
+ }
+
+ /**
+ * Join the primary keys into one single primary key.
+ *
+ * @return A string that is guaranteed to be unique amongst all records of
+ * this type.
+ */
+ public String getPrimaryKey() {
+ return generateMashupKey(getPrimaryKeys());
+ }
+
+ /**
+ * Generates a cache key from a map of values.
+ *
+ * @param keys Map of values.
+ * @return String mashup of key values.
+ */
+ protected static String generateMashupKey(final Map<String, String> keys) {
+ StringBuilder builder = new StringBuilder();
+ for (Object value : keys.values()) {
+ if (builder.length() > 0) {
+ builder.append("-");
+ }
+ builder.append(value);
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Override equals check to use primary key(s) for comparison.
+ */
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof BaseRecord)) {
+ return false;
+ }
+
+ BaseRecord baseObject = (BaseRecord) obj;
+ Map<String, String> keyset1 = this.getPrimaryKeys();
+ Map<String, String> keyset2 = baseObject.getPrimaryKeys();
+ return keyset1.equals(keyset2);
+ }
+
+ /**
+ * Override hash code to use primary key(s) for comparison.
+ */
+ @Override
+ public int hashCode() {
+ Map<String, String> keyset = this.getPrimaryKeys();
+ return keyset.hashCode();
+ }
+
+ @Override
+ public int compareTo(BaseRecord record) {
+ if (record == null) {
+ return -1;
+ }
+ // Descending date order
+ return (int) (record.getDateModified() - this.getDateModified());
+ }
+
+ /**
+ * Called when the modification time and current time is available, checks for
+ * expirations.
+ *
+ * @param currentTime The current timestamp in ms from the data store, to be
+ * compared against the modification and creation dates of the
+ * object.
+ * @return boolean True if the record has been updated and should be
+ * committed to the data store. Override for customized behavior.
+ */
+ public boolean checkExpired(long currentTime) {
+ long expiration = getExpirationMs();
+ if (getDateModified() > 0 && expiration > 0) {
+ return (getDateModified() + expiration) < currentTime;
+ }
+ return false;
+ }
+
+ /**
+ * Validates the record. Called when the record is created, populated from the
+ * state store, and before committing to the state store.
+ * @return If the record is valid.
+ */
+ public boolean validate() {
+ return getDateCreated() > 0 && getDateModified() > 0;
+ }
+
+ @Override
+ public String toString() {
+ return getPrimaryKey();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc3303e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/QueryResult.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/QueryResult.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/QueryResult.java
new file mode 100644
index 0000000..64c2c71
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/QueryResult.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Encapsulates a state store query result that includes a set of records and a
+ * time stamp for the result.
+ */
+public class QueryResult<T extends BaseRecord> {
+
+ /** Data result. */
+ private final List<T> records;
+ /** Time stamp of the data results. */
+ private final long timestamp;
+
+ public QueryResult(final List<T> recs, final long time) {
+ this.records = recs;
+ this.timestamp = time;
+ }
+
+ /**
+ * Get the result of the query.
+ *
+ * @return List of records.
+ */
+ public List<T> getRecords() {
+ return Collections.unmodifiableList(this.records);
+ }
+
+ /**
+ * The timetamp in driver time of this query.
+ *
+ * @return Timestamp in driver time.
+ */
+ public long getTimestamp() {
+ return this.timestamp;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc3303e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/package-info.java
new file mode 100644
index 0000000..63b13af
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/package-info.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Contains the abstract definitions of the state store data records. The state
+ * store supports multiple multiple data records.
+ * <p>
+ * Data records inherit from a common class
+ * {@link org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord
+ * BaseRecord}. Data records are serialized when written to the data store using
+ * a modular serialization implementation. The default is profobuf
+ * serialization. Data is stored as rows of records of the same type with each
+ * data member in a record representing a column.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/40] hadoop git commit: HADOOP-14774. S3A case
"testRandomReadOverBuffer" failed due to improper range parameter.
Contributed by Yonger.
Posted by in...@apache.org.
HADOOP-14774. S3A case "testRandomReadOverBuffer" failed due to improper range parameter.
Contributed by Yonger.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/83449abb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/83449abb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/83449abb
Branch: refs/heads/HDFS-10467
Commit: 83449abb40633a36b22de538cf82fca240af067e
Parents: c41118a
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 7 19:55:56 2017 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 7 19:55:56 2017 +0100
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83449abb/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
index b88b7c1..94d7701 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
@@ -152,7 +152,7 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead {
streamStatistics.streamOpened();
try {
GetObjectRequest request = new GetObjectRequest(bucket, key)
- .withRange(targetPos, contentRangeFinish);
+ .withRange(targetPos, contentRangeFinish - 1);
if (S3AEncryptionMethods.SSE_C.equals(serverSideEncryptionAlgorithm) &&
StringUtils.isNotBlank(serverSideEncryptionKey)){
request.setSSECustomerKey(new SSECustomerKey(serverSideEncryptionKey));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/40] hadoop git commit: HDFS-11546. Federation Router RPC server.
Contributed by Jason Kace and Inigo Goiri.
Posted by in...@apache.org.
HDFS-11546. Federation Router RPC server. Contributed by Jason Kace and Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e96a2c7f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e96a2c7f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e96a2c7f
Branch: refs/heads/HDFS-10467
Commit: e96a2c7f7ab3d94e54cbfcdb0eae68b404fd4e48
Parents: 3384811
Author: Inigo Goiri <in...@apache.org>
Authored: Thu May 11 09:57:03 2017 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu Sep 7 13:19:50 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 38 +
.../resolver/FederationNamespaceInfo.java | 46 +-
.../federation/resolver/RemoteLocation.java | 46 +-
.../federation/router/ConnectionContext.java | 104 +
.../federation/router/ConnectionManager.java | 408 ++++
.../federation/router/ConnectionPool.java | 314 +++
.../federation/router/ConnectionPoolId.java | 117 ++
.../router/RemoteLocationContext.java | 38 +-
.../server/federation/router/RemoteMethod.java | 164 ++
.../server/federation/router/RemoteParam.java | 71 +
.../hdfs/server/federation/router/Router.java | 58 +-
.../federation/router/RouterRpcClient.java | 856 ++++++++
.../federation/router/RouterRpcServer.java | 1867 +++++++++++++++++-
.../src/main/resources/hdfs-default.xml | 95 +
.../server/federation/FederationTestUtils.java | 80 +-
.../hdfs/server/federation/MockResolver.java | 90 +-
.../server/federation/RouterConfigBuilder.java | 20 +-
.../server/federation/RouterDFSCluster.java | 535 +++--
.../server/federation/router/TestRouter.java | 31 +-
.../server/federation/router/TestRouterRpc.java | 869 ++++++++
.../router/TestRouterRpcMultiDestination.java | 216 ++
21 files changed, 5675 insertions(+), 388 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e96a2c7f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index bb3087c..341a502 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -1120,6 +1120,44 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
// HDFS Router-based federation
public static final String FEDERATION_ROUTER_PREFIX =
"dfs.federation.router.";
+ public static final String DFS_ROUTER_DEFAULT_NAMESERVICE =
+ FEDERATION_ROUTER_PREFIX + "default.nameserviceId";
+ public static final String DFS_ROUTER_HANDLER_COUNT_KEY =
+ FEDERATION_ROUTER_PREFIX + "handler.count";
+ public static final int DFS_ROUTER_HANDLER_COUNT_DEFAULT = 10;
+ public static final String DFS_ROUTER_READER_QUEUE_SIZE_KEY =
+ FEDERATION_ROUTER_PREFIX + "reader.queue.size";
+ public static final int DFS_ROUTER_READER_QUEUE_SIZE_DEFAULT = 100;
+ public static final String DFS_ROUTER_READER_COUNT_KEY =
+ FEDERATION_ROUTER_PREFIX + "reader.count";
+ public static final int DFS_ROUTER_READER_COUNT_DEFAULT = 1;
+ public static final String DFS_ROUTER_HANDLER_QUEUE_SIZE_KEY =
+ FEDERATION_ROUTER_PREFIX + "handler.queue.size";
+ public static final int DFS_ROUTER_HANDLER_QUEUE_SIZE_DEFAULT = 100;
+ public static final String DFS_ROUTER_RPC_BIND_HOST_KEY =
+ FEDERATION_ROUTER_PREFIX + "rpc-bind-host";
+ public static final int DFS_ROUTER_RPC_PORT_DEFAULT = 8888;
+ public static final String DFS_ROUTER_RPC_ADDRESS_KEY =
+ FEDERATION_ROUTER_PREFIX + "rpc-address";
+ public static final String DFS_ROUTER_RPC_ADDRESS_DEFAULT =
+ "0.0.0.0:" + DFS_ROUTER_RPC_PORT_DEFAULT;
+ public static final String DFS_ROUTER_RPC_ENABLE =
+ FEDERATION_ROUTER_PREFIX + "rpc.enable";
+ public static final boolean DFS_ROUTER_RPC_ENABLE_DEFAULT = true;
+
+ // HDFS Router NN client
+ public static final String DFS_ROUTER_NAMENODE_CONNECTION_POOL_SIZE =
+ FEDERATION_ROUTER_PREFIX + "connection.pool-size";
+ public static final int DFS_ROUTER_NAMENODE_CONNECTION_POOL_SIZE_DEFAULT =
+ 64;
+ public static final String DFS_ROUTER_NAMENODE_CONNECTION_POOL_CLEAN =
+ FEDERATION_ROUTER_PREFIX + "connection.pool.clean.ms";
+ public static final long DFS_ROUTER_NAMENODE_CONNECTION_POOL_CLEAN_DEFAULT =
+ TimeUnit.MINUTES.toMillis(1);
+ public static final String DFS_ROUTER_NAMENODE_CONNECTION_CLEAN_MS =
+ FEDERATION_ROUTER_PREFIX + "connection.clean.ms";
+ public static final long DFS_ROUTER_NAMENODE_CONNECTION_CLEAN_MS_DEFAULT =
+ TimeUnit.SECONDS.toMillis(10);
// HDFS Router State Store connection
public static final String FEDERATION_FILE_RESOLVER_CLIENT_CLASS =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e96a2c7f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java
index bbaeca3..edcd308 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java
@@ -23,15 +23,14 @@ import org.apache.hadoop.hdfs.server.federation.router.RemoteLocationContext;
* Represents information about a single nameservice/namespace in a federated
* HDFS cluster.
*/
-public class FederationNamespaceInfo
- implements Comparable<FederationNamespaceInfo>, RemoteLocationContext {
+public class FederationNamespaceInfo extends RemoteLocationContext {
/** Block pool identifier. */
- private String blockPoolId;
+ private final String blockPoolId;
/** Cluster identifier. */
- private String clusterId;
+ private final String clusterId;
/** Nameservice identifier. */
- private String nameserviceId;
+ private final String nameserviceId;
public FederationNamespaceInfo(String bpId, String clId, String nsId) {
this.blockPoolId = bpId;
@@ -39,15 +38,16 @@ public class FederationNamespaceInfo
this.nameserviceId = nsId;
}
- /**
- * The HDFS nameservice id for this namespace.
- *
- * @return Nameservice identifier.
- */
+ @Override
public String getNameserviceId() {
return this.nameserviceId;
}
+ @Override
+ public String getDest() {
+ return this.nameserviceId;
+ }
+
/**
* The HDFS cluster id for this namespace.
*
@@ -67,33 +67,7 @@ public class FederationNamespaceInfo
}
@Override
- public int hashCode() {
- return this.nameserviceId.hashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == null) {
- return false;
- } else if (obj instanceof FederationNamespaceInfo) {
- return this.compareTo((FederationNamespaceInfo) obj) == 0;
- } else {
- return false;
- }
- }
-
- @Override
- public int compareTo(FederationNamespaceInfo info) {
- return this.nameserviceId.compareTo(info.getNameserviceId());
- }
-
- @Override
public String toString() {
return this.nameserviceId + "->" + this.blockPoolId + ":" + this.clusterId;
}
-
- @Override
- public String getDest() {
- return this.nameserviceId;
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e96a2c7f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java
index eef136d..6aa12ce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java
@@ -17,34 +17,51 @@
*/
package org.apache.hadoop.hdfs.server.federation.resolver;
-import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.hadoop.hdfs.server.federation.router.RemoteLocationContext;
/**
* A single in a remote namespace consisting of a nameservice ID
* and a HDFS path.
*/
-public class RemoteLocation implements RemoteLocationContext {
+public class RemoteLocation extends RemoteLocationContext {
/** Identifier of the remote namespace for this location. */
- private String nameserviceId;
+ private final String nameserviceId;
+ /** Identifier of the namenode in the namespace for this location. */
+ private final String namenodeId;
/** Path in the remote location. */
- private String path;
+ private final String path;
/**
* Create a new remote location.
*
+ * @param nsId
+ * @param pPath
+ */
+ public RemoteLocation(String nsId, String pPath) {
+ this(nsId, null, pPath);
+ }
+
+ /**
+ * Create a new remote location pointing to a particular namenode in the
+ * namespace.
+ *
* @param nsId Destination namespace.
* @param pPath Path in the destination namespace.
*/
- public RemoteLocation(String nsId, String pPath) {
+ public RemoteLocation(String nsId, String nnId, String pPath) {
this.nameserviceId = nsId;
+ this.namenodeId = nnId;
this.path = pPath;
}
@Override
public String getNameserviceId() {
- return this.nameserviceId;
+ String ret = this.nameserviceId;
+ if (this.namenodeId != null) {
+ ret += "-" + this.namenodeId;
+ }
+ return ret;
}
@Override
@@ -54,21 +71,6 @@ public class RemoteLocation implements RemoteLocationContext {
@Override
public String toString() {
- return this.nameserviceId + "->" + this.path;
- }
-
- @Override
- public int hashCode() {
- return new HashCodeBuilder(17, 31)
- .append(this.nameserviceId)
- .append(this.path)
- .toHashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- return (obj != null &&
- obj.getClass() == this.getClass() &&
- obj.hashCode() == this.hashCode());
+ return getNameserviceId() + "->" + this.path;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e96a2c7f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java
new file mode 100644
index 0000000..1d27b51
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.hadoop.hdfs.NameNodeProxiesClient.ProxyAndInfo;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.ipc.RPC;
+
+/**
+ * Context to track a connection in a {@link ConnectionPool}. When a client uses
+ * a connection, it increments a counter to mark it as active. Once the client
+ * is done with the connection, it decreases the counter. It also takes care of
+ * closing the connection once is not active.
+ */
+public class ConnectionContext {
+
+ /** Client for the connection. */
+ private final ProxyAndInfo<ClientProtocol> client;
+ /** How many threads are using this connection. */
+ private int numThreads = 0;
+ /** If the connection is closed. */
+ private boolean closed = false;
+
+
+ public ConnectionContext(ProxyAndInfo<ClientProtocol> connection) {
+ this.client = connection;
+ }
+
+ /**
+ * Check if the connection is active.
+ *
+ * @return True if the connection is active.
+ */
+ public synchronized boolean isActive() {
+ return this.numThreads > 0;
+ }
+
+ /**
+ * Check if the connection is closed.
+ *
+ * @return If the connection is closed.
+ */
+ public synchronized boolean isClosed() {
+ return this.closed;
+ }
+
+ /**
+ * Check if the connection can be used. It checks if the connection is used by
+ * another thread or already closed.
+ *
+ * @return True if the connection can be used.
+ */
+ public synchronized boolean isUsable() {
+ return !isActive() && !isClosed();
+ }
+
+ /**
+ * Get the connection client.
+ *
+ * @return Connection client.
+ */
+ public synchronized ProxyAndInfo<ClientProtocol> getClient() {
+ this.numThreads++;
+ return this.client;
+ }
+
+ /**
+ * Release this connection. If the connection was closed, close the proxy.
+ * Otherwise, mark the connection as not used by us anymore.
+ */
+ public synchronized void release() {
+ if (--this.numThreads == 0 && this.closed) {
+ close();
+ }
+ }
+
+ /**
+ * We will not use this connection anymore. If it's not being used, we close
+ * it. Otherwise, we let release() do it once we are done with it.
+ */
+ public synchronized void close() {
+ this.closed = true;
+ if (this.numThreads == 0) {
+ ClientProtocol proxy = this.client.getProxy();
+ // Nobody should be using this anymore so it should close right away
+ RPC.stopProxy(proxy);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e96a2c7f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java
new file mode 100644
index 0000000..d93d498
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java
@@ -0,0 +1,408 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Implements a pool of connections for the {@link Router} to be able to open
+ * many connections to many Namenodes.
+ */
+public class ConnectionManager {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ConnectionManager.class);
+
+ /** Number of parallel new connections to create. */
+ protected static final int MAX_NEW_CONNECTIONS = 100;
+
+ /** Minimum amount of active connections: 50%. */
+ protected static final float MIN_ACTIVE_RATIO = 0.5f;
+
+
+ /** Configuration for the connection manager, pool and sockets. */
+ private final Configuration conf;
+
+ /** Min number of connections per user + nn. */
+ private final int minSize = 1;
+ /** Max number of connections per user + nn. */
+ private final int maxSize;
+
+ /** How often we close a pool for a particular user + nn. */
+ private final long poolCleanupPeriodMs;
+ /** How often we close a connection in a pool. */
+ private final long connectionCleanupPeriodMs;
+
+ /** Map of connection pools, one pool per user + NN. */
+ private final Map<ConnectionPoolId, ConnectionPool> pools;
+ /** Lock for accessing pools. */
+ private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ private final Lock readLock = readWriteLock.readLock();
+ private final Lock writeLock = readWriteLock.writeLock();
+
+ /** Queue for creating new connections. */
+ private final BlockingQueue<ConnectionPool> creatorQueue =
+ new ArrayBlockingQueue<>(MAX_NEW_CONNECTIONS);
+ /** Create new connections asynchronously. */
+ private final ConnectionCreator creator;
+ /** Periodic executor to remove stale connection pools. */
+ private final ScheduledThreadPoolExecutor cleaner =
+ new ScheduledThreadPoolExecutor(1);
+
+ /** If the connection manager is running. */
+ private boolean running = false;
+
+
+ /**
+ * Creates a proxy client connection pool manager.
+ *
+ * @param config Configuration for the connections.
+ * @param minPoolSize Min size of the connection pool.
+ * @param maxPoolSize Max size of the connection pool.
+ */
+ public ConnectionManager(Configuration config) {
+ this.conf = config;
+
+ // Configure minimum and maximum connection pools
+ this.maxSize = this.conf.getInt(
+ DFSConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_POOL_SIZE,
+ DFSConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_POOL_SIZE_DEFAULT);
+
+ // Map with the connections indexed by UGI and Namenode
+ this.pools = new HashMap<>();
+
+ // Create connections in a thread asynchronously
+ this.creator = new ConnectionCreator(creatorQueue);
+ this.creator.setDaemon(true);
+
+ // Cleanup periods
+ this.poolCleanupPeriodMs = this.conf.getLong(
+ DFSConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_POOL_CLEAN,
+ DFSConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_POOL_CLEAN_DEFAULT);
+ LOG.info("Cleaning connection pools every {} seconds",
+ TimeUnit.MILLISECONDS.toSeconds(this.poolCleanupPeriodMs));
+ this.connectionCleanupPeriodMs = this.conf.getLong(
+ DFSConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_CLEAN_MS,
+ DFSConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_CLEAN_MS_DEFAULT);
+ LOG.info("Cleaning connections every {} seconds",
+ TimeUnit.MILLISECONDS.toSeconds(this.connectionCleanupPeriodMs));
+ }
+
+ /**
+ * Start the connection manager.
+ */
+ public void start() {
+ // Start the thread that creates connections asynchronously
+ this.creator.start();
+
+ // Schedule a task to remove stale connection pools and sockets
+ long recyleTimeMs = Math.min(
+ poolCleanupPeriodMs, connectionCleanupPeriodMs);
+ LOG.info("Cleaning every {} seconds",
+ TimeUnit.MILLISECONDS.toSeconds(recyleTimeMs));
+ this.cleaner.scheduleAtFixedRate(
+ new CleanupTask(), 0, recyleTimeMs, TimeUnit.MILLISECONDS);
+
+ // Mark the manager as running
+ this.running = true;
+ }
+
+ /**
+ * Stop the connection manager by closing all the pools.
+ */
+ public void close() {
+ this.creator.shutdown();
+ this.cleaner.shutdown();
+ this.running = false;
+
+ writeLock.lock();
+ try {
+ for (ConnectionPool pool : this.pools.values()) {
+ pool.close();
+ }
+ this.pools.clear();
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /**
+ * Fetches the next available proxy client in the pool. Each client connection
+ * is reserved for a single user and cannot be reused until free.
+ *
+ * @param ugi User group information.
+ * @param nnAddress Namenode address for the connection.
+ * @return Proxy client to connect to nnId as UGI.
+ * @throws IOException If the connection cannot be obtained.
+ */
+ public ConnectionContext getConnection(
+ UserGroupInformation ugi, String nnAddress) throws IOException {
+
+ // Check if the manager is shutdown
+ if (!this.running) {
+ LOG.error(
+ "Cannot get a connection to {} because the manager isn't running",
+ nnAddress);
+ return null;
+ }
+
+ // Try to get the pool if created
+ ConnectionPoolId connectionId = new ConnectionPoolId(ugi, nnAddress);
+ ConnectionPool pool = null;
+ readLock.lock();
+ try {
+ pool = this.pools.get(connectionId);
+ } finally {
+ readLock.unlock();
+ }
+
+ // Create the pool if not created before
+ if (pool == null) {
+ writeLock.lock();
+ try {
+ pool = this.pools.get(connectionId);
+ if (pool == null) {
+ pool = new ConnectionPool(
+ this.conf, nnAddress, ugi, this.minSize, this.maxSize);
+ this.pools.put(connectionId, pool);
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ ConnectionContext conn = pool.getConnection();
+
+ // Add a new connection to the pool if it wasn't usable
+ if (conn == null || !conn.isUsable()) {
+ if (!this.creatorQueue.offer(pool)) {
+ LOG.error("Cannot add more than {} connections at the same time",
+ MAX_NEW_CONNECTIONS);
+ }
+ }
+
+ if (conn != null && conn.isClosed()) {
+ LOG.error("We got a closed connection from {}", pool);
+ conn = null;
+ }
+
+ return conn;
+ }
+
+ /**
+ * Get the number of connection pools.
+ *
+ * @return Number of connection pools.
+ */
+ public int getNumConnectionPools() {
+ readLock.lock();
+ try {
+ return pools.size();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ /**
+ * Get number of open connections.
+ *
+ * @return Number of open connections.
+ */
+ public int getNumConnections() {
+ int total = 0;
+ readLock.lock();
+ try {
+ for (ConnectionPool pool : this.pools.values()) {
+ total += pool.getNumConnections();
+ }
+ } finally {
+ readLock.unlock();
+ }
+ return total;
+ }
+
+ /**
+ * Get number of active connections.
+ *
+ * @return Number of active connections.
+ */
+ public int getNumActiveConnections() {
+ int total = 0;
+ readLock.lock();
+ try {
+ for (ConnectionPool pool : this.pools.values()) {
+ total += pool.getNumActiveConnections();
+ }
+ } finally {
+ readLock.unlock();
+ }
+ return total;
+ }
+
+ /**
+ * Get the number of connections to be created.
+ *
+ * @return Number of connections to be created.
+ */
+ public int getNumCreatingConnections() {
+ return this.creatorQueue.size();
+ }
+
+ /**
+ * Removes stale connections not accessed recently from the pool. This is
+ * invoked periodically.
+ */
+ private class CleanupTask implements Runnable {
+
+ @Override
+ public void run() {
+ long currentTime = Time.now();
+ List<ConnectionPoolId> toRemove = new LinkedList<>();
+
+ // Look for stale pools
+ readLock.lock();
+ try {
+ for (Entry<ConnectionPoolId, ConnectionPool> entry : pools.entrySet()) {
+ ConnectionPool pool = entry.getValue();
+ long lastTimeActive = pool.getLastActiveTime();
+ boolean isStale =
+ currentTime > (lastTimeActive + poolCleanupPeriodMs);
+ if (lastTimeActive > 0 && isStale) {
+ // Remove this pool
+ LOG.debug("Closing and removing stale pool {}", pool);
+ pool.close();
+ ConnectionPoolId poolId = entry.getKey();
+ toRemove.add(poolId);
+ } else {
+ // Keep this pool but clean connections inside
+ LOG.debug("Cleaning up {}", pool);
+ cleanup(pool);
+ }
+ }
+ } finally {
+ readLock.unlock();
+ }
+
+ // Remove stale pools
+ if (!toRemove.isEmpty()) {
+ writeLock.lock();
+ try {
+ for (ConnectionPoolId poolId : toRemove) {
+ pools.remove(poolId);
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+ }
+
+ /**
+ * Clean the unused connections for this pool.
+ *
+ * @param pool Connection pool to cleanup.
+ */
+ private void cleanup(ConnectionPool pool) {
+ if (pool.getNumConnections() > pool.getMinSize()) {
+ // Check if the pool hasn't been active in a while or not 50% are used
+ long timeSinceLastActive = Time.now() - pool.getLastActiveTime();
+ int total = pool.getNumConnections();
+ int active = getNumActiveConnections();
+ if (timeSinceLastActive > connectionCleanupPeriodMs ||
+ active < MIN_ACTIVE_RATIO * total) {
+ // Remove and close 1 connection
+ List<ConnectionContext> conns = pool.removeConnections(1);
+ for (ConnectionContext conn : conns) {
+ conn.close();
+ }
+ LOG.debug("Removed connection {} used {} seconds ago. " +
+ "Pool has {}/{} connections", pool.getConnectionPoolId(),
+ TimeUnit.MILLISECONDS.toSeconds(timeSinceLastActive),
+ pool.getNumConnections(), pool.getMaxSize());
+ }
+ }
+ }
+ }
+
+ /**
+ * Thread that creates connections asynchronously.
+ */
+ private static class ConnectionCreator extends Thread {
+ /** If the creator is running. */
+ private boolean running = true;
+ /** Queue to push work to. */
+ private BlockingQueue<ConnectionPool> queue;
+
+ ConnectionCreator(BlockingQueue<ConnectionPool> blockingQueue) {
+ super("Connection creator");
+ this.queue = blockingQueue;
+ }
+
+ @Override
+ public void run() {
+ while (this.running) {
+ try {
+ ConnectionPool pool = this.queue.take();
+ try {
+ int total = pool.getNumConnections();
+ int active = pool.getNumActiveConnections();
+ if (pool.getNumConnections() < pool.getMaxSize() &&
+ active >= MIN_ACTIVE_RATIO * total) {
+ ConnectionContext conn = pool.newConnection();
+ pool.addConnection(conn);
+ } else {
+ LOG.debug("Cannot add more than {} connections to {}",
+ pool.getMaxSize(), pool);
+ }
+ } catch (IOException e) {
+ LOG.error("Cannot create a new connection", e);
+ }
+ } catch (InterruptedException e) {
+ LOG.error("The connection creator was interrupted");
+ this.running = false;
+ }
+ }
+ }
+
+ /**
+ * Stop this connection creator.
+ */
+ public void shutdown() {
+ this.running = false;
+ this.interrupt();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e96a2c7f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java
new file mode 100644
index 0000000..f76f621
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java
@@ -0,0 +1,314 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import javax.net.SocketFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.NameNodeProxiesClient.ProxyAndInfo;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB;
+import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.io.retry.RetryUtils;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SaslRpcServer;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Maintains a pool of connections for each User (including tokens) + NN. The
+ * RPC client maintains a single socket, to achieve throughput similar to a NN,
+ * each request is multiplexed across multiple sockets/connections from a
+ * pool.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class ConnectionPool {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ConnectionPool.class);
+
+
+ /** Configuration settings for the connection pool. */
+ private final Configuration conf;
+
+ /** Identifier for this connection pool. */
+ private final ConnectionPoolId connectionPoolId;
+ /** Namenode this pool connects to. */
+ private final String namenodeAddress;
+ /** User for this connections. */
+ private final UserGroupInformation ugi;
+
+ /** Pool of connections. We mimic a COW array. */
+ private volatile List<ConnectionContext> connections = new ArrayList<>();
+ /** Connection index for round-robin. */
+ private final AtomicInteger clientIndex = new AtomicInteger(0);
+
+ /** Min number of connections per user. */
+ private final int minSize;
+ /** Max number of connections per user. */
+ private final int maxSize;
+
+ /** The last time a connection was active. */
+ private volatile long lastActiveTime = 0;
+
+
+ protected ConnectionPool(Configuration config, String address,
+ UserGroupInformation user, int minPoolSize, int maxPoolSize)
+ throws IOException {
+
+ this.conf = config;
+
+ // Connection pool target
+ this.ugi = user;
+ this.namenodeAddress = address;
+ this.connectionPoolId =
+ new ConnectionPoolId(this.ugi, this.namenodeAddress);
+
+ // Set configuration parameters for the pool
+ this.minSize = minPoolSize;
+ this.maxSize = maxPoolSize;
+
+ // Add minimum connections to the pool
+ for (int i=0; i<this.minSize; i++) {
+ ConnectionContext newConnection = newConnection();
+ this.connections.add(newConnection);
+ }
+ LOG.debug("Created connection pool \"{}\" with {} connections",
+ this.connectionPoolId, this.minSize);
+ }
+
+ /**
+ * Get the maximum number of connections allowed in this pool.
+ *
+ * @return Maximum number of connections.
+ */
+ protected int getMaxSize() {
+ return this.maxSize;
+ }
+
+ /**
+ * Get the minimum number of connections in this pool.
+ *
+ * @return Minimum number of connections.
+ */
+ protected int getMinSize() {
+ return this.minSize;
+ }
+
+ /**
+ * Get the connection pool identifier.
+ *
+ * @return Connection pool identifier.
+ */
+ protected ConnectionPoolId getConnectionPoolId() {
+ return this.connectionPoolId;
+ }
+
+ /**
+ * Return the next connection round-robin.
+ *
+ * @return Connection context.
+ */
+ protected ConnectionContext getConnection() {
+
+ this.lastActiveTime = Time.now();
+
+ // Get a connection from the pool following round-robin
+ ConnectionContext conn = null;
+ List<ConnectionContext> tmpConnections = this.connections;
+ int size = tmpConnections.size();
+ int threadIndex = this.clientIndex.getAndIncrement();
+ for (int i=0; i<size; i++) {
+ int index = (threadIndex + i) % size;
+ conn = tmpConnections.get(index);
+ if (conn != null && !conn.isUsable()) {
+ return conn;
+ }
+ }
+
+ // We return a connection even if it's active
+ return conn;
+ }
+
+ /**
+ * Add a connection to the current pool. It uses a Copy-On-Write approach.
+ *
+ * @param conns New connections to add to the pool.
+ */
+ public synchronized void addConnection(ConnectionContext conn) {
+ List<ConnectionContext> tmpConnections = new ArrayList<>(this.connections);
+ tmpConnections.add(conn);
+ this.connections = tmpConnections;
+
+ this.lastActiveTime = Time.now();
+ }
+
+ /**
+ * Remove connections from the current pool.
+ *
+ * @param num Number of connections to remove.
+ * @return Removed connections.
+ */
+ public synchronized List<ConnectionContext> removeConnections(int num) {
+ List<ConnectionContext> removed = new LinkedList<>();
+
+ // Remove and close the last connection
+ List<ConnectionContext> tmpConnections = new ArrayList<>();
+ for (int i=0; i<this.connections.size(); i++) {
+ ConnectionContext conn = this.connections.get(i);
+ if (i < this.minSize || i < this.connections.size() - num) {
+ tmpConnections.add(conn);
+ } else {
+ removed.add(conn);
+ }
+ }
+ this.connections = tmpConnections;
+
+ return removed;
+ }
+
+ /**
+ * Close the connection pool.
+ */
+ protected synchronized void close() {
+ long timeSinceLastActive = TimeUnit.MILLISECONDS.toSeconds(
+ Time.now() - getLastActiveTime());
+ LOG.debug("Shutting down connection pool \"{}\" used {} seconds ago",
+ this.connectionPoolId, timeSinceLastActive);
+
+ for (ConnectionContext connection : this.connections) {
+ connection.close();
+ }
+ this.connections.clear();
+ }
+
+ /**
+ * Number of connections in the pool.
+ *
+ * @return Number of connections.
+ */
+ protected int getNumConnections() {
+ return this.connections.size();
+ }
+
+ /**
+ * Number of active connections in the pool.
+ *
+ * @return Number of active connections.
+ */
+ protected int getNumActiveConnections() {
+ int ret = 0;
+
+ List<ConnectionContext> tmpConnections = this.connections;
+ for (ConnectionContext conn : tmpConnections) {
+ if (conn.isActive()) {
+ ret++;
+ }
+ }
+ return ret;
+ }
+
+ /**
+ * Get the last time the connection pool was used.
+ *
+ * @return Last time the connection pool was used.
+ */
+ protected long getLastActiveTime() {
+ return this.lastActiveTime;
+ }
+
+ @Override
+ public String toString() {
+ return this.connectionPoolId.toString();
+ }
+
+ /**
+ * Create a new proxy wrapper for a client NN connection.
+ * @return Proxy for the target ClientProtocol that contains the user's
+ * security context.
+ * @throws IOException
+ */
+ public ConnectionContext newConnection() throws IOException {
+ return newConnection(this.conf, this.namenodeAddress, this.ugi);
+ }
+
+ /**
+ * Creates a proxy wrapper for a client NN connection. Each proxy contains
+ * context for a single user/security context. To maximize throughput it is
+ * recommended to use multiple connection per user+server, allowing multiple
+ * writes and reads to be dispatched in parallel.
+ *
+ * @param conf Configuration for the connection.
+ * @param nnAddress Address of server supporting the ClientProtocol.
+ * @param ugi User context.
+ * @return Proxy for the target ClientProtocol that contains the user's
+ * security context.
+ * @throws IOException If it cannot be created.
+ */
+ protected static ConnectionContext newConnection(Configuration conf,
+ String nnAddress, UserGroupInformation ugi)
+ throws IOException {
+ RPC.setProtocolEngine(
+ conf, ClientNamenodeProtocolPB.class, ProtobufRpcEngine.class);
+
+ final RetryPolicy defaultPolicy = RetryUtils.getDefaultRetryPolicy(
+ conf,
+ HdfsClientConfigKeys.Retry.POLICY_ENABLED_KEY,
+ HdfsClientConfigKeys.Retry.POLICY_ENABLED_DEFAULT,
+ HdfsClientConfigKeys.Retry.POLICY_SPEC_KEY,
+ HdfsClientConfigKeys.Retry.POLICY_SPEC_DEFAULT,
+ HdfsConstants.SAFEMODE_EXCEPTION_CLASS_NAME);
+
+ SocketFactory factory = SocketFactory.getDefault();
+ if (UserGroupInformation.isSecurityEnabled()) {
+ SaslRpcServer.init(conf);
+ }
+ InetSocketAddress socket = NetUtils.createSocketAddr(nnAddress);
+ final long version = RPC.getProtocolVersion(ClientNamenodeProtocolPB.class);
+ ClientNamenodeProtocolPB proxy = RPC.getProtocolProxy(
+ ClientNamenodeProtocolPB.class, version, socket, ugi, conf,
+ factory, RPC.getRpcTimeout(conf), defaultPolicy, null).getProxy();
+ ClientProtocol client = new ClientNamenodeProtocolTranslatorPB(proxy);
+ Text dtService = SecurityUtil.buildTokenService(socket);
+
+ ProxyAndInfo<ClientProtocol> clientProxy =
+ new ProxyAndInfo<ClientProtocol>(client, dtService, socket);
+ ConnectionContext connection = new ConnectionContext(clientProxy);
+ return connection;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e96a2c7f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java
new file mode 100644
index 0000000..a3a78de
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+
+/**
+ * Identifier for a connection for a user to a namenode.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class ConnectionPoolId implements Comparable<ConnectionPoolId> {
+
+ /** Namenode identifier. */
+ private final String nnId;
+ /** Information about the user. */
+ private final UserGroupInformation ugi;
+
+ /**
+ * New connection pool identifier.
+ *
+ * @param ugi Information of the user issuing the request.
+ * @param nnId Namenode address with port.
+ */
+ public ConnectionPoolId(final UserGroupInformation ugi, final String nnId) {
+ this.nnId = nnId;
+ this.ugi = ugi;
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = new HashCodeBuilder(17, 31)
+ .append(this.nnId)
+ .append(this.ugi.toString())
+ .append(this.getTokenIds())
+ .toHashCode();
+ return hash;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof ConnectionPoolId) {
+ ConnectionPoolId other = (ConnectionPoolId) o;
+ if (!this.nnId.equals(other.nnId)) {
+ return false;
+ }
+ if (!this.ugi.toString().equals(other.ugi.toString())) {
+ return false;
+ }
+ String thisTokens = this.getTokenIds().toString();
+ String otherTokens = other.getTokenIds().toString();
+ return thisTokens.equals(otherTokens);
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return this.ugi + " " + this.getTokenIds() + "->" + this.nnId;
+ }
+
+ @Override
+ public int compareTo(ConnectionPoolId other) {
+ int ret = this.nnId.compareTo(other.nnId);
+ if (ret == 0) {
+ ret = this.ugi.toString().compareTo(other.ugi.toString());
+ }
+ if (ret == 0) {
+ String thisTokens = this.getTokenIds().toString();
+ String otherTokens = other.getTokenIds().toString();
+ ret = thisTokens.compareTo(otherTokens);
+ }
+ return ret;
+ }
+
+ /**
+ * Get the token identifiers for this connection.
+ * @return List with the token identifiers.
+ */
+ private List<String> getTokenIds() {
+ List<String> tokenIds = new ArrayList<>();
+ Collection<Token<? extends TokenIdentifier>> tokens = this.ugi.getTokens();
+ for (Token<? extends TokenIdentifier> token : tokens) {
+ byte[] tokenIdBytes = token.getIdentifier();
+ String tokenId = Arrays.toString(tokenIdBytes);
+ tokenIds.add(tokenId);
+ }
+ Collections.sort(tokenIds);
+ return tokenIds;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e96a2c7f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
index da6066b..a90c460 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
@@ -17,22 +17,52 @@
*/
package org.apache.hadoop.hdfs.server.federation.router;
+import org.apache.commons.lang.builder.HashCodeBuilder;
+
/**
- * Interface for objects that are unique to a namespace.
+ * Base class for objects that are unique to a namespace.
*/
-public interface RemoteLocationContext {
+public abstract class RemoteLocationContext
+ implements Comparable<RemoteLocationContext> {
/**
* Returns an identifier for a unique namespace.
*
* @return Namespace identifier.
*/
- String getNameserviceId();
+ public abstract String getNameserviceId();
/**
* Destination in this location. For example the path in a remote namespace.
*
* @return Destination in this location.
*/
- String getDest();
+ public abstract String getDest();
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder(17, 31)
+ .append(getNameserviceId())
+ .append(getDest())
+ .toHashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof RemoteLocationContext) {
+ RemoteLocationContext other = (RemoteLocationContext) obj;
+ return this.getNameserviceId().equals(other.getNameserviceId()) &&
+ this.getDest().equals(other.getDest());
+ }
+ return false;
+ }
+
+ @Override
+ public int compareTo(RemoteLocationContext info) {
+ int ret = this.getNameserviceId().compareTo(info.getNameserviceId());
+ if (ret == 0) {
+ ret = this.getDest().compareTo(info.getDest());
+ }
+ return ret;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e96a2c7f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java
new file mode 100644
index 0000000..cd57d45
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.Arrays;
+
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Determines the remote client protocol method and the parameter list for a
+ * specific location.
+ */
+public class RemoteMethod {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RemoteMethod.class);
+
+
+ /** List of parameters: static and dynamic values, matchings types. */
+ private final Object[] params;
+ /** List of method parameters types, matches parameters. */
+ private final Class<?>[] types;
+ /** String name of the ClientProtocol method. */
+ private final String methodName;
+
+ /**
+ * Create a method with no parameters.
+ *
+ * @param method The string name of the ClientProtocol method.
+ */
+ public RemoteMethod(String method) {
+ this.params = null;
+ this.types = null;
+ this.methodName = method;
+ }
+
+ /**
+ * Creates a remote method generator.
+ *
+ * @param method The string name of the ClientProtocol method.
+ * @param pTypes A list of types to use to locate the specific method.
+ * @param pParams A list of parameters for the method. The order of the
+ * parameter list must match the order and number of the types.
+ * Parameters are grouped into 2 categories:
+ * <ul>
+ * <li>Static parameters that are immutable across locations.
+ * <li>Dynamic parameters that are determined for each location by a
+ * RemoteParam object. To specify a dynamic parameter, pass an
+ * instance of RemoteParam in place of the parameter value.
+ * </ul>
+ * @throws IOException If the types and parameter lists are not valid.
+ */
+ public RemoteMethod(String method, Class<?>[] pTypes, Object... pParams)
+ throws IOException {
+
+ if (pParams.length != pTypes.length) {
+ throw new IOException("Invalid parameters for method " + method);
+ }
+
+ this.params = pParams;
+ this.types = pTypes;
+ this.methodName = method;
+ }
+
+ /**
+ * Get the represented java method.
+ *
+ * @return Method
+ * @throws IOException If the method cannot be found.
+ */
+ public Method getMethod() throws IOException {
+ try {
+ if (types != null) {
+ return ClientProtocol.class.getDeclaredMethod(methodName, types);
+ } else {
+ return ClientProtocol.class.getDeclaredMethod(methodName);
+ }
+ } catch (NoSuchMethodException e) {
+ // Re-throw as an IOException
+ LOG.error("Cannot get method {} with types {}",
+ methodName, Arrays.toString(types), e);
+ throw new IOException(e);
+ } catch (SecurityException e) {
+ LOG.error("Cannot access method {} with types {}",
+ methodName, Arrays.toString(types), e);
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * Get the calling types for this method.
+ *
+ * @return An array of calling types.
+ */
+ public Class<?>[] getTypes() {
+ return this.types;
+ }
+
+ /**
+ * Generate a list of parameters for this specific location using no context.
+ *
+ * @return A list of parameters for the method customized for the location.
+ */
+ public Object[] getParams() {
+ return this.getParams(null);
+ }
+
+ /**
+ * Get the name of the method.
+ *
+ * @return Name of the method.
+ */
+ public String getMethodName() {
+ return this.methodName;
+ }
+
+ /**
+ * Generate a list of parameters for this specific location. Parameters are
+ * grouped into 2 categories:
+ * <ul>
+ * <li>Static parameters that are immutable across locations.
+ * <li>Dynamic parameters that are determined for each location by a
+ * RemoteParam object.
+ * </ul>
+ *
+ * @param context The context identifying the location.
+ * @return A list of parameters for the method customized for the location.
+ */
+ public Object[] getParams(RemoteLocationContext context) {
+ if (this.params == null) {
+ return new Object[] {};
+ }
+ Object[] objList = new Object[this.params.length];
+ for (int i = 0; i < this.params.length; i++) {
+ Object currentObj = this.params[i];
+ if (currentObj instanceof RemoteParam) {
+ // Map the parameter using the context
+ RemoteParam paramGetter = (RemoteParam) currentObj;
+ objList[i] = paramGetter.getParameterForContext(context);
+ } else {
+ objList[i] = currentObj;
+ }
+ }
+ return objList;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e96a2c7f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteParam.java
new file mode 100644
index 0000000..8816ff6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteParam.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.util.Map;
+
+/**
+ * A dynamically assignable parameter that is location-specific.
+ * <p>
+ * There are 2 ways this mapping is determined:
+ * <ul>
+ * <li>Default: Uses the RemoteLocationContext's destination
+ * <li>Map: Uses the value of the RemoteLocationContext key provided in the
+ * parameter map.
+ * </ul>
+ */
+public class RemoteParam {
+
+ private final Map<? extends Object, ? extends Object> paramMap;
+
+ /**
+ * Constructs a default remote parameter. Always maps the value to the
+ * destination of the provided RemoveLocationContext.
+ */
+ public RemoteParam() {
+ this.paramMap = null;
+ }
+
+ /**
+ * Constructs a map based remote parameter. Determines the value using the
+ * provided RemoteLocationContext as a key into the map.
+ *
+ * @param map Map with RemoteLocationContext keys.
+ */
+ public RemoteParam(
+ Map<? extends RemoteLocationContext, ? extends Object> map) {
+ this.paramMap = map;
+ }
+
+ /**
+ * Determine the appropriate value for this parameter based on the location.
+ *
+ * @param context Context identifying the location.
+ * @return A parameter specific to this location.
+ */
+ public Object getParameterForContext(RemoteLocationContext context) {
+ if (context == null) {
+ return null;
+ } else if (this.paramMap != null) {
+ return this.paramMap.get(context);
+ } else {
+ // Default case
+ return context.getDest();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e96a2c7f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
index fe0d02a..019a5cd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
@@ -22,12 +22,14 @@ import static org.apache.hadoop.hdfs.server.federation.router.FederationUtil.new
import static org.apache.hadoop.util.ExitUtil.terminate;
import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
@@ -36,6 +38,8 @@ import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Router that provides a unified view of multiple federated HDFS clusters. It
@@ -60,7 +64,7 @@ import org.apache.hadoop.util.StringUtils;
@InterfaceStability.Evolving
public class Router extends CompositeService {
- private static final Log LOG = LogFactory.getLog(Router.class);
+ private static final Logger LOG = LoggerFactory.getLogger(Router.class);
/** Configuration for the Router. */
@@ -71,6 +75,7 @@ public class Router extends CompositeService {
/** RPC interface to the client. */
private RouterRpcServer rpcServer;
+ private InetSocketAddress rpcAddress;
/** Interface with the State Store. */
private StateStoreService stateStore;
@@ -105,9 +110,6 @@ public class Router extends CompositeService {
protected void serviceInit(Configuration configuration) throws Exception {
this.conf = configuration;
- // TODO Interface to the State Store
- this.stateStore = null;
-
// Resolver to track active NNs
this.namenodeResolver = newActiveNamenodeResolver(
this.conf, this.stateStore);
@@ -122,6 +124,15 @@ public class Router extends CompositeService {
throw new IOException("Cannot find subcluster resolver");
}
+ if (conf.getBoolean(
+ DFSConfigKeys.DFS_ROUTER_RPC_ENABLE,
+ DFSConfigKeys.DFS_ROUTER_RPC_ENABLE_DEFAULT)) {
+ // Create RPC server
+ this.rpcServer = createRpcServer();
+ addService(this.rpcServer);
+ this.setRpcServerAddress(rpcServer.getRpcAddress());
+ }
+
super.serviceInit(conf);
}
@@ -171,11 +182,13 @@ public class Router extends CompositeService {
router.init(conf);
router.start();
} catch (Throwable e) {
- LOG.error("Failed to start router.", e);
+ LOG.error("Failed to start router", e);
terminate(1, e);
}
}
+
+
/////////////////////////////////////////////////////////
// RPC Server
/////////////////////////////////////////////////////////
@@ -183,7 +196,7 @@ public class Router extends CompositeService {
/**
* Create a new Router RPC server to proxy ClientProtocol requests.
*
- * @return RouterRpcServer
+ * @return New Router RPC Server.
* @throws IOException If the router RPC server was not started.
*/
protected RouterRpcServer createRpcServer() throws IOException {
@@ -200,6 +213,35 @@ public class Router extends CompositeService {
return this.rpcServer;
}
+ /**
+ * Set the current RPC socket for the router.
+ *
+ * @param rpcAddress RPC address.
+ */
+ protected void setRpcServerAddress(InetSocketAddress address) {
+ this.rpcAddress = address;
+
+ // Use the RPC address as our unique router Id
+ if (this.rpcAddress != null) {
+ try {
+ String hostname = InetAddress.getLocalHost().getHostName();
+ setRouterId(hostname + ":" + this.rpcAddress.getPort());
+ } catch (UnknownHostException ex) {
+ LOG.error("Cannot set unique router ID, address not resolvable {}",
+ this.rpcAddress);
+ }
+ }
+ }
+
+ /**
+ * Get the current RPC socket address for the router.
+ *
+ * @return InetSocketAddress
+ */
+ public InetSocketAddress getRpcServerAddress() {
+ return this.rpcAddress;
+ }
+
/////////////////////////////////////////////////////////
// Submodule getters
/////////////////////////////////////////////////////////
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/40] hadoop git commit: HDFS-11826. Federation Namenode Heartbeat.
Contributed by Inigo Goiri.
Posted by in...@apache.org.
HDFS-11826. Federation Namenode Heartbeat. Contributed by Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ae5e1352
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ae5e1352
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ae5e1352
Branch: refs/heads/HDFS-10467
Commit: ae5e13522e30cb10796c3b627ebd7fec9597a88c
Parents: b43bb80
Author: Inigo Goiri <in...@apache.org>
Authored: Tue Aug 1 14:40:27 2017 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu Sep 7 13:19:51 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 14 +
.../java/org/apache/hadoop/hdfs/DFSUtil.java | 38 ++
.../resolver/NamenodeStatusReport.java | 193 ++++++++++
.../federation/router/FederationUtil.java | 66 ++++
.../router/NamenodeHeartbeatService.java | 350 +++++++++++++++++++
.../hdfs/server/federation/router/Router.java | 112 ++++++
.../src/main/resources/hdfs-default.xml | 32 ++
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 8 +
.../hdfs/server/federation/MockResolver.java | 9 +-
.../server/federation/RouterConfigBuilder.java | 22 ++
.../server/federation/RouterDFSCluster.java | 43 +++
.../router/TestNamenodeHeartbeat.java | 168 +++++++++
.../server/federation/router/TestRouter.java | 3 +
13 files changed, 1057 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae5e1352/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index b9c7408..1ad34d4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -1147,6 +1147,20 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
FEDERATION_ROUTER_PREFIX + "rpc.enable";
public static final boolean DFS_ROUTER_RPC_ENABLE_DEFAULT = true;
+ // HDFS Router heartbeat
+ public static final String DFS_ROUTER_HEARTBEAT_ENABLE =
+ FEDERATION_ROUTER_PREFIX + "heartbeat.enable";
+ public static final boolean DFS_ROUTER_HEARTBEAT_ENABLE_DEFAULT = true;
+ public static final String DFS_ROUTER_HEARTBEAT_INTERVAL_MS =
+ FEDERATION_ROUTER_PREFIX + "heartbeat.interval";
+ public static final long DFS_ROUTER_HEARTBEAT_INTERVAL_MS_DEFAULT =
+ TimeUnit.SECONDS.toMillis(5);
+ public static final String DFS_ROUTER_MONITOR_NAMENODE =
+ FEDERATION_ROUTER_PREFIX + "monitor.namenode";
+ public static final String DFS_ROUTER_MONITOR_LOCAL_NAMENODE =
+ FEDERATION_ROUTER_PREFIX + "monitor.localnamenode.enable";
+ public static final boolean DFS_ROUTER_MONITOR_LOCAL_NAMENODE_DEFAULT = true;
+
// HDFS Router NN client
public static final String DFS_ROUTER_NAMENODE_CONNECTION_POOL_SIZE =
FEDERATION_ROUTER_PREFIX + "connection.pool-size";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae5e1352/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
index 7776dc2..29936f1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
@@ -1241,6 +1241,44 @@ public class DFSUtil {
}
/**
+ * Map a logical namenode ID to its web address. Use the given nameservice if
+ * specified, or the configured one if none is given.
+ *
+ * @param conf Configuration
+ * @param nsId which nameservice nnId is a part of, optional
+ * @param nnId the namenode ID to get the service addr for
+ * @return the service addr, null if it could not be determined
+ */
+ public static String getNamenodeWebAddr(final Configuration conf, String nsId,
+ String nnId) {
+
+ if (nsId == null) {
+ nsId = getOnlyNameServiceIdOrNull(conf);
+ }
+
+ String webAddrKey = DFSUtilClient.concatSuffixes(
+ DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, nsId, nnId);
+
+ String webAddr =
+ conf.get(webAddrKey, DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_DEFAULT);
+ return webAddr;
+ }
+
+ /**
+ * Get all of the Web addresses of the individual NNs in a given nameservice.
+ *
+ * @param conf Configuration
+ * @param nsId the nameservice whose NNs addresses we want.
+ * @param defaultValue default address to return in case key is not found.
+ * @return A map from nnId -> Web address of each NN in the nameservice.
+ */
+ public static Map<String, InetSocketAddress> getWebAddressesForNameserviceId(
+ Configuration conf, String nsId, String defaultValue) {
+ return DFSUtilClient.getAddressesForNameserviceId(conf, nsId, defaultValue,
+ DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY);
+ }
+
+ /**
* If the configuration refers to only a single nameservice, return the
* name of that nameservice. If it refers to 0 or more than 1, return null.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae5e1352/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java
index 9259048..f8759e8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java
@@ -39,8 +39,29 @@ public class NamenodeStatusReport {
private HAServiceState status = HAServiceState.STANDBY;
private boolean safeMode = false;
+ /** Datanodes stats. */
+ private int liveDatanodes = -1;
+ private int deadDatanodes = -1;
+ /** Decommissioning datanodes. */
+ private int decomDatanodes = -1;
+ /** Live decommissioned datanodes. */
+ private int liveDecomDatanodes = -1;
+ /** Dead decommissioned datanodes. */
+ private int deadDecomDatanodes = -1;
+
+ /** Space stats. */
+ private long availableSpace = -1;
+ private long numOfFiles = -1;
+ private long numOfBlocks = -1;
+ private long numOfBlocksMissing = -1;
+ private long numOfBlocksPendingReplication = -1;
+ private long numOfBlocksUnderReplicated = -1;
+ private long numOfBlocksPendingDeletion = -1;
+ private long totalSpace = -1;
+
/** If the fields are valid. */
private boolean registrationValid = false;
+ private boolean statsValid = false;
private boolean haStateValid = false;
public NamenodeStatusReport(String ns, String nn, String rpc, String service,
@@ -54,6 +75,15 @@ public class NamenodeStatusReport {
}
/**
+ * If the statistics are valid.
+ *
+ * @return If the statistics are valid.
+ */
+ public boolean statsValid() {
+ return this.statsValid;
+ }
+
+ /**
* If the registration is valid.
*
* @return If the registration is valid.
@@ -187,6 +217,169 @@ public class NamenodeStatusReport {
return this.safeMode;
}
+ /**
+ * Set the datanode information.
+ *
+ * @param numLive Number of live nodes.
+ * @param numDead Number of dead nodes.
+ * @param numDecom Number of decommissioning nodes.
+ * @param numLiveDecom Number of decommissioned live nodes.
+ * @param numDeadDecom Number of decommissioned dead nodes.
+ */
+ public void setDatanodeInfo(int numLive, int numDead, int numDecom,
+ int numLiveDecom, int numDeadDecom) {
+ this.liveDatanodes = numLive;
+ this.deadDatanodes = numDead;
+ this.decomDatanodes = numDecom;
+ this.liveDecomDatanodes = numLiveDecom;
+ this.deadDecomDatanodes = numDeadDecom;
+ this.statsValid = true;
+ }
+
+ /**
+ * Get the number of live blocks.
+ *
+ * @return The number of dead nodes.
+ */
+ public int getNumLiveDatanodes() {
+ return this.liveDatanodes;
+ }
+
+ /**
+ * Get the number of dead blocks.
+ *
+ * @return The number of dead nodes.
+ */
+ public int getNumDeadDatanodes() {
+ return this.deadDatanodes;
+ }
+
+ /**
+ * Get the number of decommissionining nodes.
+ *
+ * @return The number of decommissionining nodes.
+ */
+ public int getNumDecommissioningDatanodes() {
+ return this.decomDatanodes;
+ }
+
+ /**
+ * Get the number of live decommissioned nodes.
+ *
+ * @return The number of live decommissioned nodes.
+ */
+ public int getNumDecomLiveDatanodes() {
+ return this.liveDecomDatanodes;
+ }
+
+ /**
+ * Get the number of dead decommissioned nodes.
+ *
+ * @return The number of dead decommissioned nodes.
+ */
+ public int getNumDecomDeadDatanodes() {
+ return this.deadDecomDatanodes;
+ }
+
+ /**
+ * Set the filesystem information.
+ *
+ * @param available Available capacity.
+ * @param total Total capacity.
+ * @param numFiles Number of files.
+ * @param numBlocks Total number of blocks.
+ * @param numBlocksMissing Number of missing blocks.
+ * @param numOfBlocksPendingReplication Number of blocks pending replication.
+ * @param numOfBlocksUnderReplicated Number of blocks under replication.
+ * @param numOfBlocksPendingDeletion Number of blocks pending deletion.
+ */
+ public void setNamesystemInfo(long available, long total,
+ long numFiles, long numBlocks, long numBlocksMissing,
+ long numBlocksPendingReplication, long numBlocksUnderReplicated,
+ long numBlocksPendingDeletion) {
+ this.totalSpace = total;
+ this.availableSpace = available;
+ this.numOfBlocks = numBlocks;
+ this.numOfBlocksMissing = numBlocksMissing;
+ this.numOfBlocksPendingReplication = numBlocksPendingReplication;
+ this.numOfBlocksUnderReplicated = numBlocksUnderReplicated;
+ this.numOfBlocksPendingDeletion = numBlocksPendingDeletion;
+ this.numOfFiles = numFiles;
+ this.statsValid = true;
+ }
+
+ /**
+ * Get the number of blocks.
+ *
+ * @return The number of blocks.
+ */
+ public long getNumBlocks() {
+ return this.numOfBlocks;
+ }
+
+ /**
+ * Get the number of files.
+ *
+ * @return The number of files.
+ */
+ public long getNumFiles() {
+ return this.numOfFiles;
+ }
+
+ /**
+ * Get the total space.
+ *
+ * @return The total space.
+ */
+ public long getTotalSpace() {
+ return this.totalSpace;
+ }
+
+ /**
+ * Get the available space.
+ *
+ * @return The available space.
+ */
+ public long getAvailableSpace() {
+ return this.availableSpace;
+ }
+
+ /**
+ * Get the number of missing blocks.
+ *
+ * @return Number of missing blocks.
+ */
+ public long getNumBlocksMissing() {
+ return this.numOfBlocksMissing;
+ }
+
+ /**
+ * Get the number of pending replication blocks.
+ *
+ * @return Number of pending replication blocks.
+ */
+ public long getNumOfBlocksPendingReplication() {
+ return this.numOfBlocksPendingReplication;
+ }
+
+ /**
+ * Get the number of under replicated blocks.
+ *
+ * @return Number of under replicated blocks.
+ */
+ public long getNumOfBlocksUnderReplicated() {
+ return this.numOfBlocksUnderReplicated;
+ }
+
+ /**
+ * Get the number of pending deletion blocks.
+ *
+ * @return Number of pending deletion blocks.
+ */
+ public long getNumOfBlocksPendingDeletion() {
+ return this.numOfBlocksPendingDeletion;
+ }
+
@Override
public String toString() {
return String.format("%s-%s:%s",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae5e1352/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
index 0129a37..78c473a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
@@ -17,13 +17,22 @@
*/
package org.apache.hadoop.hdfs.server.federation.router;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
import java.lang.reflect.Constructor;
+import java.net.URL;
+import java.net.URLConnection;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,6 +49,63 @@ public final class FederationUtil {
}
/**
+ * Get a JMX data from a web endpoint.
+ *
+ * @param beanQuery JMX bean.
+ * @param webAddress Web address of the JMX endpoint.
+ * @return JSON with the JMX data
+ */
+ public static JSONArray getJmx(String beanQuery, String webAddress) {
+ JSONArray ret = null;
+ BufferedReader reader = null;
+ try {
+ String host = webAddress;
+ int port = -1;
+ if (webAddress.indexOf(":") > 0) {
+ String[] webAddressSplit = webAddress.split(":");
+ host = webAddressSplit[0];
+ port = Integer.parseInt(webAddressSplit[1]);
+ }
+ URL jmxURL = new URL("http", host, port, "/jmx?qry=" + beanQuery);
+ URLConnection conn = jmxURL.openConnection();
+ conn.setConnectTimeout(5 * 1000);
+ conn.setReadTimeout(5 * 1000);
+ InputStream in = conn.getInputStream();
+ InputStreamReader isr = new InputStreamReader(in, "UTF-8");
+ reader = new BufferedReader(isr);
+
+ StringBuilder sb = new StringBuilder();
+ String line = null;
+ while ((line = reader.readLine()) != null) {
+ sb.append(line);
+ }
+ String jmxOutput = sb.toString();
+
+ // Parse JSON
+ JSONObject json = new JSONObject(jmxOutput);
+ ret = json.getJSONArray("beans");
+ } catch (IOException e) {
+ LOG.error("Cannot read JMX bean {} from server {}: {}",
+ beanQuery, webAddress, e.getMessage());
+ } catch (JSONException e) {
+ LOG.error("Cannot parse JMX output for {} from server {}: {}",
+ beanQuery, webAddress, e.getMessage());
+ } catch (Exception e) {
+ LOG.error("Cannot parse JMX output for {} from server {}: {}",
+ beanQuery, webAddress, e);
+ } finally {
+ if (reader != null) {
+ try {
+ reader.close();
+ } catch (IOException e) {
+ LOG.error("Problem closing {}", webAddress, e);
+ }
+ }
+ }
+ return ret;
+ }
+
+ /**
* Create an instance of an interface with a constructor using a context.
*
* @param conf Configuration for the class names.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae5e1352/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
new file mode 100644
index 0000000..fe4f939
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
@@ -0,0 +1,350 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HEARTBEAT_INTERVAL_MS;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HEARTBEAT_INTERVAL_MS_DEFAULT;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ha.HAServiceProtocol;
+import org.apache.hadoop.ha.HAServiceStatus;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.NameNodeProxies;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport;
+import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
+import org.apache.hadoop.hdfs.tools.NNHAServiceTarget;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The {@link Router} periodically checks the state of a Namenode (usually on
+ * the same server) and reports their high availability (HA) state and
+ * load/space status to the
+ * {@link org.apache.hadoop.hdfs.server.federation.store.StateStoreService}
+ * . Note that this is an optional role as a Router can be independent of any
+ * subcluster.
+ * <p>
+ * For performance with Namenode HA, the Router uses the high availability state
+ * information in the State Store to forward the request to the Namenode that is
+ * most likely to be active.
+ * <p>
+ * Note that this service can be embedded into the Namenode itself to simplify
+ * the operation.
+ */
+public class NamenodeHeartbeatService extends PeriodicService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NamenodeHeartbeatService.class);
+
+
+ /** Configuration for the heartbeat. */
+ private Configuration conf;
+
+ /** Router performing the heartbeating. */
+ private final ActiveNamenodeResolver resolver;
+
+ /** Interface to the tracked NN. */
+ private final String nameserviceId;
+ private final String namenodeId;
+
+ /** Namenode HA target. */
+ private NNHAServiceTarget localTarget;
+ /** RPC address for the namenode. */
+ private String rpcAddress;
+ /** Service RPC address for the namenode. */
+ private String serviceAddress;
+ /** Service RPC address for the namenode. */
+ private String lifelineAddress;
+ /** HTTP address for the namenode. */
+ private String webAddress;
+
+ /**
+ * Create a new Namenode status updater.
+ * @param resolver Namenode resolver service to handle NN registration.
+ * @param nameserviceId Identifier of the nameservice.
+ * @param namenodeId Identifier of the namenode in HA.
+ */
+ public NamenodeHeartbeatService(
+ ActiveNamenodeResolver resolver, String nsId, String nnId) {
+ super(NamenodeHeartbeatService.class.getSimpleName() + " " + nsId + " " +
+ nnId);
+
+ this.resolver = resolver;
+
+ this.nameserviceId = nsId;
+ this.namenodeId = nnId;
+
+ }
+
+ @Override
+ protected void serviceInit(Configuration configuration) throws Exception {
+
+ this.conf = configuration;
+
+ if (this.namenodeId != null && !this.namenodeId.isEmpty()) {
+ this.localTarget = new NNHAServiceTarget(
+ conf, nameserviceId, namenodeId);
+ } else {
+ this.localTarget = null;
+ }
+
+ // Get the RPC address for the clients to connect
+ this.rpcAddress = getRpcAddress(conf, nameserviceId, namenodeId);
+ LOG.info("{}-{} RPC address: {}",
+ nameserviceId, namenodeId, rpcAddress);
+
+ // Get the Service RPC address for monitoring
+ this.serviceAddress =
+ DFSUtil.getNamenodeServiceAddr(conf, nameserviceId, namenodeId);
+ if (this.serviceAddress == null) {
+ LOG.error("Cannot locate RPC service address for NN {}-{}, " +
+ "using RPC address {}", nameserviceId, namenodeId, this.rpcAddress);
+ this.serviceAddress = this.rpcAddress;
+ }
+ LOG.info("{}-{} Service RPC address: {}",
+ nameserviceId, namenodeId, serviceAddress);
+
+ // Get the Lifeline RPC address for faster monitoring
+ this.lifelineAddress =
+ DFSUtil.getNamenodeLifelineAddr(conf, nameserviceId, namenodeId);
+ if (this.lifelineAddress == null) {
+ this.lifelineAddress = this.serviceAddress;
+ }
+ LOG.info("{}-{} Lifeline RPC address: {}",
+ nameserviceId, namenodeId, lifelineAddress);
+
+ // Get the Web address for UI
+ this.webAddress =
+ DFSUtil.getNamenodeWebAddr(conf, nameserviceId, namenodeId);
+ LOG.info("{}-{} Web address: {}", nameserviceId, namenodeId, webAddress);
+
+ this.setIntervalMs(conf.getLong(
+ DFS_ROUTER_HEARTBEAT_INTERVAL_MS,
+ DFS_ROUTER_HEARTBEAT_INTERVAL_MS_DEFAULT));
+
+
+ super.serviceInit(configuration);
+ }
+
+ @Override
+ public void periodicInvoke() {
+ updateState();
+ }
+
+ /**
+ * Get the RPC address for a Namenode.
+ * @param conf Configuration.
+ * @param nsId Name service identifier.
+ * @param nnId Name node identifier.
+ * @return RPC address in format hostname:1234.
+ */
+ private static String getRpcAddress(
+ Configuration conf, String nsId, String nnId) {
+
+ // Get it from the regular RPC setting
+ String confKey = DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY;
+ String ret = conf.get(confKey);
+
+ if (nsId != null && nnId != null) {
+ // Get if for the proper nameservice and namenode
+ confKey = DFSUtil.addKeySuffixes(confKey, nsId, nnId);
+ ret = conf.get(confKey);
+
+ // If not available, get it from the map
+ if (ret == null) {
+ Map<String, InetSocketAddress> rpcAddresses =
+ DFSUtil.getRpcAddressesForNameserviceId(conf, nsId, null);
+ if (rpcAddresses.containsKey(nnId)) {
+ InetSocketAddress sockAddr = rpcAddresses.get(nnId);
+ InetAddress addr = sockAddr.getAddress();
+ ret = addr.getHostAddress() + ":" + sockAddr.getPort();
+ }
+ }
+ }
+ return ret;
+ }
+
+ /**
+ * Update the state of the Namenode.
+ */
+ private void updateState() {
+ NamenodeStatusReport report = getNamenodeStatusReport();
+ if (!report.registrationValid()) {
+ // Not operational
+ LOG.error("Namenode is not operational: {}", getNamenodeDesc());
+ } else if (report.haStateValid()) {
+ // block and HA status available
+ LOG.debug("Received service state: {} from HA namenode: {}",
+ report.getState(), getNamenodeDesc());
+ } else if (localTarget == null) {
+ // block info available, HA status not expected
+ LOG.debug(
+ "Reporting non-HA namenode as operational: " + getNamenodeDesc());
+ } else {
+ // block info available, HA status should be available, but was not
+ // fetched do nothing and let the current state stand
+ return;
+ }
+ try {
+ if (!resolver.registerNamenode(report)) {
+ LOG.warn("Cannot register namenode {}", report);
+ }
+ } catch (IOException e) {
+ LOG.info("Cannot register namenode in the State Store");
+ } catch (Exception ex) {
+ LOG.error("Unhandled exception updating NN registration for {}",
+ getNamenodeDesc(), ex);
+ }
+ }
+
+ /**
+ * Get the status report for the Namenode monitored by this heartbeater.
+ * @return Namenode status report.
+ */
+ protected NamenodeStatusReport getNamenodeStatusReport() {
+ NamenodeStatusReport report = new NamenodeStatusReport(nameserviceId,
+ namenodeId, rpcAddress, serviceAddress, lifelineAddress, webAddress);
+
+ try {
+ LOG.debug("Probing NN at service address: {}", serviceAddress);
+
+ URI serviceURI = new URI("hdfs://" + serviceAddress);
+ // Read the filesystem info from RPC (required)
+ NamenodeProtocol nn = NameNodeProxies
+ .createProxy(this.conf, serviceURI, NamenodeProtocol.class)
+ .getProxy();
+
+ if (nn != null) {
+ NamespaceInfo info = nn.versionRequest();
+ if (info != null) {
+ report.setNamespaceInfo(info);
+ }
+ }
+ if (!report.registrationValid()) {
+ return report;
+ }
+
+ // Check for safemode from the client protocol. Currently optional, but
+ // should be required at some point for QoS
+ try {
+ ClientProtocol client = NameNodeProxies
+ .createProxy(this.conf, serviceURI, ClientProtocol.class)
+ .getProxy();
+ if (client != null) {
+ boolean isSafeMode = client.setSafeMode(
+ SafeModeAction.SAFEMODE_GET, false);
+ report.setSafeMode(isSafeMode);
+ }
+ } catch (Exception e) {
+ LOG.error("Cannot fetch safemode state for {}", getNamenodeDesc(), e);
+ }
+
+ // Read the stats from JMX (optional)
+ updateJMXParameters(webAddress, report);
+
+ if (localTarget != null) {
+ // Try to get the HA status
+ try {
+ // Determine if NN is active
+ // TODO: dynamic timeout
+ HAServiceProtocol haProtocol = localTarget.getProxy(conf, 30*1000);
+ HAServiceStatus status = haProtocol.getServiceStatus();
+ report.setHAServiceState(status.getState());
+ } catch (Throwable e) {
+ // Failed to fetch HA status, ignoring failure
+ LOG.error("Cannot fetch HA status for {}: {}",
+ getNamenodeDesc(), e.getMessage(), e);
+ }
+ }
+ } catch(IOException e) {
+ LOG.error("Cannot communicate with {}: {}",
+ getNamenodeDesc(), e.getMessage());
+ } catch(Throwable e) {
+ // Generic error that we don't know about
+ LOG.error("Unexpected exception while communicating with {}: {}",
+ getNamenodeDesc(), e.getMessage(), e);
+ }
+ return report;
+ }
+
+ /**
+ * Get the description of the Namenode to monitor.
+ * @return Description of the Namenode to monitor.
+ */
+ public String getNamenodeDesc() {
+ if (namenodeId != null && !namenodeId.isEmpty()) {
+ return nameserviceId + "-" + namenodeId + ":" + serviceAddress;
+ } else {
+ return nameserviceId + ":" + serviceAddress;
+ }
+ }
+
+ /**
+ * Get the parameters for a Namenode from JMX and add them to the report.
+ * @param webAddress Web interface of the Namenode to monitor.
+ * @param report Namenode status report to update with JMX data.
+ */
+ private void updateJMXParameters(
+ String address, NamenodeStatusReport report) {
+ try {
+ // TODO part of this should be moved to its own utility
+ String query = "Hadoop:service=NameNode,name=FSNamesystem*";
+ JSONArray aux = FederationUtil.getJmx(query, address);
+ if (aux != null) {
+ for (int i = 0; i < aux.length(); i++) {
+ JSONObject jsonObject = aux.getJSONObject(i);
+ String name = jsonObject.getString("name");
+ if (name.equals("Hadoop:service=NameNode,name=FSNamesystemState")) {
+ report.setDatanodeInfo(
+ jsonObject.getInt("NumLiveDataNodes"),
+ jsonObject.getInt("NumDeadDataNodes"),
+ jsonObject.getInt("NumDecommissioningDataNodes"),
+ jsonObject.getInt("NumDecomLiveDataNodes"),
+ jsonObject.getInt("NumDecomDeadDataNodes"));
+ } else if (name.equals(
+ "Hadoop:service=NameNode,name=FSNamesystem")) {
+ report.setNamesystemInfo(
+ jsonObject.getLong("CapacityRemaining"),
+ jsonObject.getLong("CapacityTotal"),
+ jsonObject.getLong("FilesTotal"),
+ jsonObject.getLong("BlocksTotal"),
+ jsonObject.getLong("MissingBlocks"),
+ jsonObject.getLong("PendingReplicationBlocks"),
+ jsonObject.getLong("UnderReplicatedBlocks"),
+ jsonObject.getLong("PendingDeletionBlocks"));
+ }
+ }
+ }
+ } catch (Exception e) {
+ LOG.error("Cannot get stat from {} using JMX", getNamenodeDesc(), e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae5e1352/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
index 019a5cd..cfddf20 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
@@ -25,12 +25,16 @@ import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
@@ -85,6 +89,8 @@ public class Router extends CompositeService {
/** Interface to identify the active NN for a nameservice or blockpool ID. */
private ActiveNamenodeResolver namenodeResolver;
+ /** Updates the namenode status in the namenode resolver. */
+ private Collection<NamenodeHeartbeatService> namenodeHearbeatServices;
/** Usage string for help message. */
@@ -133,6 +139,22 @@ public class Router extends CompositeService {
this.setRpcServerAddress(rpcServer.getRpcAddress());
}
+ if (conf.getBoolean(
+ DFSConfigKeys.DFS_ROUTER_HEARTBEAT_ENABLE,
+ DFSConfigKeys.DFS_ROUTER_HEARTBEAT_ENABLE_DEFAULT)) {
+
+ // Create status updater for each monitored Namenode
+ this.namenodeHearbeatServices = createNamenodeHearbeatServices();
+ for (NamenodeHeartbeatService hearbeatService :
+ this.namenodeHearbeatServices) {
+ addService(hearbeatService);
+ }
+
+ if (this.namenodeHearbeatServices.isEmpty()) {
+ LOG.error("Heartbeat is enabled but there are no namenodes to monitor");
+ }
+ }
+
super.serviceInit(conf);
}
@@ -243,6 +265,96 @@ public class Router extends CompositeService {
}
/////////////////////////////////////////////////////////
+ // Namenode heartbeat monitors
+ /////////////////////////////////////////////////////////
+
+ /**
+ * Create each of the services that will monitor a Namenode.
+ *
+ * @return List of heartbeat services.
+ */
+ protected Collection<NamenodeHeartbeatService>
+ createNamenodeHearbeatServices() {
+
+ Map<String, NamenodeHeartbeatService> ret = new HashMap<>();
+
+ if (conf.getBoolean(
+ DFSConfigKeys.DFS_ROUTER_MONITOR_LOCAL_NAMENODE,
+ DFSConfigKeys.DFS_ROUTER_MONITOR_LOCAL_NAMENODE_DEFAULT)) {
+ // Create a local heartbet service
+ NamenodeHeartbeatService localHeartbeatService =
+ createLocalNamenodeHearbeatService();
+ if (localHeartbeatService != null) {
+ String nnDesc = localHeartbeatService.getNamenodeDesc();
+ ret.put(nnDesc, localHeartbeatService);
+ }
+ }
+
+ // Create heartbeat services for a list specified by the admin
+ String namenodes = this.conf.get(
+ DFSConfigKeys.DFS_ROUTER_MONITOR_NAMENODE);
+ if (namenodes != null) {
+ for (String namenode : namenodes.split(",")) {
+ String[] namenodeSplit = namenode.split("\\.");
+ String nsId = null;
+ String nnId = null;
+ if (namenodeSplit.length == 2) {
+ nsId = namenodeSplit[0];
+ nnId = namenodeSplit[1];
+ } else if (namenodeSplit.length == 1) {
+ nsId = namenode;
+ } else {
+ LOG.error("Wrong Namenode to monitor: {}", namenode);
+ }
+ if (nsId != null) {
+ NamenodeHeartbeatService heartbeatService =
+ createNamenodeHearbeatService(nsId, nnId);
+ if (heartbeatService != null) {
+ ret.put(heartbeatService.getNamenodeDesc(), heartbeatService);
+ }
+ }
+ }
+ }
+
+ return ret.values();
+ }
+
+ /**
+ * Create a new status updater for the local Namenode.
+ *
+ * @return Updater of the status for the local Namenode.
+ */
+ protected NamenodeHeartbeatService createLocalNamenodeHearbeatService() {
+ // Detect NN running in this machine
+ String nsId = DFSUtil.getNamenodeNameServiceId(conf);
+ String nnId = null;
+ if (HAUtil.isHAEnabled(conf, nsId)) {
+ nnId = HAUtil.getNameNodeId(conf, nsId);
+ if (nnId == null) {
+ LOG.error("Cannot find namenode id for local {}", nsId);
+ }
+ }
+
+ return createNamenodeHearbeatService(nsId, nnId);
+ }
+
+ /**
+ * Create a heartbeat monitor for a particular Namenode.
+ *
+ * @param nsId Identifier of the nameservice to monitor.
+ * @param nnId Identifier of the namenode (HA) to monitor.
+ * @return Updater of the status for the specified Namenode.
+ */
+ protected NamenodeHeartbeatService createNamenodeHearbeatService(
+ String nsId, String nnId) {
+
+ LOG.info("Creating heartbeat service for Namenode {} in {}", nnId, nsId);
+ NamenodeHeartbeatService ret = new NamenodeHeartbeatService(
+ namenodeResolver, nsId, nnId);
+ return ret;
+ }
+
+ /////////////////////////////////////////////////////////
// Submodule getters
/////////////////////////////////////////////////////////
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae5e1352/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 3189468..b7c0f5d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4782,4 +4782,36 @@
</description>
</property>
+ <property>
+ <name>dfs.federation.router.heartbeat.enable</name>
+ <value>true</value>
+ <description>
+ Enables the Router to heartbeat into the State Store.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.heartbeat.interval</name>
+ <value>5000</value>
+ <description>
+ How often the Router should heartbeat into the State Store in milliseconds.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.monitor.namenode</name>
+ <value></value>
+ <description>
+ The identifier of the namenodes to monitor and heartbeat.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.monitor.localnamenode.enable</name>
+ <value>true</value>
+ <description>
+ If the Router should monitor the namenode in the local machine.
+ </description>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae5e1352/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 0345cf5..da91006 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -582,6 +582,14 @@ public class MiniDFSCluster implements AutoCloseable {
public void setStartOpt(StartupOption startOpt) {
this.startOpt = startOpt;
}
+
+ public String getNameserviceId() {
+ return this.nameserviceId;
+ }
+
+ public String getNamenodeId() {
+ return this.nnId;
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae5e1352/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
index 2875750..87427fd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
@@ -56,9 +56,16 @@ public class MockResolver
private Set<FederationNamespaceInfo> namespaces = new HashSet<>();
private String defaultNamespace = null;
+ public MockResolver() {
+ this.cleanRegistrations();
+ }
+
+ public MockResolver(Configuration conf) {
+ this();
+ }
public MockResolver(Configuration conf, StateStoreService store) {
- this.cleanRegistrations();
+ this();
}
public void addLocation(String mount, String nsId, String location) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae5e1352/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
index 39fcf7a..21555c5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
@@ -28,6 +28,8 @@ public class RouterConfigBuilder {
private Configuration conf;
private boolean enableRpcServer = false;
+ private boolean enableHeartbeat = false;
+ private boolean enableLocalHeartbeat = false;
public RouterConfigBuilder(Configuration configuration) {
this.conf = configuration;
@@ -39,6 +41,13 @@ public class RouterConfigBuilder {
public RouterConfigBuilder all() {
this.enableRpcServer = true;
+ this.enableHeartbeat = true;
+ this.enableLocalHeartbeat = true;
+ return this;
+ }
+
+ public RouterConfigBuilder enableLocalHeartbeat(boolean enable) {
+ this.enableLocalHeartbeat = enable;
return this;
}
@@ -47,12 +56,25 @@ public class RouterConfigBuilder {
return this;
}
+ public RouterConfigBuilder heartbeat(boolean enable) {
+ this.enableHeartbeat = enable;
+ return this;
+ }
+
public RouterConfigBuilder rpc() {
return this.rpc(true);
}
+ public RouterConfigBuilder heartbeat() {
+ return this.heartbeat(true);
+ }
+
public Configuration build() {
conf.setBoolean(DFSConfigKeys.DFS_ROUTER_RPC_ENABLE, this.enableRpcServer);
+ conf.setBoolean(DFSConfigKeys.DFS_ROUTER_HEARTBEAT_ENABLE,
+ this.enableHeartbeat);
+ conf.setBoolean(DFSConfigKeys.DFS_ROUTER_MONITOR_LOCAL_NAMENODE,
+ this.enableLocalHeartbeat);
return conf;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae5e1352/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
index 4031b7f..0830c19 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSCluster.NameNodeInfo;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.MiniDFSNNTopology.NNConf;
import org.apache.hadoop.hdfs.MiniDFSNNTopology.NSConf;
@@ -754,6 +755,48 @@ public class RouterDFSCluster {
}
/**
+ * Switch a namenode in a nameservice to be the active.
+ * @param nsId Nameservice identifier.
+ * @param nnId Namenode identifier.
+ */
+ public void switchToActive(String nsId, String nnId) {
+ try {
+ int total = cluster.getNumNameNodes();
+ NameNodeInfo[] nns = cluster.getNameNodeInfos();
+ for (int i = 0; i < total; i++) {
+ NameNodeInfo nn = nns[i];
+ if (nn.getNameserviceId().equals(nsId) &&
+ nn.getNamenodeId().equals(nnId)) {
+ cluster.transitionToActive(i);
+ }
+ }
+ } catch (Throwable e) {
+ LOG.error("Cannot transition to active", e);
+ }
+ }
+
+ /**
+ * Switch a namenode in a nameservice to be in standby.
+ * @param nsId Nameservice identifier.
+ * @param nnId Namenode identifier.
+ */
+ public void switchToStandby(String nsId, String nnId) {
+ try {
+ int total = cluster.getNumNameNodes();
+ NameNodeInfo[] nns = cluster.getNameNodeInfos();
+ for (int i = 0; i < total; i++) {
+ NameNodeInfo nn = nns[i];
+ if (nn.getNameserviceId().equals(nsId) &&
+ nn.getNamenodeId().equals(nnId)) {
+ cluster.transitionToStandby(i);
+ }
+ }
+ } catch (Throwable e) {
+ LOG.error("Cannot transition to standby", e);
+ }
+ }
+
+ /**
* Stop the federated HDFS cluster.
*/
public void shutdown() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae5e1352/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestNamenodeHeartbeat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestNamenodeHeartbeat.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestNamenodeHeartbeat.java
new file mode 100644
index 0000000..877fb02
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestNamenodeHeartbeat.java
@@ -0,0 +1,168 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMENODES;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMESERVICES;
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.MockResolver;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
+import org.apache.hadoop.service.Service.STATE;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+
+/**
+ * Test the service that heartbeats the state of the namenodes to the State
+ * Store.
+ */
+public class TestNamenodeHeartbeat {
+
+ private static RouterDFSCluster cluster;
+ private static ActiveNamenodeResolver namenodeResolver;
+ private static List<NamenodeHeartbeatService> services;
+
+ @Rule
+ public TestName name = new TestName();
+
+ @BeforeClass
+ public static void globalSetUp() throws Exception {
+
+ cluster = new RouterDFSCluster(true, 2);
+
+ // Start NNs and DNs and wait until ready
+ cluster.startCluster();
+
+ // Mock locator that records the heartbeats
+ List<String> nss = cluster.getNameservices();
+ String ns = nss.get(0);
+ Configuration conf = cluster.generateNamenodeConfiguration(ns);
+ namenodeResolver = new MockResolver(conf);
+ namenodeResolver.setRouterId("testrouter");
+
+ // Create one heartbeat service per NN
+ services = new ArrayList<>();
+ for (NamenodeContext nn : cluster.getNamenodes()) {
+ String nsId = nn.getNameserviceId();
+ String nnId = nn.getNamenodeId();
+ NamenodeHeartbeatService service = new NamenodeHeartbeatService(
+ namenodeResolver, nsId, nnId);
+ service.init(conf);
+ service.start();
+ services.add(service);
+ }
+ }
+
+ @AfterClass
+ public static void tearDown() throws IOException {
+ cluster.shutdown();
+ for (NamenodeHeartbeatService service: services) {
+ service.stop();
+ service.close();
+ }
+ }
+
+ @Test
+ public void testNamenodeHeartbeatService() throws IOException {
+
+ RouterDFSCluster testCluster = new RouterDFSCluster(true, 1);
+ Configuration heartbeatConfig = testCluster.generateNamenodeConfiguration(
+ NAMESERVICES[0]);
+ NamenodeHeartbeatService server = new NamenodeHeartbeatService(
+ namenodeResolver, NAMESERVICES[0], NAMENODES[0]);
+ server.init(heartbeatConfig);
+ assertEquals(STATE.INITED, server.getServiceState());
+ server.start();
+ assertEquals(STATE.STARTED, server.getServiceState());
+ server.stop();
+ assertEquals(STATE.STOPPED, server.getServiceState());
+ server.close();
+ }
+
+ @Test
+ public void testHearbeat() throws InterruptedException, IOException {
+
+ // Set NAMENODE1 to active for all nameservices
+ if (cluster.isHighAvailability()) {
+ for (String ns : cluster.getNameservices()) {
+ cluster.switchToActive(ns, NAMENODES[0]);
+ cluster.switchToStandby(ns, NAMENODES[1]);
+ }
+ }
+
+ // Wait for heartbeats to record
+ Thread.sleep(5000);
+
+ // Verify the locator has matching NN entries for each NS
+ for (String ns : cluster.getNameservices()) {
+ List<? extends FederationNamenodeContext> nns =
+ namenodeResolver.getNamenodesForNameserviceId(ns);
+
+ // Active
+ FederationNamenodeContext active = nns.get(0);
+ assertEquals(NAMENODES[0], active.getNamenodeId());
+
+ // Standby
+ FederationNamenodeContext standby = nns.get(1);
+ assertEquals(NAMENODES[1], standby.getNamenodeId());
+ }
+
+ // Switch active NNs in 1/2 nameservices
+ List<String> nss = cluster.getNameservices();
+ String failoverNS = nss.get(0);
+ String normalNs = nss.get(1);
+
+ cluster.switchToStandby(failoverNS, NAMENODES[0]);
+ cluster.switchToActive(failoverNS, NAMENODES[1]);
+
+ // Wait for heartbeats to record
+ Thread.sleep(5000);
+
+ // Verify the locator has recorded the failover for the failover NS
+ List<? extends FederationNamenodeContext> failoverNSs =
+ namenodeResolver.getNamenodesForNameserviceId(failoverNS);
+ // Active
+ FederationNamenodeContext active = failoverNSs.get(0);
+ assertEquals(NAMENODES[1], active.getNamenodeId());
+
+ // Standby
+ FederationNamenodeContext standby = failoverNSs.get(1);
+ assertEquals(NAMENODES[0], standby.getNamenodeId());
+
+ // Verify the locator has the same records for the other ns
+ List<? extends FederationNamenodeContext> normalNss =
+ namenodeResolver.getNamenodesForNameserviceId(normalNs);
+ // Active
+ active = normalNss.get(0);
+ assertEquals(NAMENODES[0], active.getNamenodeId());
+ // Standby
+ standby = normalNss.get(1);
+ assertEquals(NAMENODES[1], standby.getNamenodeId());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae5e1352/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
index d8afb39..2074d3d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
@@ -98,6 +98,9 @@ public class TestRouter {
// Rpc only
testRouterStartup(new RouterConfigBuilder(conf).rpc().build());
+ // Heartbeat only
+ testRouterStartup(new RouterConfigBuilder(conf).heartbeat().build());
+
// Run with all services
testRouterStartup(new RouterConfigBuilder(conf).all().build());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/40] hadoop git commit: HDFS-12312. Rebasing HDFS-10467 (2).
Contributed by Inigo Goiri.
Posted by in...@apache.org.
HDFS-12312. Rebasing HDFS-10467 (2). Contributed by Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d4aa3098
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d4aa3098
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d4aa3098
Branch: refs/heads/HDFS-10467
Commit: d4aa30986ef3ebfb63199dffe848397811878fce
Parents: bca69cc
Author: Inigo Goiri <in...@apache.org>
Authored: Wed Aug 16 17:31:37 2017 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu Sep 7 13:19:52 2017 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs | 1 -
.../hadoop/hdfs/server/federation/router/RouterRpcServer.java | 1 +
2 files changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4aa3098/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index d51a8e2..d122ff7 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -31,7 +31,6 @@ function hadoop_usage
hadoop_add_option "--hosts filename" "list of hosts to use in worker mode"
hadoop_add_option "--workers" "turn on worker mode"
-<<<<<<< HEAD
hadoop_add_subcommand "balancer" daemon "run a cluster balancing utility"
hadoop_add_subcommand "cacheadmin" admin "configure the HDFS cache"
hadoop_add_subcommand "classpath" client "prints the class path needed to get the hadoop jar and the required libraries"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4aa3098/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
index eaaab39..c77d255 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
@@ -1946,6 +1946,7 @@ public class RouterRpcServer extends AbstractService implements ClientProtocol {
}
long inodeId = 0;
return new HdfsFileStatus(0, true, 0, 0, modTime, accessTime, permission,
+ EnumSet.noneOf(HdfsFileStatus.Flags.class),
owner, group, new byte[0], DFSUtil.string2Bytes(name), inodeId,
childrenNum, null, (byte) 0, null);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/40] hadoop git commit: HDFS-11554. [Documentation] Router-based
federation documentation. Contributed by Inigo Goiri.
Posted by in...@apache.org.
HDFS-11554. [Documentation] Router-based federation documentation. Contributed by Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bca69cc1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bca69cc1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bca69cc1
Branch: refs/heads/HDFS-10467
Commit: bca69cc1728c1dc758cd704616dca43645e01808
Parents: 2ba43ee
Author: Inigo Goiri <in...@apache.org>
Authored: Wed Aug 16 17:23:29 2017 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu Sep 7 13:19:52 2017 -0700
----------------------------------------------------------------------
.../src/site/markdown/HDFSRouterFederation.md | 170 +++++++++++++++++++
.../site/resources/images/routerfederation.png | Bin 0 -> 24961 bytes
2 files changed, 170 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bca69cc1/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md
new file mode 100644
index 0000000..f094238
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md
@@ -0,0 +1,170 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+HDFS Router-based Federation
+============================
+
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
+
+Introduction
+------------
+
+NameNodes have scalability limits because of the metadata overhead comprised of inodes (files and directories) and file blocks, the number of Datanode heartbeats, and the number of HDFS RPC client requests.
+The common solution is to split the filesystem into smaller subclusters [HDFS Federation](.Federation.html) and provide a federated view [ViewFs](.ViewFs.html).
+The problem is how to maintain the split of the subclusters (e.g., namespace partition), which forces users to connect to multiple subclusters and manage the allocation of folders/files to them.
+
+
+Architecture
+------------
+
+A natural extension to this partitioned federation is to add a layer of software responsible for federating the namespaces.
+This extra layer allows users to access any subcluster transparently, lets subclusters manage their own block pools independently, and supports rebalancing of data across subclusters.
+To accomplish these goals, the federation layer directs block accesses to the proper subcluster, maintains the state of the namespaces, and provides mechanisms for data rebalancing.
+This layer must be scalable, highly available, and fault tolerant.
+
+This federation layer comprises multiple components.
+The _Router_ component that has the same interface as a NameNode, and forwards the client requests to the correct subcluster, based on ground-truth information from a State Store.
+The _State Store_ combines a remote _Mount Table_ (in the flavor of [ViewFs](.ViewFs.html), but shared between clients) and utilization (load/capacity) information about the subclusters.
+This approach has the same architecture as [YARN federation](../hadoop-yarn/Federation.html).
+
+
+
+
+### Example flow
+The simplest configuration deploys a Router on each NameNode machine.
+The Router monitors the local NameNode and heartbeats the state to the State Store.
+When a regular DFS client contacts any of the Routers to access a file in the federated filesystem, the Router checks the Mount Table in the State Store (i.e., the local cache) to find out which subcluster contains the file.
+Then it checks the Membership table in the State Store (i.e., the local cache) for the NameNode responsible for the subcluster.
+After it has identified the correct NameNode, the Router proxies the request.
+The client accesses Datanodes directly.
+
+
+### Router
+There can be multiple Routers in the system with soft state.
+Each Router has two roles:
+
+* Federated interface: expose a single, global NameNode interface to the clients and forward the requests to the active NameNode in the correct subcluster
+* NameNode heartbeat: maintain the information about a NameNode in the State Store
+
+#### Federated interface
+The Router receives a client request, checks the State Store for the correct subcluster, and forwards the request to the active NameNode of that subcluster.
+The reply from the NameNode then flows in the opposite direction.
+The Routers are stateless and can be behind a load balancer.
+For performance, the Router also caches remote mount table entries and the state of the subclusters.
+To make sure that changes have been propagated to all Routers, each Router heartbeats its state to the State Store.
+
+The communications between the Routers and the State Store are cached (with timed expiration for freshness).
+This improves the performance of the system.
+
+#### NameNode heartbeat
+For this role, the Router periodically checks the state of a NameNode (usually on the same server) and reports their high availability (HA) state and load/space status to the State Store.
+Note that this is an optional role, as a Router can be independent of any subcluster.
+For performance with NameNode HA, the Router uses the high availability state information in the State Store to forward the request to the NameNode that is most likely to be active.
+Note that this service can be embedded into the NameNode itself to simplify the operation.
+
+#### Availability and fault tolerance
+The Router operates with failures at multiple levels.
+
+* **Federated interface HA:**
+The Routers are stateless and metadata operations are atomic at the NameNodes.
+If a Router becomes unavailable, any Router can take over for it.
+The clients configure their DFS HA client (e.g., ConfiguredFailoverProvider or RequestHedgingProxyProvider) with all the Routers in the federation as endpoints.
+
+* **NameNode heartbeat HA:**
+For high availability and flexibility, multiple Routers can monitor the same NameNode and heartbeat the information to the State Store.
+This increases clients' resiliency to stale information, should a Router fail.
+Conflicting NameNode information in the State Store is resolved by each Router via a quorum.
+
+* **Unavailable NameNodes:**
+If a Router cannot contact the active NameNode, then it will try the other NameNodes in the subcluster.
+It will first try those reported as standby and then the unavailable ones.
+If the Router cannot reach any NameNode, then it throws an exception.
+
+* **Expired NameNodes:**
+If a NameNode heartbeat has not been recorded in the State Store for a multiple of the heartbeat interval, the monitoring Router will record that the NameNode has expired and no Routers will attempt to access it.
+If an updated heartbeat is subsequently recorded for the NameNode, the monitoring Router will restore the NameNode from the expired state.
+
+#### Interfaces
+To interact with the users and the administrators, the Router exposes multiple interfaces.
+
+* **RPC:**
+The Router RPC implements the most common interfaces clients use to interact with HDFS.
+The current implementation has been tested using analytics workloads written in plain MapReduce, Spark, and Hive (on Tez, Spark, and MapReduce).
+Advanced functions like snapshotting, encryption and tiered storage are left for future versions.
+All unimplemented functions will throw exceptions.
+
+* **Admin:**
+Adminstrators can query information from clusters and add/remove entries from the mount table over RPC.
+This interface is also exposed through the command line to get and modify information from the federation.
+
+* **Web UI:**
+The Router exposes a Web UI visualizing the state of the federation, mimicking the current NameNode UI.
+It displays information about the mount table, membership information about each subcluster, and the status of the Routers.
+
+* **WebHDFS:**
+The Router provides the HDFS REST interface (WebHDFS) in addition to the RPC one.
+
+* **JMX:**
+It exposes metrics through JMX mimicking the NameNode.
+This is used by the Web UI to get the cluster status.
+
+Some operations are not available in Router-based federation.
+The Router throws exceptions for those.
+Examples users may encounter include the following.
+
+* Rename file/folder in two different nameservices.
+* Copy file/folder in two different nameservices.
+* Write into a file/folder being rebalanced.
+
+
+### State Store
+The (logically centralized, but physically distributed) State Store maintains:
+
+* The state of the subclusters in terms of their block access load, available disk space, HA state, etc.
+* The mapping between folder/files and subclusters, i.e. the remote mount table.
+
+The backend of the State Store is pluggable.
+We leverage the fault tolerance of the backend implementations.
+The main information stored in the State Store and its implementation:
+
+* **Membership**:
+The membership information encodes the state of the NameNodes in the federation.
+This includes information about the subcluster, such as storage capacity and the number of nodes.
+The Router periodically heartbeats this information about one or more NameNodes.
+Given that multiple Routers can monitor a single NameNode, the heartbeat from every Router is stored.
+The Routers apply a quorum of the data when querying this information from the State Store.
+The Routers discard the entries older than a certain threshold (e.g., ten Router heartbeat periods).
+
+* **Mount Table**:
+This table hosts the mapping between folders and subclusters.
+It is similar to the mount table in [ViewFs](.ViewFs.html) where it specifies the federated folder, the destination subcluster and the path in that folder.
+
+
+Deployment
+----------
+
+By default, the Router is ready to take requests and monitor the NameNode in the local machine.
+It needs to know the State Store endpoint by setting `dfs.federation.router.store.driver.class`.
+The rest of the options are documented in [hdfs-default.xml](./hdfs-default.xml).
+
+Once the Router is configured, it can be started:
+
+ [hdfs]$ $HADOOP_HOME/bin/hdfs router
+
+To manage the mount table:
+
+ [hdfs]$ $HADOOP_HOME/bin/hdfs federation -add /tmp DC1 /tmp
+ [hdfs]$ $HADOOP_HOME/bin/hdfs federation -add /data/wl1 DC2 /data/wl1
+ [hdfs]$ $HADOOP_HOME/bin/hdfs federation -add /data/wl2 DC3 /data/wl2
+ [hdfs]$ $HADOOP_HOME/bin/hdfs federation -ls
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bca69cc1/hadoop-hdfs-project/hadoop-hdfs/src/site/resources/images/routerfederation.png
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/resources/images/routerfederation.png b/hadoop-hdfs-project/hadoop-hdfs/src/site/resources/images/routerfederation.png
new file mode 100644
index 0000000..2b158bb
Binary files /dev/null and b/hadoop-hdfs-project/hadoop-hdfs/src/site/resources/images/routerfederation.png differ
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/40] hadoop git commit: HDFS-10629. Federation Roter. Contributed
by Jason Kace and Inigo Goiri.
Posted by in...@apache.org.
HDFS-10629. Federation Roter. Contributed by Jason Kace and Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ffdfd2e6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ffdfd2e6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ffdfd2e6
Branch: refs/heads/HDFS-10467
Commit: ffdfd2e6b59445dc524410934e78dab656f74405
Parents: 83449ab
Author: Inigo <in...@apache.org>
Authored: Tue Mar 28 14:30:59 2017 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu Sep 7 13:19:50 2017 -0700
----------------------------------------------------------------------
.../hadoop-hdfs/src/main/bin/hdfs | 5 +
.../hadoop-hdfs/src/main/bin/hdfs.cmd | 8 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 17 +
.../resolver/ActiveNamenodeResolver.java | 117 +++
.../resolver/FederationNamenodeContext.java | 87 +++
.../FederationNamenodeServiceState.java | 46 ++
.../resolver/FederationNamespaceInfo.java | 99 +++
.../resolver/FileSubclusterResolver.java | 75 ++
.../resolver/NamenodePriorityComparator.java | 63 ++
.../resolver/NamenodeStatusReport.java | 195 +++++
.../federation/resolver/PathLocation.java | 122 +++
.../federation/resolver/RemoteLocation.java | 74 ++
.../federation/resolver/package-info.java | 41 +
.../federation/router/FederationUtil.java | 117 +++
.../router/RemoteLocationContext.java | 38 +
.../hdfs/server/federation/router/Router.java | 263 +++++++
.../federation/router/RouterRpcServer.java | 102 +++
.../server/federation/router/package-info.java | 31 +
.../federation/store/StateStoreService.java | 77 ++
.../server/federation/store/package-info.java | 62 ++
.../src/main/resources/hdfs-default.xml | 16 +
.../server/federation/FederationTestUtils.java | 233 ++++++
.../hdfs/server/federation/MockResolver.java | 290 +++++++
.../server/federation/RouterConfigBuilder.java | 40 +
.../server/federation/RouterDFSCluster.java | 767 +++++++++++++++++++
.../server/federation/router/TestRouter.java | 96 +++
26 files changed, 3080 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffdfd2e6/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index e6405b5..b1f44a4 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -57,6 +57,7 @@ function hadoop_usage
hadoop_add_subcommand "oiv" admin "apply the offline fsimage viewer to an fsimage"
hadoop_add_subcommand "oiv_legacy" admin "apply the offline fsimage viewer to a legacy fsimage"
hadoop_add_subcommand "portmap" daemon "run a portmap service"
+ hadoop_add_subcommand "router" daemon "run the DFS router"
hadoop_add_subcommand "secondarynamenode" daemon "run the DFS secondary namenode"
hadoop_add_subcommand "snapshotDiff" client "diff two snapshots of a directory or diff the current directory contents with a snapshot"
hadoop_add_subcommand "storagepolicies" admin "list/get/set block storage policies"
@@ -176,6 +177,10 @@ function hdfscmd_case
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME=org.apache.hadoop.portmap.Portmap
;;
+ router)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.federation.router.Router'
+ ;;
secondarynamenode)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode'
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffdfd2e6/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
index 2181e47..b9853d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
@@ -59,7 +59,7 @@ if "%1" == "--loglevel" (
)
)
- set hdfscommands=dfs namenode secondarynamenode journalnode zkfc datanode dfsadmin haadmin fsck balancer jmxget oiv oev fetchdt getconf groups snapshotDiff lsSnapshottableDir cacheadmin mover storagepolicies classpath crypto debug
+ set hdfscommands=dfs namenode secondarynamenode journalnode zkfc datanode dfsadmin haadmin fsck balancer jmxget oiv oev fetchdt getconf groups snapshotDiff lsSnapshottableDir cacheadmin mover storagepolicies classpath crypto router debug
for %%i in ( %hdfscommands% ) do (
if %hdfs-command% == %%i set hdfscommand=true
)
@@ -179,6 +179,11 @@ goto :eof
set CLASS=org.apache.hadoop.hdfs.tools.CryptoAdmin
goto :eof
+:router
+ set CLASS=org.apache.hadoop.hdfs.server.federation.router.Router
+ set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_ROUTER_OPTS%
+ goto :eof
+
:debug
set CLASS=org.apache.hadoop.hdfs.tools.DebugAdmin
goto :eof
@@ -219,6 +224,7 @@ goto :eof
@echo secondarynamenode run the DFS secondary namenode
@echo namenode run the DFS namenode
@echo journalnode run the DFS journalnode
+ @echo router run the DFS router
@echo zkfc run the ZK Failover Controller daemon
@echo datanode run a DFS datanode
@echo dfsadmin run a DFS admin client
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffdfd2e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index d06e378..4fd9d41 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -1109,6 +1109,23 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
"dfs.use.dfs.network.topology";
public static final boolean DFS_USE_DFS_NETWORK_TOPOLOGY_DEFAULT = true;
+ // HDFS federation
+ public static final String FEDERATION_PREFIX = "dfs.federation.";
+
+ // HDFS Router-based federation
+ public static final String FEDERATION_ROUTER_PREFIX =
+ "dfs.federation.router.";
+
+ // HDFS Router State Store connection
+ public static final String FEDERATION_FILE_RESOLVER_CLIENT_CLASS =
+ FEDERATION_ROUTER_PREFIX + "file.resolver.client.class";
+ public static final String FEDERATION_FILE_RESOLVER_CLIENT_CLASS_DEFAULT =
+ "org.apache.hadoop.hdfs.server.federation.MockResolver";
+ public static final String FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS =
+ FEDERATION_ROUTER_PREFIX + "namenode.resolver.client.class";
+ public static final String FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS_DEFAULT =
+ "org.apache.hadoop.hdfs.server.federation.MockResolver";
+
// dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry
@Deprecated
public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffdfd2e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java
new file mode 100644
index 0000000..477053d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Locates the most active NN for a given nameservice ID or blockpool ID. This
+ * interface is used by the {@link org.apache.hadoop.hdfs.server.federation.
+ * router.RouterRpcServer RouterRpcServer} to:
+ * <ul>
+ * <li>Determine the target NN for a given subcluster.
+ * <li>List of all namespaces discovered/active in the federation.
+ * <li>Update the currently active NN empirically.
+ * </ul>
+ * The interface is also used by the {@link org.apache.hadoop.hdfs.server.
+ * federation.router.NamenodeHeartbeatService NamenodeHeartbeatService} to
+ * register a discovered NN.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface ActiveNamenodeResolver {
+
+ /**
+ * Report a successful, active NN address for a nameservice or blockPool.
+ *
+ * @param ns Nameservice identifier.
+ * @param successfulAddress The address the successful responded to the
+ * command.
+ * @throws IOException If the state store cannot be accessed.
+ */
+ void updateActiveNamenode(
+ String ns, InetSocketAddress successfulAddress) throws IOException;
+
+ /**
+ * Returns a prioritized list of the most recent cached registration entries
+ * for a single nameservice ID.
+ * Returns an empty list if none are found. Returns entries in preference of:
+ * <ul>
+ * <li>The most recent ACTIVE NN
+ * <li>The most recent STANDBY NN
+ * <li>The most recent UNAVAILABLE NN
+ * </ul>
+ *
+ * @param nameserviceId Nameservice identifier.
+ * @return Prioritized list of namenode contexts.
+ * @throws IOException If the state store cannot be accessed.
+ */
+ List<? extends FederationNamenodeContext>
+ getNamenodesForNameserviceId(String nameserviceId) throws IOException;
+
+ /**
+ * Returns a prioritized list of the most recent cached registration entries
+ * for a single block pool ID.
+ * Returns an empty list if none are found. Returns entries in preference of:
+ * <ul>
+ * <li>The most recent ACTIVE NN
+ * <li>The most recent STANDBY NN
+ * <li>The most recent UNAVAILABLE NN
+ * </ul>
+ *
+ * @param blockPoolId Block pool identifier for the nameservice.
+ * @return Prioritized list of namenode contexts.
+ * @throws IOException If the state store cannot be accessed.
+ */
+ List<? extends FederationNamenodeContext>
+ getNamenodesForBlockPoolId(String blockPoolId) throws IOException;
+
+ /**
+ * Register a namenode in the State Store.
+ *
+ * @param report Namenode status report.
+ * @return True if the node was registered and successfully committed to the
+ * data store.
+ * @throws IOException Throws exception if the namenode could not be
+ * registered.
+ */
+ boolean registerNamenode(NamenodeStatusReport report) throws IOException;
+
+ /**
+ * Get a list of all namespaces that are registered and active in the
+ * federation.
+ *
+ * @return List of name spaces in the federation
+ * @throws Throws exception if the namespace list is not available.
+ */
+ Set<FederationNamespaceInfo> getNamespaces() throws IOException;
+
+ /**
+ * Assign a unique identifier for the parent router service.
+ * Required to report the status to the namenode resolver.
+ *
+ * @param router Unique string identifier for the router.
+ */
+ void setRouterId(String routerId);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffdfd2e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeContext.java
new file mode 100644
index 0000000..68ef02a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeContext.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+/**
+ * Interface for a discovered NN and its current server endpoints.
+ */
+public interface FederationNamenodeContext {
+
+ /**
+ * Get the RPC server address of the namenode.
+ *
+ * @return RPC server address in the form of host:port.
+ */
+ String getRpcAddress();
+
+ /**
+ * Get the Service RPC server address of the namenode.
+ *
+ * @return Service RPC server address in the form of host:port.
+ */
+ String getServiceAddress();
+
+ /**
+ * Get the Lifeline RPC server address of the namenode.
+ *
+ * @return Lifeline RPC server address in the form of host:port.
+ */
+ String getLifelineAddress();
+
+ /**
+ * Get the HTTP server address of the namenode.
+ *
+ * @return HTTP address in the form of host:port.
+ */
+ String getWebAddress();
+
+ /**
+ * Get the unique key representing the namenode.
+ *
+ * @return Combination of the nameservice and the namenode IDs.
+ */
+ String getNamenodeKey();
+
+ /**
+ * Identifier for the nameservice/namespace.
+ *
+ * @return Namenode nameservice identifier.
+ */
+ String getNameserviceId();
+
+ /**
+ * Identifier for the namenode.
+ *
+ * @return String
+ */
+ String getNamenodeId();
+
+ /**
+ * The current state of the namenode (active, standby, etc).
+ *
+ * @return FederationNamenodeServiceState State of the namenode.
+ */
+ FederationNamenodeServiceState getState();
+
+ /**
+ * The update date.
+ *
+ * @return Long with the update date.
+ */
+ long getDateModified();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffdfd2e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java
new file mode 100644
index 0000000..c773f82
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+
+/**
+ * Namenode state in the federation. The order of this enum is used to evaluate
+ * NN priority for RPC calls.
+ */
+public enum FederationNamenodeServiceState {
+ ACTIVE, // HAServiceState.ACTIVE or operational.
+ STANDBY, // HAServiceState.STANDBY.
+ UNAVAILABLE, // When the namenode cannot be reached.
+ EXPIRED; // When the last update is too old.
+
+ public static FederationNamenodeServiceState getState(HAServiceState state) {
+ switch(state) {
+ case ACTIVE:
+ return FederationNamenodeServiceState.ACTIVE;
+ case STANDBY:
+ return FederationNamenodeServiceState.STANDBY;
+ case INITIALIZING:
+ return FederationNamenodeServiceState.UNAVAILABLE;
+ case STOPPING:
+ return FederationNamenodeServiceState.UNAVAILABLE;
+ default:
+ return FederationNamenodeServiceState.UNAVAILABLE;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffdfd2e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java
new file mode 100644
index 0000000..bbaeca3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import org.apache.hadoop.hdfs.server.federation.router.RemoteLocationContext;
+
+/**
+ * Represents information about a single nameservice/namespace in a federated
+ * HDFS cluster.
+ */
+public class FederationNamespaceInfo
+ implements Comparable<FederationNamespaceInfo>, RemoteLocationContext {
+
+ /** Block pool identifier. */
+ private String blockPoolId;
+ /** Cluster identifier. */
+ private String clusterId;
+ /** Nameservice identifier. */
+ private String nameserviceId;
+
+ public FederationNamespaceInfo(String bpId, String clId, String nsId) {
+ this.blockPoolId = bpId;
+ this.clusterId = clId;
+ this.nameserviceId = nsId;
+ }
+
+ /**
+ * The HDFS nameservice id for this namespace.
+ *
+ * @return Nameservice identifier.
+ */
+ public String getNameserviceId() {
+ return this.nameserviceId;
+ }
+
+ /**
+ * The HDFS cluster id for this namespace.
+ *
+ * @return Cluster identifier.
+ */
+ public String getClusterId() {
+ return this.clusterId;
+ }
+
+ /**
+ * The HDFS block pool id for this namespace.
+ *
+ * @return Block pool identifier.
+ */
+ public String getBlockPoolId() {
+ return this.blockPoolId;
+ }
+
+ @Override
+ public int hashCode() {
+ return this.nameserviceId.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ } else if (obj instanceof FederationNamespaceInfo) {
+ return this.compareTo((FederationNamespaceInfo) obj) == 0;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public int compareTo(FederationNamespaceInfo info) {
+ return this.nameserviceId.compareTo(info.getNameserviceId());
+ }
+
+ @Override
+ public String toString() {
+ return this.nameserviceId + "->" + this.blockPoolId + ":" + this.clusterId;
+ }
+
+ @Override
+ public String getDest() {
+ return this.nameserviceId;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffdfd2e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FileSubclusterResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FileSubclusterResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FileSubclusterResolver.java
new file mode 100644
index 0000000..af9f493
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FileSubclusterResolver.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Interface to map a file path in the global name space to a specific
+ * subcluster and path in an HDFS name space.
+ * <p>
+ * Each path in the global/federated namespace may map to 1-N different HDFS
+ * locations. Each location specifies a single nameservice and a single HDFS
+ * path. The behavior is similar to MergeFS and Nfly and allows the merger
+ * of multiple HDFS locations into a single path. See HADOOP-8298 and
+ * HADOOP-12077
+ * <p>
+ * For example, a directory listing will fetch listings for each destination
+ * path and combine them into a single set of results.
+ * <p>
+ * When multiple destinations are available for a path, the destinations are
+ * prioritized in a consistent manner. This allows the proxy server to
+ * guess the best/most likely destination and attempt it first.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface FileSubclusterResolver {
+
+ /**
+ * Get the destinations for a global path. Results are from the mount table
+ * cache. If multiple destinations are available, the first result is the
+ * highest priority destination.
+ *
+ * @param path Global path.
+ * @return Location in a destination namespace or null if it does not exist.
+ * @throws IOException Throws exception if the data is not available.
+ */
+ PathLocation getDestinationForPath(String path) throws IOException;
+
+ /**
+ * Get a list of mount points for a path. Results are from the mount table
+ * cache.
+ *
+ * @return List of mount points present at this path or zero-length list if
+ * none are found.
+ * @throws IOException Throws exception if the data is not available.
+ */
+ List<String> getMountPoints(String path) throws IOException;
+
+ /**
+ * Get the default namespace for the cluster.
+ *
+ * @return Default namespace identifier.
+ */
+ String getDefaultNamespace();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffdfd2e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodePriorityComparator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodePriorityComparator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodePriorityComparator.java
new file mode 100644
index 0000000..fe82f29
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodePriorityComparator.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import java.util.Comparator;
+
+/**
+ * Compares NNs in the same namespace and prioritizes by their status. The
+ * priorities are:
+ * <ul>
+ * <li>ACTIVE
+ * <li>STANDBY
+ * <li>UNAVAILABLE
+ * </ul>
+ * When two NNs have the same state, the last modification date is the tie
+ * breaker, newest has priority. Expired NNs are excluded.
+ */
+public class NamenodePriorityComparator
+ implements Comparator<FederationNamenodeContext> {
+
+ @Override
+ public int compare(FederationNamenodeContext o1,
+ FederationNamenodeContext o2) {
+ FederationNamenodeServiceState state1 = o1.getState();
+ FederationNamenodeServiceState state2 = o2.getState();
+
+ if (state1 == state2) {
+ // Both have the same state, use mode dates
+ return compareModDates(o1, o2);
+ } else {
+ // Enum is ordered by priority
+ return state1.compareTo(state2);
+ }
+ }
+
+ /**
+ * Compare the modification dates.
+ *
+ * @param o1 Context 1.
+ * @param o2 Context 2.
+ * @return Comparison between dates.
+ */
+ private int compareModDates(FederationNamenodeContext o1,
+ FederationNamenodeContext o2) {
+ // Reverse sort, lowest position is highest priority.
+ return (int) (o2.getDateModified() - o1.getDateModified());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffdfd2e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java
new file mode 100644
index 0000000..9259048
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java
@@ -0,0 +1,195 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
+
+/**
+ * Status of the namenode.
+ */
+public class NamenodeStatusReport {
+
+ /** Namenode information. */
+ private String nameserviceId = "";
+ private String namenodeId = "";
+ private String clusterId = "";
+ private String blockPoolId = "";
+ private String rpcAddress = "";
+ private String serviceAddress = "";
+ private String lifelineAddress = "";
+ private String webAddress = "";
+
+ /** Namenode state. */
+ private HAServiceState status = HAServiceState.STANDBY;
+ private boolean safeMode = false;
+
+ /** If the fields are valid. */
+ private boolean registrationValid = false;
+ private boolean haStateValid = false;
+
+ public NamenodeStatusReport(String ns, String nn, String rpc, String service,
+ String lifeline, String web) {
+ this.nameserviceId = ns;
+ this.namenodeId = nn;
+ this.rpcAddress = rpc;
+ this.serviceAddress = service;
+ this.lifelineAddress = lifeline;
+ this.webAddress = web;
+ }
+
+ /**
+ * If the registration is valid.
+ *
+ * @return If the registration is valid.
+ */
+ public boolean registrationValid() {
+ return this.registrationValid;
+ }
+
+ /**
+ * If the HA state is valid.
+ *
+ * @return If the HA state is valid.
+ */
+ public boolean haStateValid() {
+ return this.haStateValid;
+ }
+
+ /**
+ * Get the state of the Namenode being monitored.
+ *
+ * @return State of the Namenode.
+ */
+ public FederationNamenodeServiceState getState() {
+ if (!registrationValid) {
+ return FederationNamenodeServiceState.UNAVAILABLE;
+ } else if (haStateValid) {
+ return FederationNamenodeServiceState.getState(status);
+ } else {
+ return FederationNamenodeServiceState.ACTIVE;
+ }
+ }
+
+ /**
+ * Get the name service identifier.
+ *
+ * @return The name service identifier.
+ */
+ public String getNameserviceId() {
+ return this.nameserviceId;
+ }
+
+ /**
+ * Get the namenode identifier.
+ *
+ * @return The namenode identifier.
+ */
+ public String getNamenodeId() {
+ return this.namenodeId;
+ }
+
+ /**
+ * Get the cluster identifier.
+ *
+ * @return The cluster identifier.
+ */
+ public String getClusterId() {
+ return this.clusterId;
+ }
+
+ /**
+ * Get the block pool identifier.
+ *
+ * @return The block pool identifier.
+ */
+ public String getBlockPoolId() {
+ return this.blockPoolId;
+ }
+
+ /**
+ * Get the RPC address.
+ *
+ * @return The RPC address.
+ */
+ public String getRpcAddress() {
+ return this.rpcAddress;
+ }
+
+ /**
+ * Get the Service RPC address.
+ *
+ * @return The Service RPC address.
+ */
+ public String getServiceAddress() {
+ return this.serviceAddress;
+ }
+
+ /**
+ * Get the Lifeline RPC address.
+ *
+ * @return The Lifeline RPC address.
+ */
+ public String getLifelineAddress() {
+ return this.lifelineAddress;
+ }
+
+ /**
+ * Get the web address.
+ *
+ * @return The web address.
+ */
+ public String getWebAddress() {
+ return this.webAddress;
+ }
+
+ /**
+ * Get the HA service state.
+ *
+ * @return The HA service state.
+ */
+ public void setHAServiceState(HAServiceState state) {
+ this.status = state;
+ this.haStateValid = true;
+ }
+
+ /**
+ * Set the namespace information.
+ *
+ * @param info Namespace information.
+ */
+ public void setNamespaceInfo(NamespaceInfo info) {
+ this.clusterId = info.getClusterID();
+ this.blockPoolId = info.getBlockPoolID();
+ this.registrationValid = true;
+ }
+
+ public void setSafeMode(boolean safemode) {
+ this.safeMode = safemode;
+ }
+
+ public boolean getSafemode() {
+ return this.safeMode;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("%s-%s:%s",
+ nameserviceId, namenodeId, serviceAddress);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffdfd2e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/PathLocation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/PathLocation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/PathLocation.java
new file mode 100644
index 0000000..d90565c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/PathLocation.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * A map of the properties and target destinations (name space + path) for
+ * a path in the global/federated namespace.
+ * This data is generated from the @see MountTable records.
+ */
+public class PathLocation {
+
+ /** Source path in global namespace. */
+ private final String sourcePath;
+
+ /** Remote paths in the target namespaces. */
+ private final List<RemoteLocation> destinations;
+
+ /** List of name spaces present. */
+ private final Set<String> namespaces;
+
+
+ /**
+ * Create a new PathLocation.
+ *
+ * @param source Source path in the global name space.
+ * @param dest Destinations of the mount table entry.
+ * @param namespaces Unique identifier representing the combination of
+ * name spaces present in the destination list.
+ */
+ public PathLocation(
+ String source, List<RemoteLocation> dest, Set<String> nss) {
+ this.sourcePath = source;
+ this.destinations = dest;
+ this.namespaces = nss;
+ }
+
+ /**
+ * Create a path location from another path.
+ *
+ * @param other Other path location to copy from.
+ */
+ public PathLocation(PathLocation other) {
+ this.sourcePath = other.sourcePath;
+ this.destinations = new LinkedList<RemoteLocation>(other.destinations);
+ this.namespaces = new HashSet<String>(other.namespaces);
+ }
+
+ /**
+ * Get the source path in the global namespace for this path location.
+ *
+ * @return The path in the global namespace.
+ */
+ public String getSourcePath() {
+ return this.sourcePath;
+ }
+
+ /**
+ * Get the list of subclusters defined for the destinations.
+ */
+ public Set<String> getNamespaces() {
+ return Collections.unmodifiableSet(this.namespaces);
+ }
+
+ @Override
+ public String toString() {
+ RemoteLocation loc = getDefaultLocation();
+ return loc.getNameserviceId() + "->" + loc.getDest();
+ }
+
+ /**
+ * Check if this location supports multiple clusters/paths.
+ *
+ * @return If it has multiple destinations.
+ */
+ public boolean hasMultipleDestinations() {
+ return this.destinations.size() > 1;
+ }
+
+ /**
+ * Get the list of locations found in the mount table.
+ * The first result is the highest priority path.
+ *
+ * @return List of remote locations.
+ */
+ public List<RemoteLocation> getDestinations() {
+ return Collections.unmodifiableList(this.destinations);
+ }
+
+ /**
+ * Get the default or highest priority location.
+ *
+ * @return The default location.
+ */
+ public RemoteLocation getDefaultLocation() {
+ if (destinations.isEmpty() || destinations.get(0).getDest() == null) {
+ throw new UnsupportedOperationException(
+ "Unsupported path " + sourcePath + " please check mount table");
+ }
+ return destinations.get(0);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffdfd2e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java
new file mode 100644
index 0000000..eef136d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.hadoop.hdfs.server.federation.router.RemoteLocationContext;
+
+/**
+ * A single in a remote namespace consisting of a nameservice ID
+ * and a HDFS path.
+ */
+public class RemoteLocation implements RemoteLocationContext {
+
+ /** Identifier of the remote namespace for this location. */
+ private String nameserviceId;
+ /** Path in the remote location. */
+ private String path;
+
+ /**
+ * Create a new remote location.
+ *
+ * @param nsId Destination namespace.
+ * @param pPath Path in the destination namespace.
+ */
+ public RemoteLocation(String nsId, String pPath) {
+ this.nameserviceId = nsId;
+ this.path = pPath;
+ }
+
+ @Override
+ public String getNameserviceId() {
+ return this.nameserviceId;
+ }
+
+ @Override
+ public String getDest() {
+ return this.path;
+ }
+
+ @Override
+ public String toString() {
+ return this.nameserviceId + "->" + this.path;
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder(17, 31)
+ .append(this.nameserviceId)
+ .append(this.path)
+ .toHashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return (obj != null &&
+ obj.getClass() == this.getClass() &&
+ obj.hashCode() == this.hashCode());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffdfd2e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/package-info.java
new file mode 100644
index 0000000..d8be9e3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/package-info.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The resolver package contains indepedent data resolvers used in HDFS
+ * federation. The data resolvers collect data from the cluster, including from
+ * the state store. The resolvers expose APIs used by HDFS federation to collect
+ * aggregated, cached data for use in Real-time request processing. The
+ * resolvers are perf-sensitive and are used in the flow of the
+ * {@link RouterRpcServer} request path.
+ * <p>
+ * The principal resolvers are:
+ * <ul>
+ * <li>{@link ActiveNamenodeResolver} Real-time interface for locating the most
+ * recently active NN for a nameservice.
+ * <li>{@link FileSubclusterResolver} Real-time interface for determining the NN
+ * and local file path for a given file/folder based on the global namespace
+ * path.
+ * </ul>
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffdfd2e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
new file mode 100644
index 0000000..6e7e865
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.lang.reflect.Constructor;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+
+/**
+ * Utilities for managing HDFS federation.
+ */
+public final class FederationUtil {
+
+ private static final Log LOG = LogFactory.getLog(FederationUtil.class);
+
+ private FederationUtil() {
+ // Utility Class
+ }
+
+ /**
+ * Create an instance of an interface with a constructor using a state store
+ * constructor.
+ *
+ * @param conf Configuration
+ * @param context Context object to pass to the instance.
+ * @param contextType Type of the context passed to the constructor.
+ * @param configurationKeyName Configuration key to retrieve the class to load
+ * @param defaultClassName Default class to load if the configuration key is
+ * not set
+ * @param clazz Class/interface that must be implemented by the instance.
+ * @return New instance of the specified class that implements the desired
+ * interface and a single parameter constructor containing a
+ * StateStore reference.
+ */
+ private static <T, R> T newInstance(final Configuration conf,
+ final R context, final Class<R> contextClass,
+ final String configKeyName, final String defaultClassName,
+ final Class<T> clazz) {
+
+ String className = conf.get(configKeyName, defaultClassName);
+ try {
+ Class<?> instance = conf.getClassByName(className);
+ if (clazz.isAssignableFrom(instance)) {
+ if (contextClass == null) {
+ // Default constructor if no context
+ @SuppressWarnings("unchecked")
+ Constructor<T> constructor =
+ (Constructor<T>) instance.getConstructor();
+ return constructor.newInstance();
+ } else {
+ // Constructor with context
+ @SuppressWarnings("unchecked")
+ Constructor<T> constructor = (Constructor<T>) instance.getConstructor(
+ Configuration.class, contextClass);
+ return constructor.newInstance(conf, context);
+ }
+ } else {
+ throw new RuntimeException("Class " + className + " not instance of "
+ + clazz.getCanonicalName());
+ }
+ } catch (ReflectiveOperationException e) {
+ LOG.error("Could not instantiate: " + className, e);
+ return null;
+ }
+ }
+
+ /**
+ * Creates an instance of a FileSubclusterResolver from the configuration.
+ *
+ * @param conf Configuration that defines the file resolver class.
+ * @param obj Context object passed to class constructor.
+ * @return FileSubclusterResolver
+ */
+ public static FileSubclusterResolver newFileSubclusterResolver(
+ Configuration conf, StateStoreService stateStore) {
+ return newInstance(conf, stateStore, StateStoreService.class,
+ DFSConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
+ DFSConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS_DEFAULT,
+ FileSubclusterResolver.class);
+ }
+
+ /**
+ * Creates an instance of an ActiveNamenodeResolver from the configuration.
+ *
+ * @param conf Configuration that defines the namenode resolver class.
+ * @param obj Context object passed to class constructor.
+ * @return ActiveNamenodeResolver
+ */
+ public static ActiveNamenodeResolver newActiveNamenodeResolver(
+ Configuration conf, StateStoreService stateStore) {
+ return newInstance(conf, stateStore, StateStoreService.class,
+ DFSConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS,
+ DFSConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS_DEFAULT,
+ ActiveNamenodeResolver.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffdfd2e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
new file mode 100644
index 0000000..da6066b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+/**
+ * Interface for objects that are unique to a namespace.
+ */
+public interface RemoteLocationContext {
+
+ /**
+ * Returns an identifier for a unique namespace.
+ *
+ * @return Namespace identifier.
+ */
+ String getNameserviceId();
+
+ /**
+ * Destination in this location. For example the path in a remote namespace.
+ *
+ * @return Destination in this location.
+ */
+ String getDest();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffdfd2e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
new file mode 100644
index 0000000..fe0d02a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
@@ -0,0 +1,263 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.apache.hadoop.hdfs.server.federation.router.FederationUtil.newActiveNamenodeResolver;
+import static org.apache.hadoop.hdfs.server.federation.router.FederationUtil.newFileSubclusterResolver;
+import static org.apache.hadoop.util.ExitUtil.terminate;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.util.ShutdownHookManager;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * Router that provides a unified view of multiple federated HDFS clusters. It
+ * has two main roles: (1) federated interface and (2) NameNode heartbeat.
+ * <p>
+ * For the federated interface, the Router receives a client request, checks the
+ * State Store for the correct subcluster, and forwards the request to the
+ * active Namenode of that subcluster. The reply from the Namenode then flows in
+ * the opposite direction. The Routers are stateless and can be behind a load
+ * balancer. HDFS clients connect to the router using the same interfaces as are
+ * used to communicate with a namenode, namely the ClientProtocol RPC interface
+ * and the WebHdfs HTTP interface exposed by the router. {@link RouterRpcServer}
+ * {@link RouterHttpServer}
+ * <p>
+ * For NameNode heartbeat, the Router periodically checks the state of a
+ * NameNode (usually on the same server) and reports their high availability
+ * (HA) state and load/space status to the State Store. Note that this is an
+ * optional role as a Router can be independent of any subcluster.
+ * {@link StateStoreService} {@link NamenodeHeartbeatService}
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class Router extends CompositeService {
+
+ private static final Log LOG = LogFactory.getLog(Router.class);
+
+
+ /** Configuration for the Router. */
+ private Configuration conf;
+
+ /** Router address/identifier. */
+ private String routerId;
+
+ /** RPC interface to the client. */
+ private RouterRpcServer rpcServer;
+
+ /** Interface with the State Store. */
+ private StateStoreService stateStore;
+
+ /** Interface to map global name space to HDFS subcluster name spaces. */
+ private FileSubclusterResolver subclusterResolver;
+
+ /** Interface to identify the active NN for a nameservice or blockpool ID. */
+ private ActiveNamenodeResolver namenodeResolver;
+
+
+ /** Usage string for help message. */
+ private static final String USAGE = "Usage: java Router";
+
+ /** Priority of the Router shutdown hook. */
+ public static final int SHUTDOWN_HOOK_PRIORITY = 30;
+
+
+ /////////////////////////////////////////////////////////
+ // Constructor
+ /////////////////////////////////////////////////////////
+
+ public Router() {
+ super(Router.class.getName());
+ }
+
+ /////////////////////////////////////////////////////////
+ // Service management
+ /////////////////////////////////////////////////////////
+
+ @Override
+ protected void serviceInit(Configuration configuration) throws Exception {
+ this.conf = configuration;
+
+ // TODO Interface to the State Store
+ this.stateStore = null;
+
+ // Resolver to track active NNs
+ this.namenodeResolver = newActiveNamenodeResolver(
+ this.conf, this.stateStore);
+ if (this.namenodeResolver == null) {
+ throw new IOException("Cannot find namenode resolver.");
+ }
+
+ // Lookup interface to map between the global and subcluster name spaces
+ this.subclusterResolver = newFileSubclusterResolver(
+ this.conf, this.stateStore);
+ if (this.subclusterResolver == null) {
+ throw new IOException("Cannot find subcluster resolver");
+ }
+
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+
+ super.serviceStop();
+ }
+
+ /**
+ * Shutdown the router.
+ */
+ public void shutDown() {
+ new Thread() {
+ @Override
+ public void run() {
+ Router.this.stop();
+ }
+ }.start();
+ }
+
+ /**
+ * Main run loop for the router.
+ *
+ * @param argv parameters.
+ */
+ public static void main(String[] argv) {
+ if (DFSUtil.parseHelpArgument(argv, Router.USAGE, System.out, true)) {
+ System.exit(0);
+ }
+
+ try {
+ StringUtils.startupShutdownMessage(Router.class, argv, LOG);
+
+ Router router = new Router();
+
+ ShutdownHookManager.get().addShutdownHook(
+ new CompositeServiceShutdownHook(router), SHUTDOWN_HOOK_PRIORITY);
+
+ Configuration conf = new HdfsConfiguration();
+ router.init(conf);
+ router.start();
+ } catch (Throwable e) {
+ LOG.error("Failed to start router.", e);
+ terminate(1, e);
+ }
+ }
+
+ /////////////////////////////////////////////////////////
+ // RPC Server
+ /////////////////////////////////////////////////////////
+
+ /**
+ * Create a new Router RPC server to proxy ClientProtocol requests.
+ *
+ * @return RouterRpcServer
+ * @throws IOException If the router RPC server was not started.
+ */
+ protected RouterRpcServer createRpcServer() throws IOException {
+ return new RouterRpcServer(this.conf, this, this.getNamenodeResolver(),
+ this.getSubclusterResolver());
+ }
+
+ /**
+ * Get the Router RPC server.
+ *
+ * @return Router RPC server.
+ */
+ public RouterRpcServer getRpcServer() {
+ return this.rpcServer;
+ }
+
+ /////////////////////////////////////////////////////////
+ // Submodule getters
+ /////////////////////////////////////////////////////////
+
+ /**
+ * Get the State Store service.
+ *
+ * @return State Store service.
+ */
+ public StateStoreService getStateStore() {
+ return this.stateStore;
+ }
+
+ /**
+ * Get the subcluster resolver for files.
+ *
+ * @return Subcluster resolver for files.
+ */
+ public FileSubclusterResolver getSubclusterResolver() {
+ return this.subclusterResolver;
+ }
+
+ /**
+ * Get the namenode resolver for a subcluster.
+ *
+ * @return The namenode resolver for a subcluster.
+ */
+ public ActiveNamenodeResolver getNamenodeResolver() {
+ return this.namenodeResolver;
+ }
+
+ /////////////////////////////////////////////////////////
+ // Router info
+ /////////////////////////////////////////////////////////
+
+ /**
+ * Unique ID for the router, typically the hostname:port string for the
+ * router's RPC server. This ID may be null on router startup before the RPC
+ * server has bound to a port.
+ *
+ * @return Router identifier.
+ */
+ public String getRouterId() {
+ return this.routerId;
+ }
+
+ /**
+ * Sets a unique ID for this router.
+ *
+ * @param router Identifier of the Router.
+ */
+ public void setRouterId(String id) {
+ this.routerId = id;
+ if (this.stateStore != null) {
+ this.stateStore.setIdentifier(this.routerId);
+ }
+ if (this.namenodeResolver != null) {
+ this.namenodeResolver.setRouterId(this.routerId);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffdfd2e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
new file mode 100644
index 0000000..24792bb
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.ipc.RPC.Server;
+import org.apache.hadoop.service.AbstractService;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * This class is responsible for handling all of the RPC calls to the It is
+ * created, started, and stopped by {@link Router}. It implements the
+ * {@link ClientProtocol} to mimic a
+ * {@link org.apache.hadoop.hdfs.server.namenode.NameNode NameNode} and proxies
+ * the requests to the active
+ * {@link org.apache.hadoop.hdfs.server.namenode.NameNode NameNode}.
+ */
+public class RouterRpcServer extends AbstractService {
+
+ /** The RPC server that listens to requests from clients. */
+ private final Server rpcServer;
+
+ /**
+ * Construct a router RPC server.
+ *
+ * @param configuration HDFS Configuration.
+ * @param nnResolver The NN resolver instance to determine active NNs in HA.
+ * @param fileResolver File resolver to resolve file paths to subclusters.
+ * @throws IOException If the RPC server could not be created.
+ */
+ public RouterRpcServer(Configuration configuration, Router router,
+ ActiveNamenodeResolver nnResolver, FileSubclusterResolver fileResolver)
+ throws IOException {
+ super(RouterRpcServer.class.getName());
+
+ this.rpcServer = null;
+ }
+
+ /**
+ * Allow access to the client RPC server for testing.
+ *
+ * @return The RPC server.
+ */
+ @VisibleForTesting
+ public Server getServer() {
+ return this.rpcServer;
+ }
+
+ @Override
+ protected void serviceInit(Configuration configuration) throws Exception {
+ super.serviceInit(configuration);
+ }
+
+ /**
+ * Start client and service RPC servers.
+ */
+ @Override
+ protected void serviceStart() throws Exception {
+ if (this.rpcServer != null) {
+ this.rpcServer.start();
+ }
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (this.rpcServer != null) {
+ this.rpcServer.stop();
+ }
+ super.serviceStop();
+ }
+
+ /**
+ * Wait until the RPC servers have shutdown.
+ */
+ void join() throws InterruptedException {
+ if (this.rpcServer != null) {
+ this.rpcServer.join();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffdfd2e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/package-info.java
new file mode 100644
index 0000000..327f39b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/package-info.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The router package includes the core services for a HDFS federation router.
+ * The {@link Router} acts as a transparent proxy in front of a cluster of
+ * multiple NameNodes and nameservices. The {@link RouterRpcServer} exposes the
+ * NameNode clientProtocol and is the primary contact point for DFS clients in a
+ * federated cluster.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffdfd2e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
new file mode 100644
index 0000000..866daa3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.service.CompositeService;
+
+/**
+ * A service to initialize a
+ * {@link org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver
+ * StateStoreDriver} and maintain the connection to the data store. There
+ * are multiple state store driver connections supported:
+ * <ul>
+ * <li>File {@link org.apache.hadoop.hdfs.server.federation.store.driver.impl.
+ * StateStoreFileImpl StateStoreFileImpl}
+ * <li>ZooKeeper {@link org.apache.hadoop.hdfs.server.federation.store.driver.
+ * impl.StateStoreZooKeeperImpl StateStoreZooKeeperImpl}
+ * </ul>
+ * <p>
+ * The service also supports the dynamic registration of data interfaces such as
+ * the following:
+ * <ul>
+ * <li>{@link MembershipStateStore}: state of the Namenodes in the
+ * federation.
+ * <li>{@link MountTableStore}: Mount table between to subclusters.
+ * See {@link org.apache.hadoop.fs.viewfs.ViewFs ViewFs}.
+ * <li>{@link RouterStateStore}: State of the routers in the federation.
+ * </ul>
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class StateStoreService extends CompositeService {
+
+ /** Identifier for the service. */
+ private String identifier;
+
+ // Stub class
+ public StateStoreService(String name) {
+ super(name);
+ }
+
+ /**
+ * Fetch a unique identifier for this state store instance. Typically it is
+ * the address of the router.
+ *
+ * @return Unique identifier for this store.
+ */
+ public String getIdentifier() {
+ return this.identifier;
+ }
+
+ /**
+ * Set a unique synchronization identifier for this store.
+ *
+ * @param id Unique identifier, typically the router's RPC address.
+ */
+ public void setIdentifier(String id) {
+ this.identifier = id;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffdfd2e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/package-info.java
new file mode 100644
index 0000000..949ec7c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/package-info.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The federation state store tracks persistent values that are shared between
+ * multiple routers.
+ * <p>
+ * Data is stored in data records that inherit from a common class. Data records
+ * are serialized when written to the data store using a modular serialization
+ * implementation. The default is profobuf serialization. Data is stored as rows
+ * of records of the same type with each data member in a record representing a
+ * column.
+ * <p>
+ * The state store uses a modular data storage
+ * {@link org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver
+ * StateStoreDriver} to handle querying, updating and deleting data records. The
+ * data storage driver is initialized and maintained by the
+ * {@link org.apache.hadoop.hdfs.server.federation.store.
+ * StateStoreService FederationStateStoreService}. The state store
+ * supports fetching all records of a type, filtering by column values or
+ * fetching a single record by its primary key.
+ * <p>
+ * The state store contains several API interfaces, one for each data records
+ * type.
+ * <p>
+ * <ul>
+ * <li>FederationMembershipStateStore: state of all Namenodes in the federation.
+ * Uses the MembershipState record.
+ * <li>FederationMountTableStore: Mount table mapping paths in the global
+ * namespace to individual subcluster paths. Uses the MountTable record.
+ * <li>RouterStateStore: State of all routers in the federation. Uses the
+ * RouterState record.
+ * </ul>
+ * <p>
+ * Each API is defined in a separate interface. The implementations of these
+ * interfaces are responsible for accessing the
+ * {@link org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver
+ * StateStoreDriver} to query, update and delete data records.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffdfd2e6/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 36c74f6..87150da 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4631,4 +4631,20 @@
</description>
</property>
+ <property>
+ <name>dfs.federation.router.file.resolver.client.class</name>
+ <value>org.apache.hadoop.hdfs.server.federation.MockResolver</value>
+ <description>
+ Class to resolve files to subclusters.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.namenode.resolver.client.class</name>
+ <value>org.apache.hadoop.hdfs.server.federation.MockResolver</value>
+ <description>
+ Class to resolve the namenode for a subcluster.
+ </description>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffdfd2e6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
new file mode 100644
index 0000000..8674682
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
@@ -0,0 +1,233 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.BufferedReader;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.Date;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
+import org.apache.hadoop.security.AccessControlException;
+
+/**
+ * Helper utilities for testing HDFS Federation.
+ */
+public final class FederationTestUtils {
+
+ public final static String NAMESERVICE1 = "ns0";
+ public final static String NAMESERVICE2 = "ns1";
+ public final static String NAMENODE1 = "nn0";
+ public final static String NAMENODE2 = "nn1";
+ public final static String NAMENODE3 = "nn2";
+ public final static String NAMENODE4 = "nn3";
+ public final static String ROUTER1 = "router0";
+ public final static String ROUTER2 = "router1";
+ public final static String ROUTER3 = "router2";
+ public final static String ROUTER4 = "router3";
+ public final static long BLOCK_SIZE_BYTES = 134217728;
+
+ private FederationTestUtils() {
+ // Utility class
+ }
+
+ public static void verifyException(Object obj, String methodName,
+ Class<? extends Exception> exceptionClass, Class<?>[] parameterTypes,
+ Object[] arguments) {
+
+ Throwable triggeredException = null;
+ try {
+ Method m = obj.getClass().getMethod(methodName, parameterTypes);
+ m.invoke(obj, arguments);
+ } catch (InvocationTargetException ex) {
+ triggeredException = ex.getTargetException();
+ } catch (Exception e) {
+ triggeredException = e;
+ }
+ if (exceptionClass != null) {
+ assertNotNull("No exception was triggered, expected exception - "
+ + exceptionClass.getName(), triggeredException);
+ assertEquals(exceptionClass, triggeredException.getClass());
+ } else {
+ assertNull("Exception was triggered but no exception was expected",
+ triggeredException);
+ }
+ }
+
+ public static NamenodeStatusReport createNamenodeReport(String ns, String nn,
+ HAServiceState state) {
+ Random rand = new Random();
+ NamenodeStatusReport report = new NamenodeStatusReport(ns, nn,
+ "localhost:" + rand.nextInt(10000), "localhost:" + rand.nextInt(10000),
+ "localhost:" + rand.nextInt(10000), "testwebaddress-" + ns + nn);
+ if (state == null) {
+ // Unavailable, no additional info
+ return report;
+ }
+ report.setHAServiceState(state);
+ report.setNamespaceInfo(new NamespaceInfo(1, "tesclusterid", ns, 0,
+ "testbuildvesion", "testsoftwareversion"));
+ return report;
+ }
+
+ public static void waitNamenodeRegistered(ActiveNamenodeResolver resolver,
+ String nameserviceId, String namenodeId,
+ FederationNamenodeServiceState finalState)
+ throws InterruptedException, IllegalStateException, IOException {
+
+ for (int loopCount = 0; loopCount < 20; loopCount++) {
+
+ if (loopCount > 0) {
+ Thread.sleep(1000);
+ }
+
+ List<? extends FederationNamenodeContext> namenodes;
+ namenodes =
+ resolver.getNamenodesForNameserviceId(nameserviceId);
+ for (FederationNamenodeContext namenode : namenodes) {
+ if (namenodeId != null
+ && !namenode.getNamenodeId().equals(namenodeId)) {
+ // Keep looking
+ continue;
+ }
+ if (finalState != null && !namenode.getState().equals(finalState)) {
+ // Wrong state, wait a bit more
+ break;
+ }
+ // Found
+ return;
+ }
+ }
+ assertTrue("Failed to verify state store registration for state - "
+ + finalState + " - " + " - " + nameserviceId + " - ", false);
+ }
+
+ public static boolean verifyDate(Date d1, Date d2, long precision) {
+ if (Math.abs(d1.getTime() - d2.getTime()) < precision) {
+ return true;
+ }
+ return false;
+ }
+
+ public static boolean addDirectory(FileSystem context, String path)
+ throws IOException {
+ context.mkdirs(new Path(path), new FsPermission("777"));
+ return verifyFileExists(context, path);
+ }
+
+ public static FileStatus getFileStatus(FileSystem context, String path)
+ throws IOException {
+ return context.getFileStatus(new Path(path));
+ }
+
+ public static boolean verifyFileExists(FileSystem context, String path) {
+ try {
+ FileStatus status = getFileStatus(context, path);
+ if (status != null) {
+ return true;
+ }
+ } catch (Exception e) {
+ return false;
+ }
+
+ return false;
+ }
+
+ public static boolean checkForFileInDirectory(FileSystem context,
+ String testPath, String targetFile) throws AccessControlException,
+ FileNotFoundException,
+ UnsupportedFileSystemException, IllegalArgumentException,
+ IOException {
+ FileStatus[] fileStatus = context.listStatus(new Path(testPath));
+ String file = null;
+ String verifyPath = testPath + "/" + targetFile;
+ if (testPath.equals("/")) {
+ verifyPath = testPath + targetFile;
+ }
+
+ Boolean found = false;
+ for (int i = 0; i < fileStatus.length; i++) {
+ FileStatus f = fileStatus[i];
+ file = Path.getPathWithoutSchemeAndAuthority(f.getPath()).toString();
+ if (file.equals(verifyPath)) {
+ found = true;
+ }
+ }
+ return found;
+ }
+
+ public static int countContents(FileSystem context, String testPath)
+ throws IOException {
+ FileStatus[] fileStatus = context.listStatus(new Path(testPath));
+ return fileStatus.length;
+ }
+
+ public static void createFile(FileSystem fs, String path, long length)
+ throws IOException {
+ FsPermission permissions = new FsPermission("700");
+ FSDataOutputStream writeStream = fs.create(new Path(path), permissions,
+ true, 1000, (short) 1, BLOCK_SIZE_BYTES, null);
+ for (int i = 0; i < length; i++) {
+ writeStream.write(i);
+ }
+ writeStream.close();
+ }
+
+ public static String readFile(FileSystem fs, String path) throws IOException {
+ // Read the file from the filesystem via the active namenode
+ Path fileName = new Path(path);
+ InputStreamReader reader = new InputStreamReader(fs.open(fileName));
+ BufferedReader bufferedReader = new BufferedReader(reader);
+ StringBuilder data = new StringBuilder();
+ String line;
+
+ while ((line = bufferedReader.readLine()) != null) {
+ data.append(line);
+ }
+
+ bufferedReader.close();
+ reader.close();
+ return data.toString();
+ }
+
+ public static boolean deleteFile(FileSystem fs, String path)
+ throws IOException {
+ return fs.delete(new Path(path), true);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/40] hadoop git commit: HDFS-10687. Federation Membership State
Store internal API. Contributed by Jason Kace and Inigo Goiri.
Posted by in...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestNamenodeResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestNamenodeResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestNamenodeResolver.java
new file mode 100644
index 0000000..2d74505
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestNamenodeResolver.java
@@ -0,0 +1,284 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMENODES;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMESERVICES;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.ROUTERS;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createNamenodeReport;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyException;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.clearRecords;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.getStateStoreConfiguration;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.newStateStore;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.waitStateStore;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreUnavailableException;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test the basic {@link ActiveNamenodeResolver} functionality.
+ */
+public class TestNamenodeResolver {
+
+ private static StateStoreService stateStore;
+ private static ActiveNamenodeResolver namenodeResolver;
+
+ @BeforeClass
+ public static void create() throws Exception {
+
+ Configuration conf = getStateStoreConfiguration();
+
+ // Reduce expirations to 5 seconds
+ conf.setLong(
+ DFSConfigKeys.FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS,
+ TimeUnit.SECONDS.toMillis(5));
+
+ stateStore = newStateStore(conf);
+ assertNotNull(stateStore);
+
+ namenodeResolver = new MembershipNamenodeResolver(conf, stateStore);
+ namenodeResolver.setRouterId(ROUTERS[0]);
+ }
+
+ @AfterClass
+ public static void destroy() throws Exception {
+ stateStore.stop();
+ stateStore.close();
+ }
+
+ @Before
+ public void setup() throws IOException, InterruptedException {
+ // Wait for state store to connect
+ stateStore.loadDriver();
+ waitStateStore(stateStore, 10000);
+
+ // Clear NN registrations
+ boolean cleared = clearRecords(stateStore, MembershipState.class);
+ assertTrue(cleared);
+ }
+
+ @Test
+ public void testStateStoreDisconnected() throws Exception {
+
+ // Add an entry to the store
+ NamenodeStatusReport report = createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[0], HAServiceState.ACTIVE);
+ assertTrue(namenodeResolver.registerNamenode(report));
+
+ // Close the data store driver
+ stateStore.closeDriver();
+ assertFalse(stateStore.isDriverReady());
+
+ // Flush the caches
+ stateStore.refreshCaches(true);
+
+ // Verify commands fail due to no cached data and no state store
+ // connectivity.
+ List<? extends FederationNamenodeContext> nns =
+ namenodeResolver.getNamenodesForBlockPoolId(NAMESERVICES[0]);
+ assertNull(nns);
+
+ verifyException(namenodeResolver, "registerNamenode",
+ StateStoreUnavailableException.class,
+ new Class[] {NamenodeStatusReport.class}, new Object[] {report});
+ }
+
+ /**
+ * Verify the first registration on the resolver.
+ *
+ * @param nsId Nameservice identifier.
+ * @param nnId Namenode identifier within the nemeservice.
+ * @param resultsCount Number of results expected.
+ * @param state Expected state for the first one.
+ * @throws IOException If we cannot get the namenodes.
+ */
+ private void verifyFirstRegistration(String nsId, String nnId,
+ int resultsCount, FederationNamenodeServiceState state)
+ throws IOException {
+ List<? extends FederationNamenodeContext> namenodes =
+ namenodeResolver.getNamenodesForNameserviceId(nsId);
+ if (resultsCount == 0) {
+ assertNull(namenodes);
+ } else {
+ assertEquals(resultsCount, namenodes.size());
+ if (namenodes.size() > 0) {
+ FederationNamenodeContext namenode = namenodes.get(0);
+ assertEquals(state, namenode.getState());
+ assertEquals(nnId, namenode.getNamenodeId());
+ }
+ }
+ }
+
+ @Test
+ public void testRegistrationExpired()
+ throws InterruptedException, IOException {
+
+ // Populate the state store with a single NN element
+ // 1) ns0:nn0 - Active
+ // Wait for the entry to expire without heartbeating
+ // Verify the NN entry is not accessible once expired.
+ NamenodeStatusReport report = createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[0], HAServiceState.ACTIVE);
+ assertTrue(namenodeResolver.registerNamenode(report));
+
+ // Load cache
+ stateStore.refreshCaches(true);
+
+ // Verify
+ verifyFirstRegistration(
+ NAMESERVICES[0], NAMENODES[0], 1,
+ FederationNamenodeServiceState.ACTIVE);
+
+ // Wait past expiration (set in conf to 5 seconds)
+ Thread.sleep(6000);
+ // Reload cache
+ stateStore.refreshCaches(true);
+
+ // Verify entry is now expired and is no longer in the cache
+ verifyFirstRegistration(
+ NAMESERVICES[0], NAMENODES[0], 0,
+ FederationNamenodeServiceState.ACTIVE);
+
+ // Heartbeat again, updates dateModified
+ assertTrue(namenodeResolver.registerNamenode(report));
+ // Reload cache
+ stateStore.refreshCaches(true);
+
+ // Verify updated entry is marked active again and accessible to RPC server
+ verifyFirstRegistration(
+ NAMESERVICES[0], NAMENODES[0], 1,
+ FederationNamenodeServiceState.ACTIVE);
+ }
+
+ @Test
+ public void testRegistrationNamenodeSelection()
+ throws InterruptedException, IOException {
+
+ // 1) ns0:nn0 - Active
+ // 2) ns0:nn1 - Standby (newest)
+ // Verify the selected entry is the active entry
+ assertTrue(namenodeResolver.registerNamenode(
+ createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[0], HAServiceState.ACTIVE)));
+ Thread.sleep(100);
+ assertTrue(namenodeResolver.registerNamenode(
+ createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[1], HAServiceState.STANDBY)));
+
+ stateStore.refreshCaches(true);
+
+ verifyFirstRegistration(
+ NAMESERVICES[0], NAMENODES[0], 2,
+ FederationNamenodeServiceState.ACTIVE);
+
+ // 1) ns0:nn0 - Expired (stale)
+ // 2) ns0:nn1 - Standby (newest)
+ // Verify the selected entry is the standby entry as the active entry is
+ // stale
+ assertTrue(namenodeResolver.registerNamenode(
+ createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[0], HAServiceState.ACTIVE)));
+
+ // Expire active registration
+ Thread.sleep(6000);
+
+ // Refresh standby registration
+ assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[1], HAServiceState.STANDBY)));
+
+ // Verify that standby is selected (active is now expired)
+ stateStore.refreshCaches(true);
+ verifyFirstRegistration(NAMESERVICES[0], NAMENODES[1], 1,
+ FederationNamenodeServiceState.STANDBY);
+
+ // 1) ns0:nn0 - Active
+ // 2) ns0:nn1 - Unavailable (newest)
+ // Verify the selected entry is the active entry
+ assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[0], HAServiceState.ACTIVE)));
+ Thread.sleep(100);
+ assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[1], null)));
+ stateStore.refreshCaches(true);
+ verifyFirstRegistration(NAMESERVICES[0], NAMENODES[0], 2,
+ FederationNamenodeServiceState.ACTIVE);
+
+ // 1) ns0:nn0 - Unavailable (newest)
+ // 2) ns0:nn1 - Standby
+ // Verify the selected entry is the standby entry
+ assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[1], HAServiceState.STANDBY)));
+ Thread.sleep(1000);
+ assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[0], null)));
+
+ stateStore.refreshCaches(true);
+ verifyFirstRegistration(NAMESERVICES[0], NAMENODES[1], 2,
+ FederationNamenodeServiceState.STANDBY);
+
+ // 1) ns0:nn0 - Active (oldest)
+ // 2) ns0:nn1 - Standby
+ // 3) ns0:nn2 - Active (newest)
+ // Verify the selected entry is the newest active entry
+ assertTrue(namenodeResolver.registerNamenode(
+ createNamenodeReport(NAMESERVICES[0], NAMENODES[0], null)));
+ Thread.sleep(100);
+ assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[1], HAServiceState.STANDBY)));
+ Thread.sleep(100);
+ assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[2], HAServiceState.ACTIVE)));
+
+ stateStore.refreshCaches(true);
+ verifyFirstRegistration(NAMESERVICES[0], NAMENODES[2], 3,
+ FederationNamenodeServiceState.ACTIVE);
+
+ // 1) ns0:nn0 - Standby (oldest)
+ // 2) ns0:nn1 - Standby (newest)
+ // 3) ns0:nn2 - Standby
+ // Verify the selected entry is the newest standby entry
+ assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[0], HAServiceState.STANDBY)));
+ assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[2], HAServiceState.STANDBY)));
+ Thread.sleep(1500);
+ assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[1], HAServiceState.STANDBY)));
+
+ stateStore.refreshCaches(true);
+ verifyFirstRegistration(NAMESERVICES[0], NAMENODES[1], 3,
+ FederationNamenodeServiceState.STANDBY);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
index fc5aebd..598b9cf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
@@ -34,9 +34,12 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileBaseImpl;
import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipStats;
import org.apache.hadoop.util.Time;
/**
@@ -96,7 +99,7 @@ public final class FederationStateStoreTestUtils {
* @throws IOException If it cannot create the State Store.
* @throws InterruptedException If we cannot wait for the store to start.
*/
- public static StateStoreService getStateStore(
+ public static StateStoreService newStateStore(
Configuration configuration) throws IOException, InterruptedException {
StateStoreService stateStore = new StateStoreService();
@@ -205,6 +208,7 @@ public final class FederationStateStoreTestUtils {
if (!synchronizeRecords(store, emptyList, recordClass)) {
return false;
}
+ store.refreshCaches(true);
return true;
}
@@ -229,4 +233,21 @@ public final class FederationStateStoreTestUtils {
}
return false;
}
+
+ public static MembershipState createMockRegistrationForNamenode(
+ String nameserviceId, String namenodeId,
+ FederationNamenodeServiceState state) throws IOException {
+ MembershipState entry = MembershipState.newInstance(
+ "routerId", nameserviceId, namenodeId, "clusterId", "test",
+ "0.0.0.0:0", "0.0.0.0:0", "0.0.0.0:0", "0.0.0.0:0", state, false);
+ MembershipStats stats = MembershipStats.newInstance();
+ stats.setNumOfActiveDatanodes(100);
+ stats.setNumOfDeadDatanodes(10);
+ stats.setNumOfDecommissioningDatanodes(20);
+ stats.setNumOfDecomActiveDatanodes(15);
+ stats.setNumOfDecomDeadDatanodes(5);
+ stats.setNumOfBlocks(10);
+ entry.setStats(stats);
+ return entry;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreBase.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreBase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreBase.java
new file mode 100644
index 0000000..7f6704e
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreBase.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.newStateStore;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.getStateStoreConfiguration;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.waitStateStore;
+import static org.junit.Assert.assertNotNull;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+/**
+ * Test the basic {@link StateStoreService} {@link MountTableStore}
+ * functionality.
+ */
+public class TestStateStoreBase {
+
+ private static StateStoreService stateStore;
+ private static Configuration conf;
+
+ protected static StateStoreService getStateStore() {
+ return stateStore;
+ }
+
+ protected static Configuration getConf() {
+ return conf;
+ }
+
+ @BeforeClass
+ public static void createBase() throws IOException, InterruptedException {
+
+ conf = getStateStoreConfiguration();
+
+ // Disable auto-reconnect to data store
+ conf.setLong(DFSConfigKeys.FEDERATION_STORE_CONNECTION_TEST_MS,
+ TimeUnit.HOURS.toMillis(1));
+ }
+
+ @AfterClass
+ public static void destroyBase() throws Exception {
+ if (stateStore != null) {
+ stateStore.stop();
+ stateStore.close();
+ stateStore = null;
+ }
+ }
+
+ @Before
+ public void setupBase() throws IOException, InterruptedException,
+ InstantiationException, IllegalAccessException {
+ if (stateStore == null) {
+ stateStore = newStateStore(conf);
+ assertNotNull(stateStore);
+ }
+ // Wait for state store to connect
+ stateStore.loadDriver();
+ waitStateStore(stateStore, TimeUnit.SECONDS.toMillis(10));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMembershipState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMembershipState.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMembershipState.java
new file mode 100644
index 0000000..26f081b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMembershipState.java
@@ -0,0 +1,463 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMENODES;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMESERVICES;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.ROUTERS;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyException;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.clearRecords;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.createMockRegistrationForNamenode;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.synchronizeRecords;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateNamenodeRegistrationRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.util.Time;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test the basic {@link MembershipStore} membership functionality.
+ */
+public class TestStateStoreMembershipState extends TestStateStoreBase {
+
+ private static MembershipStore membershipStore;
+
+ @BeforeClass
+ public static void create() {
+ // Reduce expirations to 5 seconds
+ getConf().setLong(
+ DFSConfigKeys.FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS,
+ TimeUnit.SECONDS.toMillis(5));
+ }
+
+ @Before
+ public void setup() throws IOException, InterruptedException {
+
+ membershipStore =
+ getStateStore().getRegisteredRecordStore(MembershipStore.class);
+
+ // Clear NN registrations
+ assertTrue(clearRecords(getStateStore(), MembershipState.class));
+ }
+
+ @Test
+ public void testNamenodeStateOverride() throws Exception {
+ // Populate the state store
+ // 1) ns0:nn0 - Standby
+ String ns = "ns0";
+ String nn = "nn0";
+ MembershipState report = createRegistration(
+ ns, nn, ROUTERS[1], FederationNamenodeServiceState.STANDBY);
+ assertTrue(namenodeHeartbeat(report));
+
+ // Load data into cache and calculate quorum
+ assertTrue(getStateStore().loadCache(MembershipStore.class, true));
+
+ MembershipState existingState = getNamenodeRegistration(ns, nn);
+ assertEquals(
+ FederationNamenodeServiceState.STANDBY, existingState.getState());
+
+ // Override cache
+ UpdateNamenodeRegistrationRequest request =
+ UpdateNamenodeRegistrationRequest.newInstance(
+ ns, nn, FederationNamenodeServiceState.ACTIVE);
+ assertTrue(membershipStore.updateNamenodeRegistration(request).getResult());
+
+ MembershipState newState = getNamenodeRegistration(ns, nn);
+ assertEquals(FederationNamenodeServiceState.ACTIVE, newState.getState());
+ }
+
+ @Test
+ public void testStateStoreDisconnected() throws Exception {
+
+ // Close the data store driver
+ getStateStore().closeDriver();
+ assertFalse(getStateStore().isDriverReady());
+
+ NamenodeHeartbeatRequest hbRequest = NamenodeHeartbeatRequest.newInstance();
+ hbRequest.setNamenodeMembership(
+ createMockRegistrationForNamenode(
+ "test", "test", FederationNamenodeServiceState.UNAVAILABLE));
+ verifyException(membershipStore, "namenodeHeartbeat",
+ StateStoreUnavailableException.class,
+ new Class[] {NamenodeHeartbeatRequest.class},
+ new Object[] {hbRequest });
+
+ // Information from cache, no exception should be triggered for these
+ // TODO - should cached info expire at some point?
+ GetNamenodeRegistrationsRequest getRequest =
+ GetNamenodeRegistrationsRequest.newInstance();
+ verifyException(membershipStore,
+ "getNamenodeRegistrations", null,
+ new Class[] {GetNamenodeRegistrationsRequest.class},
+ new Object[] {getRequest});
+
+ verifyException(membershipStore,
+ "getExpiredNamenodeRegistrations", null,
+ new Class[] {GetNamenodeRegistrationsRequest.class},
+ new Object[] {getRequest});
+
+ UpdateNamenodeRegistrationRequest overrideRequest =
+ UpdateNamenodeRegistrationRequest.newInstance();
+ verifyException(membershipStore,
+ "updateNamenodeRegistration", null,
+ new Class[] {UpdateNamenodeRegistrationRequest.class},
+ new Object[] {overrideRequest});
+ }
+
+ private void registerAndLoadRegistrations(
+ List<MembershipState> registrationList) throws IOException {
+ // Populate
+ assertTrue(synchronizeRecords(
+ getStateStore(), registrationList, MembershipState.class));
+
+ // Load into cache
+ assertTrue(getStateStore().loadCache(MembershipStore.class, true));
+ }
+
+ private MembershipState createRegistration(String ns, String nn,
+ String router, FederationNamenodeServiceState state) throws IOException {
+ MembershipState record = MembershipState.newInstance(
+ router, ns,
+ nn, "testcluster", "testblock-" + ns, "testrpc-"+ ns + nn,
+ "testservice-"+ ns + nn, "testlifeline-"+ ns + nn,
+ "testweb-" + ns + nn, state, false);
+ return record;
+ }
+
+ @Test
+ public void testRegistrationMajorityQuorum()
+ throws InterruptedException, IOException {
+
+ // Populate the state store with a set of non-matching elements
+ // 1) ns0:nn0 - Standby (newest)
+ // 2) ns0:nn0 - Active (oldest)
+ // 3) ns0:nn0 - Active (2nd oldest)
+ // 4) ns0:nn0 - Active (3nd oldest element, newest active element)
+ // Verify the selected entry is the newest majority opinion (4)
+ String ns = "ns0";
+ String nn = "nn0";
+
+ // Active - oldest
+ MembershipState report = createRegistration(
+ ns, nn, ROUTERS[1], FederationNamenodeServiceState.ACTIVE);
+ assertTrue(namenodeHeartbeat(report));
+ Thread.sleep(1000);
+
+ // Active - 2nd oldest
+ report = createRegistration(
+ ns, nn, ROUTERS[2], FederationNamenodeServiceState.ACTIVE);
+ assertTrue(namenodeHeartbeat(report));
+ Thread.sleep(1000);
+
+ // Active - 3rd oldest, newest active element
+ report = createRegistration(
+ ns, nn, ROUTERS[3], FederationNamenodeServiceState.ACTIVE);
+ assertTrue(namenodeHeartbeat(report));
+
+ // standby - newest overall
+ report = createRegistration(
+ ns, nn, ROUTERS[0], FederationNamenodeServiceState.STANDBY);
+ assertTrue(namenodeHeartbeat(report));
+
+ // Load and calculate quorum
+ assertTrue(getStateStore().loadCache(MembershipStore.class, true));
+
+ // Verify quorum entry
+ MembershipState quorumEntry = getNamenodeRegistration(
+ report.getNameserviceId(), report.getNamenodeId());
+ assertNotNull(quorumEntry);
+ assertEquals(quorumEntry.getRouterId(), ROUTERS[3]);
+ }
+
+ @Test
+ public void testRegistrationQuorumExcludesExpired()
+ throws InterruptedException, IOException {
+
+ // Populate the state store with some expired entries and verify the expired
+ // entries are ignored.
+ // 1) ns0:nn0 - Active
+ // 2) ns0:nn0 - Expired
+ // 3) ns0:nn0 - Expired
+ // 4) ns0:nn0 - Expired
+ // Verify the selected entry is the active entry
+ List<MembershipState> registrationList = new ArrayList<>();
+ String ns = "ns0";
+ String nn = "nn0";
+ String rpcAddress = "testrpcaddress";
+ String serviceAddress = "testserviceaddress";
+ String lifelineAddress = "testlifelineaddress";
+ String blockPoolId = "testblockpool";
+ String clusterId = "testcluster";
+ String webAddress = "testwebaddress";
+ boolean safemode = false;
+
+ // Active
+ MembershipState record = MembershipState.newInstance(
+ ROUTERS[0], ns, nn, clusterId, blockPoolId,
+ rpcAddress, serviceAddress, lifelineAddress, webAddress,
+ FederationNamenodeServiceState.ACTIVE, safemode);
+ registrationList.add(record);
+
+ // Expired
+ record = MembershipState.newInstance(
+ ROUTERS[1], ns, nn, clusterId, blockPoolId,
+ rpcAddress, serviceAddress, lifelineAddress, webAddress,
+ FederationNamenodeServiceState.EXPIRED, safemode);
+ registrationList.add(record);
+
+ // Expired
+ record = MembershipState.newInstance(
+ ROUTERS[2], ns, nn, clusterId, blockPoolId,
+ rpcAddress, serviceAddress, lifelineAddress, webAddress,
+ FederationNamenodeServiceState.EXPIRED, safemode);
+ registrationList.add(record);
+
+ // Expired
+ record = MembershipState.newInstance(
+ ROUTERS[3], ns, nn, clusterId, blockPoolId,
+ rpcAddress, serviceAddress, lifelineAddress, webAddress,
+ FederationNamenodeServiceState.EXPIRED, safemode);
+ registrationList.add(record);
+ registerAndLoadRegistrations(registrationList);
+
+ // Verify quorum entry chooses active membership
+ MembershipState quorumEntry = getNamenodeRegistration(
+ record.getNameserviceId(), record.getNamenodeId());
+ assertNotNull(quorumEntry);
+ assertEquals(ROUTERS[0], quorumEntry.getRouterId());
+ }
+
+ @Test
+ public void testRegistrationQuorumAllExpired() throws IOException {
+
+ // 1) ns0:nn0 - Expired (oldest)
+ // 2) ns0:nn0 - Expired
+ // 3) ns0:nn0 - Expired
+ // 4) ns0:nn0 - Expired
+ // Verify no entry is either selected or cached
+ List<MembershipState> registrationList = new ArrayList<>();
+ String ns = NAMESERVICES[0];
+ String nn = NAMENODES[0];
+ String rpcAddress = "testrpcaddress";
+ String serviceAddress = "testserviceaddress";
+ String lifelineAddress = "testlifelineaddress";
+ String blockPoolId = "testblockpool";
+ String clusterId = "testcluster";
+ String webAddress = "testwebaddress";
+ boolean safemode = false;
+ long startingTime = Time.now();
+
+ // Expired
+ MembershipState record = MembershipState.newInstance(
+ ROUTERS[0], ns, nn, clusterId, blockPoolId,
+ rpcAddress, webAddress, lifelineAddress, webAddress,
+ FederationNamenodeServiceState.EXPIRED, safemode);
+ record.setDateModified(startingTime - 10000);
+ registrationList.add(record);
+
+ // Expired
+ record = MembershipState.newInstance(
+ ROUTERS[1], ns, nn, clusterId, blockPoolId,
+ rpcAddress, serviceAddress, lifelineAddress, webAddress,
+ FederationNamenodeServiceState.EXPIRED, safemode);
+ record.setDateModified(startingTime);
+ registrationList.add(record);
+
+ // Expired
+ record = MembershipState.newInstance(
+ ROUTERS[2], ns, nn, clusterId, blockPoolId,
+ rpcAddress, serviceAddress, lifelineAddress, webAddress,
+ FederationNamenodeServiceState.EXPIRED, safemode);
+ record.setDateModified(startingTime);
+ registrationList.add(record);
+
+ // Expired
+ record = MembershipState.newInstance(
+ ROUTERS[3], ns, nn, clusterId, blockPoolId,
+ rpcAddress, serviceAddress, lifelineAddress, webAddress,
+ FederationNamenodeServiceState.EXPIRED, safemode);
+ record.setDateModified(startingTime);
+ registrationList.add(record);
+
+ registerAndLoadRegistrations(registrationList);
+
+ // Verify no entry is found for this nameservice
+ assertNull(getNamenodeRegistration(
+ record.getNameserviceId(), record.getNamenodeId()));
+ }
+
+ @Test
+ public void testRegistrationNoQuorum()
+ throws InterruptedException, IOException {
+
+ // Populate the state store with a set of non-matching elements
+ // 1) ns0:nn0 - Standby (newest)
+ // 2) ns0:nn0 - Standby (oldest)
+ // 3) ns0:nn0 - Active (2nd oldest)
+ // 4) ns0:nn0 - Active (3nd oldest element, newest active element)
+ // Verify the selected entry is the newest entry (1)
+ MembershipState report1 = createRegistration(
+ NAMESERVICES[0], NAMENODES[0], ROUTERS[1],
+ FederationNamenodeServiceState.STANDBY);
+ assertTrue(namenodeHeartbeat(report1));
+ Thread.sleep(100);
+ MembershipState report2 = createRegistration(
+ NAMESERVICES[0], NAMENODES[0], ROUTERS[2],
+ FederationNamenodeServiceState.ACTIVE);
+ assertTrue(namenodeHeartbeat(report2));
+ Thread.sleep(100);
+ MembershipState report3 = createRegistration(
+ NAMESERVICES[0], NAMENODES[0], ROUTERS[3],
+ FederationNamenodeServiceState.ACTIVE);
+ assertTrue(namenodeHeartbeat(report3));
+ Thread.sleep(100);
+ MembershipState report4 = createRegistration(
+ NAMESERVICES[0], NAMENODES[0], ROUTERS[0],
+ FederationNamenodeServiceState.STANDBY);
+ assertTrue(namenodeHeartbeat(report4));
+
+ // Load and calculate quorum
+ assertTrue(getStateStore().loadCache(MembershipStore.class, true));
+
+ // Verify quorum entry uses the newest data, even though it is standby
+ MembershipState quorumEntry = getNamenodeRegistration(
+ report1.getNameserviceId(), report1.getNamenodeId());
+ assertNotNull(quorumEntry);
+ assertEquals(ROUTERS[0], quorumEntry.getRouterId());
+ assertEquals(
+ FederationNamenodeServiceState.STANDBY, quorumEntry.getState());
+ }
+
+ @Test
+ public void testRegistrationExpired()
+ throws InterruptedException, IOException {
+
+ // Populate the state store with a single NN element
+ // 1) ns0:nn0 - Active
+ // Wait for the entry to expire without heartbeating
+ // Verify the NN entry is populated as EXPIRED internally in the state store
+
+ MembershipState report = createRegistration(
+ NAMESERVICES[0], NAMENODES[0], ROUTERS[0],
+ FederationNamenodeServiceState.ACTIVE);
+ assertTrue(namenodeHeartbeat(report));
+
+ // Load cache
+ assertTrue(getStateStore().loadCache(MembershipStore.class, true));
+
+ // Verify quorum and entry
+ MembershipState quorumEntry = getNamenodeRegistration(
+ report.getNameserviceId(), report.getNamenodeId());
+ assertNotNull(quorumEntry);
+ assertEquals(ROUTERS[0], quorumEntry.getRouterId());
+ assertEquals(FederationNamenodeServiceState.ACTIVE, quorumEntry.getState());
+
+ // Wait past expiration (set in conf to 5 seconds)
+ Thread.sleep(6000);
+ // Reload cache
+ assertTrue(getStateStore().loadCache(MembershipStore.class, true));
+
+ // Verify entry is now expired and is no longer in the cache
+ quorumEntry = getNamenodeRegistration(NAMESERVICES[0], NAMENODES[0]);
+ assertNull(quorumEntry);
+
+ // Verify entry is now expired and can't be used by RPC service
+ quorumEntry = getNamenodeRegistration(
+ report.getNameserviceId(), report.getNamenodeId());
+ assertNull(quorumEntry);
+
+ // Heartbeat again, updates dateModified
+ assertTrue(namenodeHeartbeat(report));
+ // Reload cache
+ assertTrue(getStateStore().loadCache(MembershipStore.class, true));
+
+ // Verify updated entry marked as active and is accessible to RPC server
+ quorumEntry = getNamenodeRegistration(
+ report.getNameserviceId(), report.getNamenodeId());
+ assertNotNull(quorumEntry);
+ assertEquals(ROUTERS[0], quorumEntry.getRouterId());
+ assertEquals(FederationNamenodeServiceState.ACTIVE, quorumEntry.getState());
+ }
+
+ /**
+ * Get a single namenode membership record from the store.
+ *
+ * @param nsId The HDFS nameservice ID to search for
+ * @param nnId The HDFS namenode ID to search for
+ * @return The single NamenodeMembershipRecord that matches the query or null
+ * if not found.
+ * @throws IOException if the query could not be executed.
+ */
+ private MembershipState getNamenodeRegistration(
+ final String nsId, final String nnId) throws IOException {
+
+ MembershipState partial = MembershipState.newInstance();
+ partial.setNameserviceId(nsId);
+ partial.setNamenodeId(nnId);
+ GetNamenodeRegistrationsRequest request =
+ GetNamenodeRegistrationsRequest.newInstance(partial);
+ GetNamenodeRegistrationsResponse response =
+ membershipStore.getNamenodeRegistrations(request);
+
+ List<MembershipState> results = response.getNamenodeMemberships();
+ if (results != null && results.size() == 1) {
+ MembershipState record = results.get(0);
+ return record;
+ }
+ return null;
+ }
+
+ /**
+ * Register a namenode heartbeat with the state store.
+ *
+ * @param store FederationMembershipStateStore instance to retrieve the
+ * membership data records.
+ * @param namenode A fully populated namenode membership record to be
+ * committed to the data store.
+ * @return True if successful, false otherwise.
+ * @throws IOException if the state store query could not be performed.
+ */
+ private boolean namenodeHeartbeat(MembershipState namenode)
+ throws IOException {
+
+ NamenodeHeartbeatRequest request =
+ NamenodeHeartbeatRequest.newInstance(namenode);
+ NamenodeHeartbeatResponse response =
+ membershipStore.namenodeHeartbeat(request);
+ return response.getResult();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
index 7f0b36a..dc51ee9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
@@ -31,11 +31,14 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Random;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
import org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils;
import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
import org.apache.hadoop.hdfs.server.federation.store.records.Query;
import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
import org.junit.AfterClass;
@@ -54,6 +57,8 @@ public class TestStateStoreDriverBase {
private static StateStoreService stateStore;
private static Configuration conf;
+ private static final Random RANDOM = new Random();
+
/**
* Get the State Store driver.
@@ -78,29 +83,47 @@ public class TestStateStoreDriverBase {
*/
public static void getStateStore(Configuration config) throws Exception {
conf = config;
- stateStore = FederationStateStoreTestUtils.getStateStore(conf);
+ stateStore = FederationStateStoreTestUtils.newStateStore(conf);
+ }
+
+ private String generateRandomString() {
+ String randomString = "/randomString-" + RANDOM.nextInt();
+ return randomString;
+ }
+
+ @SuppressWarnings("rawtypes")
+ private <T extends Enum> T generateRandomEnum(Class<T> enumClass) {
+ int x = RANDOM.nextInt(enumClass.getEnumConstants().length);
+ T data = enumClass.getEnumConstants()[x];
+ return data;
}
+ @SuppressWarnings("unchecked")
private <T extends BaseRecord> T generateFakeRecord(Class<T> recordClass)
throws IllegalArgumentException, IllegalAccessException, IOException {
- // TODO add record
+ if (recordClass == MembershipState.class) {
+ return (T) MembershipState.newInstance(generateRandomString(),
+ generateRandomString(), generateRandomString(),
+ generateRandomString(), generateRandomString(),
+ generateRandomString(), generateRandomString(),
+ generateRandomString(), generateRandomString(),
+ generateRandomEnum(FederationNamenodeServiceState.class), false);
+ }
+
return null;
}
/**
* Validate if a record is the same.
*
- * @param original
- * @param committed
+ * @param original Original record.
+ * @param committed Committed record.
* @param assertEquals Assert if the records are equal or just return.
- * @return
- * @throws IllegalArgumentException
- * @throws IllegalAccessException
+ * @return If the record is successfully validated.
*/
private boolean validateRecord(
- BaseRecord original, BaseRecord committed, boolean assertEquals)
- throws IllegalArgumentException, IllegalAccessException {
+ BaseRecord original, BaseRecord committed, boolean assertEquals) {
boolean ret = true;
@@ -131,7 +154,7 @@ public class TestStateStoreDriverBase {
}
public static void removeAll(StateStoreDriver driver) throws IOException {
- // TODO add records to remove
+ driver.removeAll(MembershipState.class);
}
public <T extends BaseRecord> void testInsert(
@@ -139,17 +162,20 @@ public class TestStateStoreDriverBase {
throws IllegalArgumentException, IllegalAccessException, IOException {
assertTrue(driver.removeAll(recordClass));
- QueryResult<T> records = driver.get(recordClass);
- assertTrue(records.getRecords().isEmpty());
+ QueryResult<T> queryResult0 = driver.get(recordClass);
+ List<T> records0 = queryResult0.getRecords();
+ assertTrue(records0.isEmpty());
// Insert single
BaseRecord record = generateFakeRecord(recordClass);
driver.put(record, true, false);
// Verify
- records = driver.get(recordClass);
- assertEquals(1, records.getRecords().size());
- validateRecord(record, records.getRecords().get(0), true);
+ QueryResult<T> queryResult1 = driver.get(recordClass);
+ List<T> records1 = queryResult1.getRecords();
+ assertEquals(1, records1.size());
+ T record0 = records1.get(0);
+ validateRecord(record, record0, true);
// Insert multiple
List<T> insertList = new ArrayList<>();
@@ -160,8 +186,9 @@ public class TestStateStoreDriverBase {
driver.putAll(insertList, true, false);
// Verify
- records = driver.get(recordClass);
- assertEquals(11, records.getRecords().size());
+ QueryResult<T> queryResult2 = driver.get(recordClass);
+ List<T> records2 = queryResult2.getRecords();
+ assertEquals(11, records2.size());
}
public <T extends BaseRecord> void testFetchErrors(StateStoreDriver driver,
@@ -319,23 +346,23 @@ public class TestStateStoreDriverBase {
public void testInsert(StateStoreDriver driver)
throws IllegalArgumentException, IllegalAccessException, IOException {
- // TODO add records
+ testInsert(driver, MembershipState.class);
}
public void testPut(StateStoreDriver driver)
throws IllegalArgumentException, ReflectiveOperationException,
IOException, SecurityException {
- // TODO add records
+ testPut(driver, MembershipState.class);
}
public void testRemove(StateStoreDriver driver)
throws IllegalArgumentException, IllegalAccessException, IOException {
- // TODO add records
+ testRemove(driver, MembershipState.class);
}
public void testFetchErrors(StateStoreDriver driver)
throws IllegalArgumentException, IllegalAccessException, IOException {
- // TODO add records
+ testFetchErrors(driver, MembershipState.class);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b43bb809/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMembershipState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMembershipState.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMembershipState.java
new file mode 100644
index 0000000..d922414
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMembershipState.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.junit.Test;
+
+/**
+ * Test the Membership State records.
+ */
+public class TestMembershipState {
+
+ private static final String ROUTER = "router";
+ private static final String NAMESERVICE = "nameservice";
+ private static final String NAMENODE = "namenode";
+ private static final String CLUSTER_ID = "cluster";
+ private static final String BLOCKPOOL_ID = "blockpool";
+ private static final String RPC_ADDRESS = "rpcaddress";
+ private static final String SERVICE_ADDRESS = "serviceaddress";
+ private static final String LIFELINE_ADDRESS = "lifelineaddress";
+ private static final String WEB_ADDRESS = "webaddress";
+ private static final boolean SAFE_MODE = false;
+
+ private static final long DATE_CREATED = 100;
+ private static final long DATE_MODIFIED = 200;
+
+ private static final long NUM_BLOCKS = 300;
+ private static final long NUM_FILES = 400;
+ private static final int NUM_DEAD = 500;
+ private static final int NUM_ACTIVE = 600;
+ private static final int NUM_DECOM = 700;
+ private static final int NUM_DECOM_ACTIVE = 800;
+ private static final int NUM_DECOM_DEAD = 900;
+ private static final long NUM_BLOCK_MISSING = 1000;
+
+ private static final long TOTAL_SPACE = 1100;
+ private static final long AVAILABLE_SPACE = 1200;
+
+ private static final FederationNamenodeServiceState STATE =
+ FederationNamenodeServiceState.ACTIVE;
+
+ private MembershipState createRecord() throws IOException {
+
+ MembershipState record = MembershipState.newInstance(
+ ROUTER, NAMESERVICE, NAMENODE, CLUSTER_ID,
+ BLOCKPOOL_ID, RPC_ADDRESS, SERVICE_ADDRESS, LIFELINE_ADDRESS,
+ WEB_ADDRESS, STATE, SAFE_MODE);
+ record.setDateCreated(DATE_CREATED);
+ record.setDateModified(DATE_MODIFIED);
+
+ MembershipStats stats = MembershipStats.newInstance();
+ stats.setNumOfBlocks(NUM_BLOCKS);
+ stats.setNumOfFiles(NUM_FILES);
+ stats.setNumOfActiveDatanodes(NUM_ACTIVE);
+ stats.setNumOfDeadDatanodes(NUM_DEAD);
+ stats.setNumOfDecommissioningDatanodes(NUM_DECOM);
+ stats.setNumOfDecomActiveDatanodes(NUM_DECOM_ACTIVE);
+ stats.setNumOfDecomDeadDatanodes(NUM_DECOM_DEAD);
+ stats.setNumOfBlocksMissing(NUM_BLOCK_MISSING);
+ stats.setTotalSpace(TOTAL_SPACE);
+ stats.setAvailableSpace(AVAILABLE_SPACE);
+ record.setStats(stats);
+ return record;
+ }
+
+ private void validateRecord(MembershipState record) throws IOException {
+
+ assertEquals(ROUTER, record.getRouterId());
+ assertEquals(NAMESERVICE, record.getNameserviceId());
+ assertEquals(CLUSTER_ID, record.getClusterId());
+ assertEquals(BLOCKPOOL_ID, record.getBlockPoolId());
+ assertEquals(RPC_ADDRESS, record.getRpcAddress());
+ assertEquals(WEB_ADDRESS, record.getWebAddress());
+ assertEquals(STATE, record.getState());
+ assertEquals(SAFE_MODE, record.getIsSafeMode());
+ assertEquals(DATE_CREATED, record.getDateCreated());
+ assertEquals(DATE_MODIFIED, record.getDateModified());
+
+ MembershipStats stats = record.getStats();
+ assertEquals(NUM_BLOCKS, stats.getNumOfBlocks());
+ assertEquals(NUM_FILES, stats.getNumOfFiles());
+ assertEquals(NUM_ACTIVE, stats.getNumOfActiveDatanodes());
+ assertEquals(NUM_DEAD, stats.getNumOfDeadDatanodes());
+ assertEquals(NUM_DECOM, stats.getNumOfDecommissioningDatanodes());
+ assertEquals(NUM_DECOM_ACTIVE, stats.getNumOfDecomActiveDatanodes());
+ assertEquals(NUM_DECOM_DEAD, stats.getNumOfDecomDeadDatanodes());
+ assertEquals(TOTAL_SPACE, stats.getTotalSpace());
+ assertEquals(AVAILABLE_SPACE, stats.getAvailableSpace());
+ }
+
+ @Test
+ public void testGetterSetter() throws IOException {
+ MembershipState record = createRecord();
+ validateRecord(record);
+ }
+
+ @Test
+ public void testSerialization() throws IOException {
+
+ MembershipState record = createRecord();
+
+ StateStoreSerializer serializer = StateStoreSerializer.getSerializer();
+ String serializedString = serializer.serializeString(record);
+ MembershipState newRecord =
+ serializer.deserialize(serializedString, MembershipState.class);
+
+ validateRecord(newRecord);
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/40] hadoop git commit: HDFS-10880. Federation Mount Table State
Store internal API. Contributed by Jason Kace and Inigo Goiri.
Posted by in...@apache.org.
HDFS-10880. Federation Mount Table State Store internal API. Contributed by Jason Kace and Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/83f2aa35
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/83f2aa35
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/83f2aa35
Branch: refs/heads/HDFS-10467
Commit: 83f2aa35325a7e260f8499142720648a6c96bc88
Parents: ae5e135
Author: Inigo Goiri <in...@apache.org>
Authored: Fri Aug 4 18:00:12 2017 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu Sep 7 13:19:51 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 7 +-
.../federation/resolver/MountTableManager.java | 80 +++
.../federation/resolver/MountTableResolver.java | 544 +++++++++++++++++++
.../federation/resolver/PathLocation.java | 124 ++++-
.../resolver/order/DestinationOrder.java | 29 +
.../federation/resolver/order/package-info.java | 29 +
.../federation/router/FederationUtil.java | 56 +-
.../hdfs/server/federation/router/Router.java | 3 +-
.../federation/store/MountTableStore.java | 49 ++
.../federation/store/StateStoreService.java | 2 +
.../store/impl/MountTableStoreImpl.java | 116 ++++
.../protocol/AddMountTableEntryRequest.java | 47 ++
.../protocol/AddMountTableEntryResponse.java | 42 ++
.../protocol/GetMountTableEntriesRequest.java | 49 ++
.../protocol/GetMountTableEntriesResponse.java | 53 ++
.../protocol/RemoveMountTableEntryRequest.java | 49 ++
.../protocol/RemoveMountTableEntryResponse.java | 42 ++
.../protocol/UpdateMountTableEntryRequest.java | 51 ++
.../protocol/UpdateMountTableEntryResponse.java | 43 ++
.../pb/AddMountTableEntryRequestPBImpl.java | 84 +++
.../pb/AddMountTableEntryResponsePBImpl.java | 76 +++
.../pb/GetMountTableEntriesRequestPBImpl.java | 76 +++
.../pb/GetMountTableEntriesResponsePBImpl.java | 104 ++++
.../pb/RemoveMountTableEntryRequestPBImpl.java | 76 +++
.../pb/RemoveMountTableEntryResponsePBImpl.java | 76 +++
.../pb/UpdateMountTableEntryRequestPBImpl.java | 96 ++++
.../pb/UpdateMountTableEntryResponsePBImpl.java | 76 +++
.../federation/store/records/MountTable.java | 301 ++++++++++
.../store/records/impl/pb/MountTablePBImpl.java | 213 ++++++++
.../src/main/proto/FederationProtocol.proto | 61 ++-
.../hdfs/server/federation/MockResolver.java | 9 +-
.../resolver/TestMountTableResolver.java | 396 ++++++++++++++
.../store/FederationStateStoreTestUtils.java | 16 +
.../store/TestStateStoreMountTable.java | 250 +++++++++
.../store/driver/TestStateStoreDriverBase.java | 12 +
.../store/records/TestMountTable.java | 176 ++++++
36 files changed, 3437 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 1ad34d4..702729b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -27,6 +27,8 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRackFaultTolerant;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.RamDiskReplicaLruTracker;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
import org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver;
import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
@@ -1178,8 +1180,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
// HDFS Router State Store connection
public static final String FEDERATION_FILE_RESOLVER_CLIENT_CLASS =
FEDERATION_ROUTER_PREFIX + "file.resolver.client.class";
- public static final String FEDERATION_FILE_RESOLVER_CLIENT_CLASS_DEFAULT =
- "org.apache.hadoop.hdfs.server.federation.MockResolver";
+ public static final Class<? extends FileSubclusterResolver>
+ FEDERATION_FILE_RESOLVER_CLIENT_CLASS_DEFAULT =
+ MountTableResolver.class;
public static final String FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS =
FEDERATION_ROUTER_PREFIX + "namenode.resolver.client.class";
public static final Class<? extends ActiveNamenodeResolver>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableManager.java
new file mode 100644
index 0000000..c2e4a5b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableManager.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse;
+
+/**
+ * Manage a mount table.
+ */
+public interface MountTableManager {
+
+ /**
+ * Add an entry to the mount table.
+ *
+ * @param request Fully populated request object.
+ * @return True if the mount table entry was successfully committed to the
+ * data store.
+ * @throws IOException Throws exception if the data store is not initialized.
+ */
+ AddMountTableEntryResponse addMountTableEntry(
+ AddMountTableEntryRequest request) throws IOException;
+
+ /**
+ * Updates an existing entry in the mount table.
+ *
+ * @param request Fully populated request object.
+ * @return True if the mount table entry was successfully committed to the
+ * data store.
+ * @throws IOException Throws exception if the data store is not initialized.
+ */
+ UpdateMountTableEntryResponse updateMountTableEntry(
+ UpdateMountTableEntryRequest request) throws IOException;
+
+ /**
+ * Remove an entry from the mount table.
+ *
+ * @param request Fully populated request object.
+ * @return True the mount table entry was removed from the data store.
+ * @throws IOException Throws exception if the data store is not initialized.
+ */
+ RemoveMountTableEntryResponse removeMountTableEntry(
+ RemoveMountTableEntryRequest request) throws IOException;
+
+ /**
+ * List all mount table entries present at or below the path. Fetches from the
+ * state store.
+ *
+ * @param request Fully populated request object.
+ *
+ * @return List of all mount table entries under the path. Zero-length list if
+ * none are found.
+ * @throws IOException Throws exception if the data store cannot be queried.
+ */
+ GetMountTableEntriesResponse getMountTableEntries(
+ GetMountTableEntriesRequest request) throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
new file mode 100644
index 0000000..13e3db3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
@@ -0,0 +1,544 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentNavigableMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DFSUtilClient;
+import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
+import org.apache.hadoop.hdfs.server.federation.router.Router;
+import org.apache.hadoop.hdfs.server.federation.store.MountTableStore;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreCache;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreUnavailableException;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Mount table to map between global paths and remote locations. This allows the
+ * {@link org.apache.hadoop.hdfs.server.federation.router.Router Router} to map
+ * the global HDFS view to the remote namespaces. This is similar to
+ * {@link org.apache.hadoop.fs.viewfs.ViewFs ViewFs}.
+ * This is implemented as a tree.
+ */
+public class MountTableResolver
+ implements FileSubclusterResolver, StateStoreCache {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(MountTableResolver.class);
+
+ /** Reference to Router. */
+ private final Router router;
+ /** Reference to the State Store. */
+ private final StateStoreService stateStore;
+ /** Interface to the mount table store. */
+ private MountTableStore mountTableStore;
+
+ /** If the tree has been initialized. */
+ private boolean init = false;
+ /** Path -> Remote HDFS location. */
+ private final TreeMap<String, MountTable> tree = new TreeMap<>();
+ /** Path -> Remote location. */
+ private final ConcurrentNavigableMap<String, PathLocation> locationCache =
+ new ConcurrentSkipListMap<>();
+
+ /** Default nameservice when no mount matches the math. */
+ private String defaultNameService = "";
+
+ /** Synchronization for both the tree and the cache. */
+ private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ private final Lock readLock = readWriteLock.readLock();
+ private final Lock writeLock = readWriteLock.writeLock();
+
+
+ @VisibleForTesting
+ public MountTableResolver(Configuration conf) {
+ this(conf, (StateStoreService)null);
+ }
+
+ public MountTableResolver(Configuration conf, Router routerService) {
+ this.router = routerService;
+ if (this.router != null) {
+ this.stateStore = this.router.getStateStore();
+ } else {
+ this.stateStore = null;
+ }
+
+ registerCacheExternal();
+ initDefaultNameService(conf);
+ }
+
+ public MountTableResolver(Configuration conf, StateStoreService store) {
+ this.router = null;
+ this.stateStore = store;
+
+ registerCacheExternal();
+ initDefaultNameService(conf);
+ }
+
+ /**
+ * Request cache updates from the State Store for this resolver.
+ */
+ private void registerCacheExternal() {
+ if (this.stateStore != null) {
+ this.stateStore.registerCacheExternal(this);
+ }
+ }
+
+ /**
+ * Nameservice for APIs that cannot be resolved to a specific one.
+ *
+ * @param conf Configuration for this resolver.
+ */
+ private void initDefaultNameService(Configuration conf) {
+ try {
+ this.defaultNameService = conf.get(
+ DFS_ROUTER_DEFAULT_NAMESERVICE,
+ DFSUtil.getNamenodeNameServiceId(conf));
+ } catch (HadoopIllegalArgumentException e) {
+ LOG.error("Cannot find default name service, setting it to the first");
+ Collection<String> nsIds = DFSUtilClient.getNameServiceIds(conf);
+ this.defaultNameService = nsIds.iterator().next();
+ LOG.info("Default name service: {}", this.defaultNameService);
+ }
+ }
+
+ /**
+ * Get a reference for the Router for this resolver.
+ *
+ * @return Router for this resolver.
+ */
+ protected Router getRouter() {
+ return this.router;
+ }
+
+ /**
+ * Get the mount table store for this resolver.
+ *
+ * @return Mount table store.
+ * @throws IOException If it cannot connect to the State Store.
+ */
+ protected MountTableStore getMountTableStore() throws IOException {
+ if (this.mountTableStore == null) {
+ this.mountTableStore = this.stateStore.getRegisteredRecordStore(
+ MountTableStore.class);
+ if (this.mountTableStore == null) {
+ throw new IOException("State Store does not have an interface for " +
+ MountTableStore.class);
+ }
+ }
+ return this.mountTableStore;
+ }
+
+ /**
+ * Add a mount entry to the table.
+ *
+ * @param entry The mount table record to add from the state store.
+ */
+ public void addEntry(final MountTable entry) {
+ writeLock.lock();
+ try {
+ String srcPath = entry.getSourcePath();
+ this.tree.put(srcPath, entry);
+ invalidateLocationCache(srcPath);
+ } finally {
+ writeLock.unlock();
+ }
+ this.init = true;
+ }
+
+ /**
+ * Remove a mount table entry.
+ *
+ * @param srcPath Source path for the entry to remove.
+ */
+ public void removeEntry(final String srcPath) {
+ writeLock.lock();
+ try {
+ this.tree.remove(srcPath);
+ invalidateLocationCache(srcPath);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /**
+ * Invalidates all cache entries below this path. It requires the write lock.
+ *
+ * @param src Source path.
+ */
+ private void invalidateLocationCache(final String path) {
+ if (locationCache.isEmpty()) {
+ return;
+ }
+ // Determine next lexicographic entry after source path
+ String nextSrc = path + Character.MAX_VALUE;
+ ConcurrentNavigableMap<String, PathLocation> subMap =
+ locationCache.subMap(path, nextSrc);
+ for (final String key : subMap.keySet()) {
+ locationCache.remove(key);
+ }
+ }
+
+ /**
+ * Updates the mount path tree with a new set of mount table entries. It also
+ * updates the needed caches.
+ *
+ * @param entries Full set of mount table entries to update.
+ */
+ @VisibleForTesting
+ public void refreshEntries(final Collection<MountTable> entries) {
+ // The tree read/write must be atomic
+ writeLock.lock();
+ try {
+ // New entries
+ Map<String, MountTable> newEntries = new ConcurrentHashMap<>();
+ for (MountTable entry : entries) {
+ String srcPath = entry.getSourcePath();
+ newEntries.put(srcPath, entry);
+ }
+
+ // Old entries (reversed to sort from the leaves to the root)
+ Set<String> oldEntries = new TreeSet<>(Collections.reverseOrder());
+ for (MountTable entry : getTreeValues("/")) {
+ String srcPath = entry.getSourcePath();
+ oldEntries.add(srcPath);
+ }
+
+ // Entries that need to be removed
+ for (String srcPath : oldEntries) {
+ if (!newEntries.containsKey(srcPath)) {
+ this.tree.remove(srcPath);
+ invalidateLocationCache(srcPath);
+ LOG.info("Removed stale mount point {} from resolver", srcPath);
+ }
+ }
+
+ // Entries that need to be added
+ for (MountTable entry : entries) {
+ String srcPath = entry.getSourcePath();
+ if (!oldEntries.contains(srcPath)) {
+ // Add node, it does not exist
+ this.tree.put(srcPath, entry);
+ LOG.info("Added new mount point {} to resolver", srcPath);
+ } else {
+ // Node exists, check for updates
+ MountTable existingEntry = this.tree.get(srcPath);
+ if (existingEntry != null && !existingEntry.equals(entry)) {
+ // Entry has changed
+ invalidateLocationCache(srcPath);
+ LOG.info("Updated mount point {} in resolver");
+ }
+ }
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ this.init = true;
+ }
+
+ /**
+ * Replaces the current in-memory cached of the mount table with a new
+ * version fetched from the data store.
+ */
+ @Override
+ public boolean loadCache(boolean force) {
+ try {
+ // Our cache depends on the store, update it first
+ MountTableStore mountTable = this.getMountTableStore();
+ mountTable.loadCache(force);
+
+ GetMountTableEntriesRequest request =
+ GetMountTableEntriesRequest.newInstance("/");
+ GetMountTableEntriesResponse response =
+ mountTable.getMountTableEntries(request);
+ List<MountTable> records = response.getEntries();
+ refreshEntries(records);
+ } catch (IOException e) {
+ LOG.error("Cannot fetch mount table entries from State Store", e);
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Clears all data.
+ */
+ public void clear() {
+ LOG.info("Clearing all mount location caches");
+ writeLock.lock();
+ try {
+ this.locationCache.clear();
+ this.tree.clear();
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public PathLocation getDestinationForPath(final String path)
+ throws IOException {
+ verifyMountTable();
+ readLock.lock();
+ try {
+ return this.locationCache.computeIfAbsent(
+ path, this::lookupLocation);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ /**
+ * Build the path location to insert into the cache atomically. It must hold
+ * the read lock.
+ * @param path Path to check/insert.
+ * @return New remote location.
+ */
+ public PathLocation lookupLocation(final String path) {
+ PathLocation ret = null;
+ MountTable entry = findDeepest(path);
+ if (entry != null) {
+ ret = buildLocation(path, entry);
+ } else {
+ // Not found, use default location
+ RemoteLocation remoteLocation =
+ new RemoteLocation(defaultNameService, path);
+ List<RemoteLocation> locations =
+ Collections.singletonList(remoteLocation);
+ ret = new PathLocation(null, locations);
+ }
+ return ret;
+ }
+
+ /**
+ * Get the mount table entry for a path.
+ *
+ * @param path Path to look for.
+ * @return Mount table entry the path belongs.
+ * @throws IOException If the State Store could not be reached.
+ */
+ public MountTable getMountPoint(final String path) throws IOException {
+ verifyMountTable();
+ return findDeepest(path);
+ }
+
+ @Override
+ public List<String> getMountPoints(final String path) throws IOException {
+ verifyMountTable();
+
+ Set<String> children = new TreeSet<>();
+ readLock.lock();
+ try {
+ String from = path;
+ String to = path + Character.MAX_VALUE;
+ SortedMap<String, MountTable> subMap = this.tree.subMap(from, to);
+
+ boolean exists = false;
+ for (String subPath : subMap.keySet()) {
+ String child = subPath;
+
+ // Special case for /
+ if (!path.equals(Path.SEPARATOR)) {
+ // Get the children
+ int ini = path.length();
+ child = subPath.substring(ini);
+ }
+
+ if (child.isEmpty()) {
+ // This is a mount point but without children
+ exists = true;
+ } else if (child.startsWith(Path.SEPARATOR)) {
+ // This is a mount point with children
+ exists = true;
+ child = child.substring(1);
+
+ // We only return immediate children
+ int fin = child.indexOf(Path.SEPARATOR);
+ if (fin > -1) {
+ child = child.substring(0, fin);
+ }
+ if (!child.isEmpty()) {
+ children.add(child);
+ }
+ }
+ }
+ if (!exists) {
+ return null;
+ }
+ return new LinkedList<>(children);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ /**
+ * Get all the mount records at or beneath a given path.
+ * @param path Path to get the mount points from.
+ * @return List of mount table records under the path or null if the path is
+ * not found.
+ * @throws IOException If it's not connected to the State Store.
+ */
+ public List<MountTable> getMounts(final String path) throws IOException {
+ verifyMountTable();
+
+ return getTreeValues(path, false);
+ }
+
+ /**
+ * Check if the Mount Table is ready to be used.
+ * @throws StateStoreUnavailableException If it cannot connect to the store.
+ */
+ private void verifyMountTable() throws StateStoreUnavailableException {
+ if (!this.init) {
+ throw new StateStoreUnavailableException("Mount Table not initialized");
+ }
+ }
+
+ @Override
+ public String toString() {
+ readLock.lock();
+ try {
+ return this.tree.toString();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ /**
+ * Build a location for this result beneath the discovered mount point.
+ *
+ * @param result Tree node search result.
+ * @return PathLocation containing the namespace, local path.
+ */
+ private static PathLocation buildLocation(
+ final String path, final MountTable entry) {
+
+ String srcPath = entry.getSourcePath();
+ if (!path.startsWith(srcPath)) {
+ LOG.error("Cannot build location, {} not a child of {}", path, srcPath);
+ return null;
+ }
+ String remainingPath = path.substring(srcPath.length());
+ if (remainingPath.startsWith(Path.SEPARATOR)) {
+ remainingPath = remainingPath.substring(1);
+ }
+
+ List<RemoteLocation> locations = new LinkedList<>();
+ for (RemoteLocation oneDst : entry.getDestinations()) {
+ String nsId = oneDst.getNameserviceId();
+ String dest = oneDst.getDest();
+ String newPath = dest;
+ if (!newPath.endsWith(Path.SEPARATOR)) {
+ newPath += Path.SEPARATOR;
+ }
+ newPath += remainingPath;
+ RemoteLocation remoteLocation = new RemoteLocation(nsId, newPath);
+ locations.add(remoteLocation);
+ }
+ DestinationOrder order = entry.getDestOrder();
+ return new PathLocation(srcPath, locations, order);
+ }
+
+ @Override
+ public String getDefaultNamespace() {
+ return this.defaultNameService;
+ }
+
+ /**
+ * Find the deepest mount point for a path.
+ * @param path Path to look for.
+ * @return Mount table entry.
+ */
+ private MountTable findDeepest(final String path) {
+ readLock.lock();
+ try {
+ Entry<String, MountTable> entry = this.tree.floorEntry(path);
+ while (entry != null && !path.startsWith(entry.getKey())) {
+ entry = this.tree.lowerEntry(entry.getKey());
+ }
+ if (entry == null) {
+ return null;
+ }
+ return entry.getValue();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ /**
+ * Get the mount table entries under a path.
+ * @param path Path to search from.
+ * @return Mount Table entries.
+ */
+ private List<MountTable> getTreeValues(final String path) {
+ return getTreeValues(path, false);
+ }
+
+ /**
+ * Get the mount table entries under a path.
+ * @param path Path to search from.
+ * @param reverse If the order should be reversed.
+ * @return Mount Table entries.
+ */
+ private List<MountTable> getTreeValues(final String path, boolean reverse) {
+ LinkedList<MountTable> ret = new LinkedList<>();
+ readLock.lock();
+ try {
+ String from = path;
+ String to = path + Character.MAX_VALUE;
+ SortedMap<String, MountTable> subMap = this.tree.subMap(from, to);
+ for (MountTable entry : subMap.values()) {
+ if (!reverse) {
+ ret.add(entry);
+ } else {
+ ret.addFirst(entry);
+ }
+ }
+ } finally {
+ readLock.unlock();
+ }
+ return ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/PathLocation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/PathLocation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/PathLocation.java
index d90565c..945d81d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/PathLocation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/PathLocation.java
@@ -23,21 +23,27 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Set;
+import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* A map of the properties and target destinations (name space + path) for
- * a path in the global/federated namespace.
+ * a path in the global/federated name space.
* This data is generated from the @see MountTable records.
*/
public class PathLocation {
+ private static final Logger LOG = LoggerFactory.getLogger(PathLocation.class);
+
+
/** Source path in global namespace. */
private final String sourcePath;
- /** Remote paths in the target namespaces. */
+ /** Remote paths in the target name spaces. */
private final List<RemoteLocation> destinations;
-
- /** List of name spaces present. */
- private final Set<String> namespaces;
+ /** Order for the destinations. */
+ private final DestinationOrder destOrder;
/**
@@ -45,14 +51,23 @@ public class PathLocation {
*
* @param source Source path in the global name space.
* @param dest Destinations of the mount table entry.
- * @param namespaces Unique identifier representing the combination of
- * name spaces present in the destination list.
+ * @param order Order of the locations.
*/
public PathLocation(
- String source, List<RemoteLocation> dest, Set<String> nss) {
+ String source, List<RemoteLocation> dest, DestinationOrder order) {
this.sourcePath = source;
- this.destinations = dest;
- this.namespaces = nss;
+ this.destinations = Collections.unmodifiableList(dest);
+ this.destOrder = order;
+ }
+
+ /**
+ * Create a new PathLocation with default HASH order.
+ *
+ * @param source Source path in the global name space.
+ * @param dest Destinations of the mount table entry.
+ */
+ public PathLocation(String source, List<RemoteLocation> dest) {
+ this(source, dest, DestinationOrder.HASH);
}
/**
@@ -60,10 +75,55 @@ public class PathLocation {
*
* @param other Other path location to copy from.
*/
- public PathLocation(PathLocation other) {
+ public PathLocation(final PathLocation other) {
this.sourcePath = other.sourcePath;
- this.destinations = new LinkedList<RemoteLocation>(other.destinations);
- this.namespaces = new HashSet<String>(other.namespaces);
+ this.destinations = Collections.unmodifiableList(other.destinations);
+ this.destOrder = other.destOrder;
+ }
+
+ /**
+ * Create a path location from another path with the destinations sorted.
+ *
+ * @param other Other path location to copy from.
+ * @param firstNsId Identifier of the namespace to place first.
+ */
+ public PathLocation(PathLocation other, String firstNsId) {
+ this.sourcePath = other.sourcePath;
+ this.destOrder = other.destOrder;
+ this.destinations = orderedNamespaces(other.destinations, firstNsId);
+ }
+
+ /**
+ * Prioritize a location/destination by its name space/nameserviceId.
+ * This destination might be used by other threads, so the source is not
+ * modifiable.
+ *
+ * @param original List of destinations to order.
+ * @param nsId The name space/nameserviceID to prioritize.
+ * @return Prioritized list of detinations that cannot be modified.
+ */
+ private static List<RemoteLocation> orderedNamespaces(
+ final List<RemoteLocation> original, final String nsId) {
+ if (original.size() <= 1) {
+ return original;
+ }
+
+ LinkedList<RemoteLocation> newDestinations = new LinkedList<>();
+ boolean found = false;
+ for (RemoteLocation dest : original) {
+ if (dest.getNameserviceId().equals(nsId)) {
+ found = true;
+ newDestinations.addFirst(dest);
+ } else {
+ newDestinations.add(dest);
+ }
+ }
+
+ if (!found) {
+ LOG.debug("Cannot find location with namespace {} in {}",
+ nsId, original);
+ }
+ return Collections.unmodifiableList(newDestinations);
}
/**
@@ -76,16 +136,37 @@ public class PathLocation {
}
/**
- * Get the list of subclusters defined for the destinations.
+ * Get the subclusters defined for the destinations.
+ *
+ * @return Set containing the subclusters.
*/
public Set<String> getNamespaces() {
- return Collections.unmodifiableSet(this.namespaces);
+ Set<String> namespaces = new HashSet<>();
+ List<RemoteLocation> locations = this.getDestinations();
+ for (RemoteLocation location : locations) {
+ String nsId = location.getNameserviceId();
+ namespaces.add(nsId);
+ }
+ return namespaces;
}
@Override
public String toString() {
- RemoteLocation loc = getDefaultLocation();
- return loc.getNameserviceId() + "->" + loc.getDest();
+ StringBuilder sb = new StringBuilder();
+ for (RemoteLocation destination : this.destinations) {
+ String nsId = destination.getNameserviceId();
+ String path = destination.getDest();
+ if (sb.length() > 0) {
+ sb.append(",");
+ }
+ sb.append(nsId + "->" + path);
+ }
+ if (this.destinations.size() > 1) {
+ sb.append(" [");
+ sb.append(this.destOrder.toString());
+ sb.append("]");
+ }
+ return sb.toString();
}
/**
@@ -108,6 +189,15 @@ public class PathLocation {
}
/**
+ * Get the order for the destinations.
+ *
+ * @return Order for the destinations.
+ */
+ public DestinationOrder getDestinationOrder() {
+ return this.destOrder;
+ }
+
+ /**
* Get the default or highest priority location.
*
* @return The default location.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/DestinationOrder.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/DestinationOrder.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/DestinationOrder.java
new file mode 100644
index 0000000..4bccf10
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/DestinationOrder.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver.order;
+
+/**
+ * Order of the destinations when we have multiple of them. When the resolver
+ * of files to subclusters (FileSubclusterResolver) has multiple destinations,
+ * this determines which location should be checked first.
+ */
+public enum DestinationOrder {
+ HASH, // Follow consistent hashing
+ LOCAL, // Local first
+ RANDOM // Random order
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/package-info.java
new file mode 100644
index 0000000..f90152f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/package-info.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A federated location can be resolved to multiple subclusters. This package
+ * takes care of the order in which this multiple destinations should be used.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+
+package org.apache.hadoop.hdfs.server.federation.resolver.order;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
index 78c473a..99af2d8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
@@ -136,65 +136,19 @@ public final class FederationUtil {
}
/**
- * Create an instance of an interface with a constructor using a state store
- * constructor.
- *
- * @param conf Configuration
- * @param context Context object to pass to the instance.
- * @param contextType Type of the context passed to the constructor.
- * @param configurationKeyName Configuration key to retrieve the class to load
- * @param defaultClassName Default class to load if the configuration key is
- * not set
- * @param clazz Class/interface that must be implemented by the instance.
- * @return New instance of the specified class that implements the desired
- * interface and a single parameter constructor containing a
- * StateStore reference.
- */
- private static <T, R> T newInstance(final Configuration conf,
- final R context, final Class<R> contextClass,
- final String configKeyName, final String defaultClassName,
- final Class<T> clazz) {
-
- String className = conf.get(configKeyName, defaultClassName);
- try {
- Class<?> instance = conf.getClassByName(className);
- if (clazz.isAssignableFrom(instance)) {
- if (contextClass == null) {
- // Default constructor if no context
- @SuppressWarnings("unchecked")
- Constructor<T> constructor =
- (Constructor<T>) instance.getConstructor();
- return constructor.newInstance();
- } else {
- // Constructor with context
- @SuppressWarnings("unchecked")
- Constructor<T> constructor = (Constructor<T>) instance.getConstructor(
- Configuration.class, contextClass);
- return constructor.newInstance(conf, context);
- }
- } else {
- throw new RuntimeException("Class " + className + " not instance of "
- + clazz.getCanonicalName());
- }
- } catch (ReflectiveOperationException e) {
- LOG.error("Could not instantiate: " + className, e);
- return null;
- }
- }
-
- /**
* Creates an instance of a FileSubclusterResolver from the configuration.
*
* @param conf Configuration that defines the file resolver class.
- * @param obj Context object passed to class constructor.
- * @return FileSubclusterResolver
+ * @param router Router service.
+ * @return New file subcluster resolver.
*/
public static FileSubclusterResolver newFileSubclusterResolver(
- Configuration conf, StateStoreService stateStore) {
- return newInstance(conf, stateStore, StateStoreService.class,
+ Configuration conf, Router router) {
+ Class<? extends FileSubclusterResolver> clazz = conf.getClass(
DFSConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
DFSConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS_DEFAULT,
FileSubclusterResolver.class);
+ return newInstance(conf, router, Router.class, clazz);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
index cfddf20..213a58f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
@@ -124,8 +124,7 @@ public class Router extends CompositeService {
}
// Lookup interface to map between the global and subcluster name spaces
- this.subclusterResolver = newFileSubclusterResolver(
- this.conf, this.stateStore);
+ this.subclusterResolver = newFileSubclusterResolver(this.conf, this);
if (this.subclusterResolver == null) {
throw new IOException("Cannot find subcluster resolver");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MountTableStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MountTableStore.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MountTableStore.java
new file mode 100644
index 0000000..b439659
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MountTableStore.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+
+/**
+ * Management API for the HDFS mount table information stored in
+ * {@link org.apache.hadoop.hdfs.server.federation.store.records.MountTable
+ * MountTable} records. The mount table contains entries that map a particular
+ * global namespace path one or more HDFS nameservices (NN) + target path. It is
+ * possible to map mount locations for root folders, directories or individual
+ * files.
+ * <p>
+ * Once fetched from the
+ * {@link org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver
+ * StateStoreDriver}, MountTable records are cached in a tree for faster access.
+ * Each path in the global namespace is mapped to a nameserivce ID and local
+ * path upon request. The cache is periodically updated by the @{link
+ * StateStoreCacheUpdateService}.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public abstract class MountTableStore extends CachedRecordStore<MountTable>
+ implements MountTableManager {
+
+ public MountTableStore(StateStoreDriver driver) {
+ super(MountTable.class, driver);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
index 73f607f..3aa3ffd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
import org.apache.hadoop.hdfs.server.federation.store.impl.MembershipStoreImpl;
+import org.apache.hadoop.hdfs.server.federation.store.impl.MountTableStoreImpl;
import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
import org.apache.hadoop.service.CompositeService;
@@ -136,6 +137,7 @@ public class StateStoreService extends CompositeService {
// Add supported record stores
addRecordStore(MembershipStoreImpl.class);
+ addRecordStore(MountTableStoreImpl.class);
// Check the connection to the State Store periodically
this.monitorService = new StateStoreConnectionMonitorService(this);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MountTableStoreImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MountTableStoreImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MountTableStoreImpl.java
new file mode 100644
index 0000000..e6affb2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MountTableStoreImpl.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.impl;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.server.federation.store.MountTableStore;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.hdfs.server.federation.store.records.Query;
+import org.apache.hadoop.util.Time;
+
+/**
+ * Implementation of the {@link MountTableStore} state store API.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class MountTableStoreImpl extends MountTableStore {
+
+ public MountTableStoreImpl(StateStoreDriver driver) {
+ super(driver);
+ }
+
+ @Override
+ public AddMountTableEntryResponse addMountTableEntry(
+ AddMountTableEntryRequest request) throws IOException {
+ boolean status = getDriver().put(request.getEntry(), false, true);
+ AddMountTableEntryResponse response =
+ AddMountTableEntryResponse.newInstance();
+ response.setStatus(status);
+ return response;
+ }
+
+ @Override
+ public UpdateMountTableEntryResponse updateMountTableEntry(
+ UpdateMountTableEntryRequest request) throws IOException {
+ MountTable entry = request.getEntry();
+ boolean status = getDriver().put(entry, true, true);
+ UpdateMountTableEntryResponse response =
+ UpdateMountTableEntryResponse.newInstance();
+ response.setStatus(status);
+ return response;
+ }
+
+ @Override
+ public RemoveMountTableEntryResponse removeMountTableEntry(
+ RemoveMountTableEntryRequest request) throws IOException {
+ final String srcPath = request.getSrcPath();
+ final MountTable partial = MountTable.newInstance();
+ partial.setSourcePath(srcPath);
+ final Query<MountTable> query = new Query<>(partial);
+ int removedRecords = getDriver().remove(getRecordClass(), query);
+ boolean status = (removedRecords == 1);
+ RemoveMountTableEntryResponse response =
+ RemoveMountTableEntryResponse.newInstance();
+ response.setStatus(status);
+ return response;
+ }
+
+ @Override
+ public GetMountTableEntriesResponse getMountTableEntries(
+ GetMountTableEntriesRequest request) throws IOException {
+
+ // Get all values from the cache
+ List<MountTable> records = getCachedRecords();
+
+ // Sort and filter
+ Collections.sort(records);
+ String reqSrcPath = request.getSrcPath();
+ if (reqSrcPath != null && !reqSrcPath.isEmpty()) {
+ // Return only entries beneath this path
+ Iterator<MountTable> it = records.iterator();
+ while (it.hasNext()) {
+ MountTable record = it.next();
+ String srcPath = record.getSourcePath();
+ if (!srcPath.startsWith(reqSrcPath)) {
+ it.remove();
+ }
+ }
+ }
+
+ GetMountTableEntriesResponse response =
+ GetMountTableEntriesResponse.newInstance();
+ response.setEntries(records);
+ response.setTimestamp(Time.now());
+ return response;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntryRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntryRequest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntryRequest.java
new file mode 100644
index 0000000..2d9f102
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntryRequest.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+
+/**
+ * API request for adding a mount table entry to the state store.
+ */
+public abstract class AddMountTableEntryRequest {
+
+ public static AddMountTableEntryRequest newInstance() {
+ return StateStoreSerializer.newRecord(AddMountTableEntryRequest.class);
+ }
+
+ public static AddMountTableEntryRequest newInstance(MountTable newEntry) {
+ AddMountTableEntryRequest request = newInstance();
+ request.setEntry(newEntry);
+ return request;
+ }
+
+ @Public
+ @Unstable
+ public abstract MountTable getEntry();
+
+ @Public
+ @Unstable
+ public abstract void setEntry(MountTable mount);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntryResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntryResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntryResponse.java
new file mode 100644
index 0000000..9bc7f92
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntryResponse.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API response for adding a mount table entry to the state store.
+ */
+public abstract class AddMountTableEntryResponse {
+
+ public static AddMountTableEntryResponse newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(AddMountTableEntryResponse.class);
+ }
+
+ @Public
+ @Unstable
+ public abstract boolean getStatus();
+
+ @Public
+ @Unstable
+ public abstract void setStatus(boolean result);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetMountTableEntriesRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetMountTableEntriesRequest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetMountTableEntriesRequest.java
new file mode 100644
index 0000000..cd6c278
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetMountTableEntriesRequest.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API request for listing mount table entries present in the state store.
+ */
+public abstract class GetMountTableEntriesRequest {
+
+ public static GetMountTableEntriesRequest newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(GetMountTableEntriesRequest.class);
+ }
+
+ public static GetMountTableEntriesRequest newInstance(String srcPath)
+ throws IOException {
+ GetMountTableEntriesRequest request = newInstance();
+ request.setSrcPath(srcPath);
+ return request;
+ }
+
+ @Public
+ @Unstable
+ public abstract String getSrcPath();
+
+ @Public
+ @Unstable
+ public abstract void setSrcPath(String path);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetMountTableEntriesResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetMountTableEntriesResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetMountTableEntriesResponse.java
new file mode 100644
index 0000000..cebc3f6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetMountTableEntriesResponse.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+
+/**
+ * API response for listing mount table entries present in the state store.
+ */
+public abstract class GetMountTableEntriesResponse {
+
+ public static GetMountTableEntriesResponse newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(GetMountTableEntriesResponse.class);
+ }
+
+ @Public
+ @Unstable
+ public abstract List<MountTable> getEntries() throws IOException;
+
+ @Public
+ @Unstable
+ public abstract void setEntries(List<MountTable> entries)
+ throws IOException;
+
+ @Public
+ @Unstable
+ public abstract long getTimestamp();
+
+ @Public
+ @Unstable
+ public abstract void setTimestamp(long time);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntryRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntryRequest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntryRequest.java
new file mode 100644
index 0000000..642ee0d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntryRequest.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API request for removing a mount table path present in the state store.
+ */
+public abstract class RemoveMountTableEntryRequest {
+
+ public static RemoveMountTableEntryRequest newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(RemoveMountTableEntryRequest.class);
+ }
+
+ public static RemoveMountTableEntryRequest newInstance(String path)
+ throws IOException {
+ RemoveMountTableEntryRequest request = newInstance();
+ request.setSrcPath(path);
+ return request;
+ }
+
+ @Public
+ @Unstable
+ public abstract String getSrcPath();
+
+ @Public
+ @Unstable
+ public abstract void setSrcPath(String path);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntryResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntryResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntryResponse.java
new file mode 100644
index 0000000..70f117d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntryResponse.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API response for removing a mount table path present in the state store.
+ */
+public abstract class RemoveMountTableEntryResponse {
+
+ public static RemoveMountTableEntryResponse newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(RemoveMountTableEntryResponse.class);
+ }
+
+ @Public
+ @Unstable
+ public abstract boolean getStatus();
+
+ @Public
+ @Unstable
+ public abstract void setStatus(boolean result);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateMountTableEntryRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateMountTableEntryRequest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateMountTableEntryRequest.java
new file mode 100644
index 0000000..afd5128
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateMountTableEntryRequest.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+
+/**
+ * API request for updating the destination of an existing mount point in the
+ * state store.
+ */
+public abstract class UpdateMountTableEntryRequest {
+
+ public static UpdateMountTableEntryRequest newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(UpdateMountTableEntryRequest.class);
+ }
+
+ public static UpdateMountTableEntryRequest newInstance(MountTable entry)
+ throws IOException {
+ UpdateMountTableEntryRequest request = newInstance();
+ request.setEntry(entry);
+ return request;
+ }
+
+ @Public
+ @Unstable
+ public abstract MountTable getEntry() throws IOException;
+
+ @Public
+ @Unstable
+ public abstract void setEntry(MountTable mount) throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateMountTableEntryResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateMountTableEntryResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateMountTableEntryResponse.java
new file mode 100644
index 0000000..7097e10
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateMountTableEntryResponse.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API response for updating the destination of an existing mount point in the
+ * state store.
+ */
+public abstract class UpdateMountTableEntryResponse {
+
+ public static UpdateMountTableEntryResponse newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(UpdateMountTableEntryResponse.class);
+ }
+
+ @Public
+ @Unstable
+ public abstract boolean getStatus();
+
+ @Public
+ @Unstable
+ public abstract void setStatus(boolean result);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntryRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntryRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntryRequestPBImpl.java
new file mode 100644
index 0000000..35455d2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntryRequestPBImpl.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryRequestProtoOrBuilder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProto;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.MountTablePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * AddMountTableEntryRequest.
+ */
+public class AddMountTableEntryRequestPBImpl
+ extends AddMountTableEntryRequest implements PBRecord {
+
+ private FederationProtocolPBTranslator<AddMountTableEntryRequestProto,
+ AddMountTableEntryRequestProto.Builder,
+ AddMountTableEntryRequestProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<AddMountTableEntryRequestProto,
+ AddMountTableEntryRequestProto.Builder,
+ AddMountTableEntryRequestProtoOrBuilder>(
+ AddMountTableEntryRequestProto.class);
+
+ public AddMountTableEntryRequestPBImpl() {
+ }
+
+ public AddMountTableEntryRequestPBImpl(AddMountTableEntryRequestProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public AddMountTableEntryRequestProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public MountTable getEntry() {
+ MountTableRecordProto entryProto =
+ this.translator.getProtoOrBuilder().getEntry();
+ return new MountTablePBImpl(entryProto);
+ }
+
+ @Override
+ public void setEntry(MountTable mount) {
+ if (mount instanceof MountTablePBImpl) {
+ MountTablePBImpl mountPB = (MountTablePBImpl)mount;
+ MountTableRecordProto mountProto = mountPB.getProto();
+ translator.getBuilder().setEntry(mountProto);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntryResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntryResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntryResponsePBImpl.java
new file mode 100644
index 0000000..c1d9a65
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntryResponsePBImpl.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryResponseProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * AddMountTableEntryResponse.
+ */
+public class AddMountTableEntryResponsePBImpl
+ extends AddMountTableEntryResponse implements PBRecord {
+
+ private FederationProtocolPBTranslator<AddMountTableEntryResponseProto,
+ AddMountTableEntryResponseProto.Builder,
+ AddMountTableEntryResponseProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<AddMountTableEntryResponseProto,
+ AddMountTableEntryResponseProto.Builder,
+ AddMountTableEntryResponseProtoOrBuilder>(
+ AddMountTableEntryResponseProto.class);
+
+ public AddMountTableEntryResponsePBImpl() {
+ }
+
+ public AddMountTableEntryResponsePBImpl(
+ AddMountTableEntryResponseProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public AddMountTableEntryResponseProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public boolean getStatus() {
+ return this.translator.getProtoOrBuilder().getStatus();
+ }
+
+ @Override
+ public void setStatus(boolean result) {
+ this.translator.getBuilder().setStatus(result);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetMountTableEntriesRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetMountTableEntriesRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetMountTableEntriesRequestPBImpl.java
new file mode 100644
index 0000000..3e0d1a6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetMountTableEntriesRequestPBImpl.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesRequestProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * GetMountTableEntriesRequest.
+ */
+public class GetMountTableEntriesRequestPBImpl
+ extends GetMountTableEntriesRequest implements PBRecord {
+
+ private FederationProtocolPBTranslator<GetMountTableEntriesRequestProto,
+ GetMountTableEntriesRequestProto.Builder,
+ GetMountTableEntriesRequestProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<GetMountTableEntriesRequestProto,
+ GetMountTableEntriesRequestProto.Builder,
+ GetMountTableEntriesRequestProtoOrBuilder>(
+ GetMountTableEntriesRequestProto.class);
+
+ public GetMountTableEntriesRequestPBImpl() {
+ }
+
+ public GetMountTableEntriesRequestPBImpl(
+ GetMountTableEntriesRequestProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public GetMountTableEntriesRequestProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public String getSrcPath() {
+ return this.translator.getProtoOrBuilder().getSrcPath();
+ }
+
+ @Override
+ public void setSrcPath(String path) {
+ this.translator.getBuilder().setSrcPath(path);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetMountTableEntriesResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetMountTableEntriesResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetMountTableEntriesResponsePBImpl.java
new file mode 100644
index 0000000..9d64bc9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetMountTableEntriesResponsePBImpl.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesResponseProtoOrBuilder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProto;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.MountTablePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * GetMountTableEntriesResponse.
+ */
+public class GetMountTableEntriesResponsePBImpl
+ extends GetMountTableEntriesResponse implements PBRecord {
+
+ private FederationProtocolPBTranslator<GetMountTableEntriesResponseProto,
+ GetMountTableEntriesResponseProto.Builder,
+ GetMountTableEntriesResponseProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<GetMountTableEntriesResponseProto,
+ GetMountTableEntriesResponseProto.Builder,
+ GetMountTableEntriesResponseProtoOrBuilder>(
+ GetMountTableEntriesResponseProto.class);
+
+ public GetMountTableEntriesResponsePBImpl() {
+ }
+
+ public GetMountTableEntriesResponsePBImpl(
+ GetMountTableEntriesResponseProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public GetMountTableEntriesResponseProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public List<MountTable> getEntries() throws IOException {
+ List<MountTableRecordProto> entries =
+ this.translator.getProtoOrBuilder().getEntriesList();
+ List<MountTable> ret = new ArrayList<MountTable>();
+ for (MountTableRecordProto entry : entries) {
+ MountTable record = new MountTablePBImpl(entry);
+ ret.add(record);
+ }
+ return ret;
+ }
+
+ @Override
+ public void setEntries(List<MountTable> records) throws IOException {
+ this.translator.getBuilder().clearEntries();
+ for (MountTable entry : records) {
+ if (entry instanceof MountTablePBImpl) {
+ MountTablePBImpl entryPB = (MountTablePBImpl)entry;
+ this.translator.getBuilder().addEntries(entryPB.getProto());
+ }
+ }
+ }
+
+ @Override
+ public long getTimestamp() {
+ return this.translator.getProtoOrBuilder().getTimestamp();
+ }
+
+ @Override
+ public void setTimestamp(long time) {
+ this.translator.getBuilder().setTimestamp(time);
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/40] hadoop git commit: HDFS-10631. Federation State Store
ZooKeeper implementation. Contributed by Jason Kace and Inigo Goiri.
Posted by in...@apache.org.
HDFS-10631. Federation State Store ZooKeeper implementation. Contributed by Jason Kace and Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/504b639b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/504b639b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/504b639b
Branch: refs/heads/HDFS-10467
Commit: 504b639b1f0553c5a464ae18b8ef0b91ed276514
Parents: d4aa309
Author: Inigo Goiri <in...@apache.org>
Authored: Mon Aug 21 11:40:41 2017 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu Sep 7 13:19:52 2017 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 9 +
.../driver/impl/StateStoreSerializableImpl.java | 19 ++
.../driver/impl/StateStoreZooKeeperImpl.java | 298 +++++++++++++++++++
.../store/driver/TestStateStoreDriverBase.java | 2 +-
.../store/driver/TestStateStoreZK.java | 105 +++++++
5 files changed, 432 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/504b639b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index 93216db..d22d6ee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -203,6 +203,15 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-test</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/504b639b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializableImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializableImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializableImpl.java
index e9b3fdf..e2038fa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializableImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializableImpl.java
@@ -30,6 +30,11 @@ import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
*/
public abstract class StateStoreSerializableImpl extends StateStoreBaseImpl {
+ /** Mark for slashes in path names. */
+ protected static final String SLASH_MARK = "0SLASH0";
+ /** Mark for colon in path names. */
+ protected static final String COLON_MARK = "_";
+
/** Default serializer for this driver. */
private StateStoreSerializer serializer;
@@ -74,4 +79,18 @@ public abstract class StateStoreSerializableImpl extends StateStoreBaseImpl {
String data, Class<T> clazz, boolean includeDates) throws IOException {
return serializer.deserialize(data, clazz);
}
+
+ /**
+ * Get the primary key for a record. If we don't want to store in folders, we
+ * need to remove / from the name.
+ *
+ * @param record Record to get the primary key for.
+ * @return Primary key for the record.
+ */
+ protected static String getPrimaryKey(BaseRecord record) {
+ String primaryKey = record.getPrimaryKey();
+ primaryKey = primaryKey.replaceAll("/", SLASH_MARK);
+ primaryKey = primaryKey.replaceAll(":", COLON_MARK);
+ return primaryKey;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/504b639b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreZooKeeperImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreZooKeeperImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreZooKeeperImpl.java
new file mode 100644
index 0000000..ddcd537
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreZooKeeperImpl.java
@@ -0,0 +1,298 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
+
+import static org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils.filterMultiple;
+import static org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils.getRecordName;
+import static org.apache.hadoop.util.curator.ZKCuratorManager.getNodePath;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.Query;
+import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
+import org.apache.hadoop.util.curator.ZKCuratorManager;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * {@link StateStoreDriver} driver implementation that uses ZooKeeper as a
+ * backend.
+ * <p>
+ * The structure of the znodes in the ensemble is:
+ * PARENT_PATH
+ * |--- MOUNT
+ * |--- MEMBERSHIP
+ * |--- REBALANCER
+ * |--- ROUTERS
+ */
+public class StateStoreZooKeeperImpl extends StateStoreSerializableImpl {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StateStoreZooKeeperImpl.class);
+
+
+ /** Configuration keys. */
+ public static final String FEDERATION_STORE_ZK_DRIVER_PREFIX =
+ DFSConfigKeys.FEDERATION_STORE_PREFIX + "driver.zk.";
+ public static final String FEDERATION_STORE_ZK_PARENT_PATH =
+ FEDERATION_STORE_ZK_DRIVER_PREFIX + "parent-path";
+ public static final String FEDERATION_STORE_ZK_PARENT_PATH_DEFAULT =
+ "/hdfs-federation";
+
+
+ /** Directory to store the state store data. */
+ private String baseZNode;
+
+ /** Interface to ZooKeeper. */
+ private ZKCuratorManager zkManager;
+
+
+ @Override
+ public boolean initDriver() {
+ LOG.info("Initializing ZooKeeper connection");
+
+ Configuration conf = getConf();
+ baseZNode = conf.get(
+ FEDERATION_STORE_ZK_PARENT_PATH,
+ FEDERATION_STORE_ZK_PARENT_PATH_DEFAULT);
+ try {
+ this.zkManager = new ZKCuratorManager(conf);
+ this.zkManager.start();
+ } catch (IOException e) {
+ LOG.error("Cannot initialize the ZK connection", e);
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public <T extends BaseRecord> boolean initRecordStorage(
+ String className, Class<T> clazz) {
+ try {
+ String checkPath = getNodePath(baseZNode, className);
+ zkManager.createRootDirRecursively(checkPath);
+ return true;
+ } catch (Exception e) {
+ LOG.error("Cannot initialize ZK node for {}: {}",
+ className, e.getMessage());
+ return false;
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (zkManager != null) {
+ zkManager.close();
+ }
+ }
+
+ @Override
+ public boolean isDriverReady() {
+ return zkManager != null;
+ }
+
+ @Override
+ public <T extends BaseRecord> QueryResult<T> get(Class<T> clazz)
+ throws IOException {
+ return get(clazz, (String)null);
+ }
+
+ @Override
+ public <T extends BaseRecord> QueryResult<T> get(Class<T> clazz, String sub)
+ throws IOException {
+ verifyDriverReady();
+ List<T> ret = new ArrayList<>();
+ String znode = getZNodeForClass(clazz);
+ try {
+ List<String> children = zkManager.getChildren(znode);
+ for (String child : children) {
+ try {
+ String path = getNodePath(znode, child);
+ Stat stat = new Stat();
+ String data = zkManager.getStringData(path, stat);
+ boolean corrupted = false;
+ if (data == null || data.equals("")) {
+ // All records should have data, otherwise this is corrupted
+ corrupted = true;
+ } else {
+ try {
+ T record = createRecord(data, stat, clazz);
+ ret.add(record);
+ } catch (IOException e) {
+ LOG.error("Cannot create record type \"{}\" from \"{}\": {}",
+ clazz.getSimpleName(), data, e.getMessage());
+ corrupted = true;
+ }
+ }
+
+ if (corrupted) {
+ LOG.error("Cannot get data for {} at {}, cleaning corrupted data",
+ child, path);
+ zkManager.delete(path);
+ }
+ } catch (Exception e) {
+ LOG.error("Cannot get data for {}: {}", child, e.getMessage());
+ }
+ }
+ } catch (Exception e) {
+ String msg = "Cannot get children for \"" + znode + "\": " +
+ e.getMessage();
+ LOG.error(msg);
+ throw new IOException(msg);
+ }
+ return new QueryResult<T>(ret, getTime());
+ }
+
+ @Override
+ public <T extends BaseRecord> boolean putAll(
+ List<T> records, boolean update, boolean error) throws IOException {
+ verifyDriverReady();
+ if (records.isEmpty()) {
+ return true;
+ }
+
+ // All records should be the same
+ T record0 = records.get(0);
+ Class<? extends BaseRecord> recordClass = record0.getClass();
+ String znode = getZNodeForClass(recordClass);
+
+ boolean status = true;
+ for (T record : records) {
+ String primaryKey = getPrimaryKey(record);
+ String recordZNode = getNodePath(znode, primaryKey);
+ byte[] data = serialize(record);
+ if (!writeNode(recordZNode, data, update, error)){
+ status = false;
+ }
+ }
+ return status;
+ }
+
+ @Override
+ public <T extends BaseRecord> int remove(
+ Class<T> clazz, Query<T> query) throws IOException {
+ verifyDriverReady();
+ if (query == null) {
+ return 0;
+ }
+
+ // Read the current data
+ List<T> records = null;
+ try {
+ QueryResult<T> result = get(clazz);
+ records = result.getRecords();
+ } catch (IOException ex) {
+ LOG.error("Cannot get existing records", ex);
+ return 0;
+ }
+
+ // Check the records to remove
+ String znode = getZNodeForClass(clazz);
+ List<T> recordsToRemove = filterMultiple(query, records);
+
+ // Remove the records
+ int removed = 0;
+ for (T existingRecord : recordsToRemove) {
+ LOG.info("Removing \"{}\"", existingRecord);
+ try {
+ String primaryKey = getPrimaryKey(existingRecord);
+ String path = getNodePath(znode, primaryKey);
+ if (zkManager.delete(path)) {
+ removed++;
+ } else {
+ LOG.error("Did not remove \"{}\"", existingRecord);
+ }
+ } catch (Exception e) {
+ LOG.error("Cannot remove \"{}\"", existingRecord, e);
+ }
+ }
+ return removed;
+ }
+
+ @Override
+ public <T extends BaseRecord> boolean removeAll(Class<T> clazz)
+ throws IOException {
+ boolean status = true;
+ String znode = getZNodeForClass(clazz);
+ LOG.info("Deleting all children under {}", znode);
+ try {
+ List<String> children = zkManager.getChildren(znode);
+ for (String child : children) {
+ String path = getNodePath(znode, child);
+ LOG.info("Deleting {}", path);
+ zkManager.delete(path);
+ }
+ } catch (Exception e) {
+ LOG.error("Cannot remove {}: {}", znode, e.getMessage());
+ status = false;
+ }
+ return status;
+ }
+
+ private boolean writeNode(
+ String znode, byte[] bytes, boolean update, boolean error) {
+ try {
+ boolean created = zkManager.create(znode);
+ if (!update && !created && error) {
+ LOG.info("Cannot write record \"{}\", it already exists", znode);
+ return false;
+ }
+
+ // Write data
+ zkManager.setData(znode, bytes, -1);
+ return true;
+ } catch (Exception e) {
+ LOG.error("Cannot write record \"{}\": {}", znode, e.getMessage());
+ }
+ return false;
+ }
+
+ /**
+ * Get the ZNode for a class.
+ *
+ * @param clazz Record class to evaluate.
+ * @return The ZNode for the class.
+ */
+ private <T extends BaseRecord> String getZNodeForClass(Class<T> clazz) {
+ String className = getRecordName(clazz);
+ return getNodePath(baseZNode, className);
+ }
+
+ /**
+ * Creates a record from a string returned by ZooKeeper.
+ *
+ * @param source Object from ZooKeeper.
+ * @param clazz The data record type to create.
+ * @return The created record.
+ * @throws IOException
+ */
+ private <T extends BaseRecord> T createRecord(
+ String data, Stat stat, Class<T> clazz) throws IOException {
+ T record = newRecord(data, clazz, false);
+ record.setDateCreated(stat.getCtime());
+ record.setDateModified(stat.getMtime());
+ return record;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/504b639b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
index 8239fb1..65e763b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
@@ -89,7 +89,7 @@ public class TestStateStoreDriverBase {
}
private String generateRandomString() {
- String randomString = "/randomString-" + RANDOM.nextInt();
+ String randomString = "randomString-" + RANDOM.nextInt();
return randomString;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/504b639b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreZK.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreZK.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreZK.java
new file mode 100644
index 0000000..36353ff
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreZK.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver;
+
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.getStateStoreConfiguration;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.RetryNTimes;
+import org.apache.curator.test.TestingServer;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test the ZooKeeper implementation of the State Store driver.
+ */
+public class TestStateStoreZK extends TestStateStoreDriverBase {
+
+ private static TestingServer curatorTestingServer;
+ private static CuratorFramework curatorFramework;
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ curatorTestingServer = new TestingServer();
+ curatorTestingServer.start();
+ String connectString = curatorTestingServer.getConnectString();
+ curatorFramework = CuratorFrameworkFactory.builder()
+ .connectString(connectString)
+ .retryPolicy(new RetryNTimes(100, 100))
+ .build();
+ curatorFramework.start();
+
+ // Create the ZK State Store
+ Configuration conf =
+ getStateStoreConfiguration(StateStoreZooKeeperImpl.class);
+ conf.set(CommonConfigurationKeys.ZK_ADDRESS, connectString);
+ // Disable auto-repair of connection
+ conf.setLong(DFSConfigKeys.FEDERATION_STORE_CONNECTION_TEST_MS,
+ TimeUnit.HOURS.toMillis(1));
+ getStateStore(conf);
+ }
+
+ @AfterClass
+ public static void tearDownCluster() {
+ curatorFramework.close();
+ try {
+ curatorTestingServer.stop();
+ } catch (IOException e) {
+ }
+ }
+
+ @Before
+ public void startup() throws IOException {
+ removeAll(getStateStoreDriver());
+ }
+
+ @Test
+ public void testInsert()
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testInsert(getStateStoreDriver());
+ }
+
+ @Test
+ public void testUpdate()
+ throws IllegalArgumentException, ReflectiveOperationException,
+ IOException, SecurityException {
+ testPut(getStateStoreDriver());
+ }
+
+ @Test
+ public void testDelete()
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testRemove(getStateStoreDriver());
+ }
+
+ @Test
+ public void testFetchErrors()
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testFetchErrors(getStateStoreDriver());
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/40] hadoop git commit: HDFS-10880. Federation Mount Table State
Store internal API. Contributed by Jason Kace and Inigo Goiri.
Posted by in...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntryRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntryRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntryRequestPBImpl.java
new file mode 100644
index 0000000..7f7c998
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntryRequestPBImpl.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryRequestProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * RemoveMountTableEntryRequest.
+ */
+public class RemoveMountTableEntryRequestPBImpl
+ extends RemoveMountTableEntryRequest implements PBRecord {
+
+ private FederationProtocolPBTranslator<RemoveMountTableEntryRequestProto,
+ RemoveMountTableEntryRequestProto.Builder,
+ RemoveMountTableEntryRequestProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<RemoveMountTableEntryRequestProto,
+ RemoveMountTableEntryRequestProto.Builder,
+ RemoveMountTableEntryRequestProtoOrBuilder>(
+ RemoveMountTableEntryRequestProto.class);
+
+ public RemoveMountTableEntryRequestPBImpl() {
+ }
+
+ public RemoveMountTableEntryRequestPBImpl(
+ RemoveMountTableEntryRequestProto proto) {
+ this.setProto(proto);
+ }
+
+ @Override
+ public RemoveMountTableEntryRequestProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public String getSrcPath() {
+ return this.translator.getProtoOrBuilder().getSrcPath();
+ }
+
+ @Override
+ public void setSrcPath(String path) {
+ this.translator.getBuilder().setSrcPath(path);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntryResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntryResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntryResponsePBImpl.java
new file mode 100644
index 0000000..0c943ac
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntryResponsePBImpl.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryResponseProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryResponseProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * RemoveMountTableEntryResponse.
+ */
+public class RemoveMountTableEntryResponsePBImpl
+ extends RemoveMountTableEntryResponse implements PBRecord {
+
+ private FederationProtocolPBTranslator<RemoveMountTableEntryResponseProto,
+ Builder, RemoveMountTableEntryResponseProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<RemoveMountTableEntryResponseProto,
+ RemoveMountTableEntryResponseProto.Builder,
+ RemoveMountTableEntryResponseProtoOrBuilder>(
+ RemoveMountTableEntryResponseProto.class);
+
+ public RemoveMountTableEntryResponsePBImpl() {
+ }
+
+ public RemoveMountTableEntryResponsePBImpl(
+ RemoveMountTableEntryResponseProto proto) {
+ this.setProto(proto);
+ }
+
+ @Override
+ public RemoveMountTableEntryResponseProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public boolean getStatus() {
+ return this.translator.getProtoOrBuilder().getStatus();
+ }
+
+ @Override
+ public void setStatus(boolean result) {
+ this.translator.getBuilder().setStatus(result);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateMountTableEntryRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateMountTableEntryRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateMountTableEntryRequestPBImpl.java
new file mode 100644
index 0000000..621bb3a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateMountTableEntryRequestPBImpl.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryRequestProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.MountTablePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * UpdateMountTableEntryRequest.
+ */
+public class UpdateMountTableEntryRequestPBImpl
+ extends UpdateMountTableEntryRequest implements PBRecord {
+
+ private FederationProtocolPBTranslator<UpdateMountTableEntryRequestProto,
+ UpdateMountTableEntryRequestProto.Builder,
+ UpdateMountTableEntryRequestProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<UpdateMountTableEntryRequestProto,
+ UpdateMountTableEntryRequestProto.Builder,
+ UpdateMountTableEntryRequestProtoOrBuilder>(
+ UpdateMountTableEntryRequestProto.class);
+
+ public UpdateMountTableEntryRequestPBImpl() {
+ }
+
+ public UpdateMountTableEntryRequestPBImpl(
+ UpdateMountTableEntryRequestProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public UpdateMountTableEntryRequestProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public MountTable getEntry() throws IOException {
+ MountTableRecordProto statsProto =
+ this.translator.getProtoOrBuilder().getEntry();
+ MountTable stats = StateStoreSerializer.newRecord(MountTable.class);
+ if (stats instanceof MountTablePBImpl) {
+ MountTablePBImpl entryPB = (MountTablePBImpl)stats;
+ entryPB.setProto(statsProto);
+ return entryPB;
+ } else {
+ throw new IOException("Cannot get stats for the membership");
+ }
+ }
+
+ @Override
+ public void setEntry(MountTable mount) throws IOException {
+ if (mount instanceof MountTablePBImpl) {
+ MountTablePBImpl mountPB = (MountTablePBImpl)mount;
+ MountTableRecordProto mountProto =
+ (MountTableRecordProto)mountPB.getProto();
+ this.translator.getBuilder().setEntry(mountProto);
+ } else {
+ throw new IOException("Cannot set mount table entry");
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateMountTableEntryResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateMountTableEntryResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateMountTableEntryResponsePBImpl.java
new file mode 100644
index 0000000..5d566d6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateMountTableEntryResponsePBImpl.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryResponseProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * UpdateMountTableEntryResponse.
+ */
+public class UpdateMountTableEntryResponsePBImpl
+ extends UpdateMountTableEntryResponse implements PBRecord {
+
+ private FederationProtocolPBTranslator<UpdateMountTableEntryResponseProto,
+ UpdateMountTableEntryResponseProto.Builder,
+ UpdateMountTableEntryResponseProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<UpdateMountTableEntryResponseProto,
+ UpdateMountTableEntryResponseProto.Builder,
+ UpdateMountTableEntryResponseProtoOrBuilder>(
+ UpdateMountTableEntryResponseProto.class);
+
+ public UpdateMountTableEntryResponsePBImpl() {
+ }
+
+ public UpdateMountTableEntryResponsePBImpl(
+ UpdateMountTableEntryResponseProto proto) {
+ this.setProto(proto);
+ }
+
+ @Override
+ public UpdateMountTableEntryResponseProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public boolean getStatus() {
+ return this.translator.getProtoOrBuilder().getStatus();
+ }
+
+ @Override
+ public void setStatus(boolean result) {
+ this.translator.getBuilder().setStatus(result);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
new file mode 100644
index 0000000..16f2b8b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
@@ -0,0 +1,301 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Data schema for
+ * {@link org.apache.hadoop.hdfs.server.federation.store.
+ * MountTableStore FederationMountTableStore} data stored in the
+ * {@link org.apache.hadoop.hdfs.server.federation.store.
+ * StateStoreService FederationStateStoreService}. Supports string
+ * serialization.
+ */
+public abstract class MountTable extends BaseRecord {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MountTable.class);
+
+
+ /**
+ * Default constructor for a mount table entry.
+ */
+ public MountTable() {
+ super();
+ }
+
+ public static MountTable newInstance() {
+ MountTable record = StateStoreSerializer.newRecord(MountTable.class);
+ record.init();
+ return record;
+ }
+
+ /**
+ * Constructor for a mount table entry with a single destinations.
+ *
+ * @param src Source path in the mount entry.
+ * @param destinations Nameservice destination of the mount point.
+ * @param dateCreated Created date.
+ * @param dateModified Modified date.
+ * @throws IOException
+ */
+ public static MountTable newInstance(final String src,
+ final Map<String, String> destinations,
+ long dateCreated, long dateModified) throws IOException {
+
+ MountTable record = newInstance(src, destinations);
+ record.setDateCreated(dateCreated);
+ record.setDateModified(dateModified);
+ return record;
+ }
+
+ /**
+ * Constructor for a mount table entry with multiple destinations.
+ *
+ * @param src Source path in the mount entry.
+ * @param destinations Nameservice destinations of the mount point.
+ * @throws IOException
+ */
+ public static MountTable newInstance(final String src,
+ final Map<String, String> destinations) throws IOException {
+ MountTable record = newInstance();
+
+ // Normalize the mount path
+ record.setSourcePath(normalizeFileSystemPath(src));
+
+ // Build a list of remote locations
+ final List<RemoteLocation> locations = new LinkedList<>();
+ for (Entry<String, String> entry : destinations.entrySet()) {
+ String nsId = entry.getKey();
+ String path = normalizeFileSystemPath(entry.getValue());
+ RemoteLocation location = new RemoteLocation(nsId, path);
+ locations.add(location);
+ }
+
+ // Set the serialized dest string
+ record.setDestinations(locations);
+
+ // Validate
+ record.validate();
+ return record;
+ }
+
+ /**
+ * Get source path in the federated namespace.
+ *
+ * @return Source path in the federated namespace.
+ */
+ public abstract String getSourcePath();
+
+ /**
+ * Set source path in the federated namespace.
+ *
+ * @param path Source path in the federated namespace.
+ */
+ public abstract void setSourcePath(String path);
+
+ /**
+ * Get a list of destinations (namespace + path) present for this entry.
+ *
+ * @return List of RemoteLocation destinations. Null if no destinations.
+ */
+ public abstract List<RemoteLocation> getDestinations();
+
+ /**
+ * Set the destination paths.
+ *
+ * @param paths Destination paths.
+ */
+ public abstract void setDestinations(List<RemoteLocation> dests);
+
+ /**
+ * Add a new destination to this mount table entry.
+ */
+ public abstract boolean addDestination(String nsId, String path);
+
+ /**
+ * Check if the entry is read only.
+ *
+ * @return If the entry is read only.
+ */
+ public abstract boolean isReadOnly();
+
+ /**
+ * Set an entry to be read only.
+ *
+ * @param ro If the entry is read only.
+ */
+ public abstract void setReadOnly(boolean ro);
+
+ /**
+ * Get the order of the destinations for this mount table entry.
+ *
+ * @return Order of the destinations.
+ */
+ public abstract DestinationOrder getDestOrder();
+
+ /**
+ * Set the order of the destinations for this mount table entry.
+ *
+ * @param order Order of the destinations.
+ */
+ public abstract void setDestOrder(DestinationOrder order);
+
+ /**
+ * Get the default location.
+ * @return The default location.
+ */
+ public RemoteLocation getDefaultLocation() {
+ List<RemoteLocation> dests = this.getDestinations();
+ if (dests == null || dests.isEmpty()) {
+ return null;
+ }
+ return dests.get(0);
+ }
+
+ @Override
+ public boolean like(final BaseRecord o) {
+ if (o instanceof MountTable) {
+ MountTable other = (MountTable)o;
+ if (getSourcePath() != null &&
+ !getSourcePath().equals(other.getSourcePath())) {
+ return false;
+ }
+ if (getDestinations() != null &&
+ !getDestinations().equals(other.getDestinations())) {
+ return false;
+ }
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(this.getSourcePath());
+ sb.append("->");
+ List<RemoteLocation> destinations = this.getDestinations();
+ sb.append(destinations);
+ if (destinations != null && destinations.size() > 1) {
+ sb.append("[" + this.getDestOrder() + "]");
+ }
+ if (this.isReadOnly()) {
+ sb.append("[RO]");
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public SortedMap<String, String> getPrimaryKeys() {
+ SortedMap<String, String> map = new TreeMap<>();
+ map.put("sourcePath", this.getSourcePath());
+ return map;
+ }
+
+ @Override
+ public boolean validate() {
+ boolean ret = super.validate();
+ if (this.getSourcePath() == null || this.getSourcePath().length() == 0) {
+ LOG.error("Invalid entry, no source path specified ", this);
+ ret = false;
+ }
+ if (!this.getSourcePath().startsWith("/")) {
+ LOG.error("Invalid entry, all mount points must start with / ", this);
+ ret = false;
+ }
+ if (this.getDestinations() == null || this.getDestinations().size() == 0) {
+ LOG.error("Invalid entry, no destination paths specified ", this);
+ ret = false;
+ }
+ for (RemoteLocation loc : getDestinations()) {
+ String nsId = loc.getNameserviceId();
+ if (nsId == null || nsId.length() == 0) {
+ LOG.error("Invalid entry, invalid destination nameservice ", this);
+ ret = false;
+ }
+ if (loc.getDest() == null || loc.getDest().length() == 0) {
+ LOG.error("Invalid entry, invalid destination path ", this);
+ ret = false;
+ }
+ if (!loc.getDest().startsWith("/")) {
+ LOG.error("Invalid entry, all destination must start with / ", this);
+ ret = false;
+ }
+ }
+ return ret;
+ }
+
+ @Override
+ public long getExpirationMs() {
+ return 0;
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder(17, 31)
+ .append(this.getSourcePath())
+ .append(this.getDestinations())
+ .append(this.isReadOnly())
+ .append(this.getDestOrder())
+ .toHashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof MountTable) {
+ MountTable other = (MountTable)obj;
+ if (!this.getSourcePath().equals(other.getSourcePath())) {
+ return false;
+ } else if (!this.getDestinations().equals(other.getDestinations())) {
+ return false;
+ } else if (this.isReadOnly() != other.isReadOnly()) {
+ return false;
+ } else if (!this.getDestOrder().equals(other.getDestOrder())) {
+ return false;
+ }
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Normalize a path for that filesystem.
+ *
+ * @param path Path to normalize.
+ * @return Normalized path.
+ */
+ private static String normalizeFileSystemPath(final String path) {
+ Path normalizedPath = new Path(path);
+ return normalizedPath.toString();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java
new file mode 100644
index 0000000..d2870bd
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java
@@ -0,0 +1,213 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records.impl.pb;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProto.DestOrder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProtoOrBuilder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoteLocationProto;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.FederationProtocolPBTranslator;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the MountTable record.
+ */
+public class MountTablePBImpl extends MountTable implements PBRecord {
+
+ private FederationProtocolPBTranslator<MountTableRecordProto, Builder,
+ MountTableRecordProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<MountTableRecordProto, Builder,
+ MountTableRecordProtoOrBuilder>(MountTableRecordProto.class);
+
+ public MountTablePBImpl() {
+ }
+
+ public MountTablePBImpl(MountTableRecordProto proto) {
+ this.setProto(proto);
+ }
+
+ @Override
+ public MountTableRecordProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public String getSourcePath() {
+ MountTableRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
+ if (!proto.hasSrcPath()) {
+ return null;
+ }
+ return proto.getSrcPath();
+ }
+
+ @Override
+ public void setSourcePath(String path) {
+ Builder builder = this.translator.getBuilder();
+ if (path == null) {
+ builder.clearSrcPath();
+ } else {
+ builder.setSrcPath(path);
+ }
+ }
+
+ @Override
+ public List<RemoteLocation> getDestinations() {
+ MountTableRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
+ if (proto.getDestinationsCount() == 0) {
+ return null;
+ }
+
+ final List<RemoteLocation> ret = new LinkedList<>();
+ final List<RemoteLocationProto> destList = proto.getDestinationsList();
+ for (RemoteLocationProto dest : destList) {
+ String nsId = dest.getNameserviceId();
+ String path = dest.getPath();
+ RemoteLocation loc = new RemoteLocation(nsId, path);
+ ret.add(loc);
+ }
+ return ret;
+ }
+
+ @Override
+ public void setDestinations(final List<RemoteLocation> dests) {
+ Builder builder = this.translator.getBuilder();
+ builder.clearDestinations();
+ for (RemoteLocation dest : dests) {
+ RemoteLocationProto.Builder itemBuilder =
+ RemoteLocationProto.newBuilder();
+ String nsId = dest.getNameserviceId();
+ String path = dest.getDest();
+ itemBuilder.setNameserviceId(nsId);
+ itemBuilder.setPath(path);
+ RemoteLocationProto item = itemBuilder.build();
+ builder.addDestinations(item);
+ }
+ }
+
+ @Override
+ public boolean addDestination(String nsId, String path) {
+ // Check if the location is already there
+ List<RemoteLocation> dests = getDestinations();
+ for (RemoteLocation dest : dests) {
+ if (dest.getNameserviceId().equals(nsId) && dest.getDest().equals(path)) {
+ return false;
+ }
+ }
+
+ // Add it to the existing list
+ Builder builder = this.translator.getBuilder();
+ RemoteLocationProto.Builder itemBuilder =
+ RemoteLocationProto.newBuilder();
+ itemBuilder.setNameserviceId(nsId);
+ itemBuilder.setPath(path);
+ RemoteLocationProto item = itemBuilder.build();
+ builder.addDestinations(item);
+ return true;
+ }
+
+ @Override
+ public void setDateModified(long time) {
+ this.translator.getBuilder().setDateModified(time);
+ }
+
+ @Override
+ public long getDateModified() {
+ return this.translator.getProtoOrBuilder().getDateModified();
+ }
+
+ @Override
+ public void setDateCreated(long time) {
+ this.translator.getBuilder().setDateCreated(time);
+ }
+
+ @Override
+ public long getDateCreated() {
+ return this.translator.getProtoOrBuilder().getDateCreated();
+ }
+
+ @Override
+ public boolean isReadOnly() {
+ MountTableRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
+ if (!proto.hasReadOnly()) {
+ return false;
+ }
+ return proto.getReadOnly();
+ }
+
+ @Override
+ public void setReadOnly(boolean ro) {
+ this.translator.getBuilder().setReadOnly(ro);
+ }
+
+ @Override
+ public DestinationOrder getDestOrder() {
+ MountTableRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
+ return convert(proto.getDestOrder());
+ }
+
+ @Override
+ public void setDestOrder(DestinationOrder order) {
+ Builder builder = this.translator.getBuilder();
+ if (order == null) {
+ builder.clearDestOrder();
+ } else {
+ builder.setDestOrder(convert(order));
+ }
+ }
+
+ private DestinationOrder convert(DestOrder order) {
+ switch (order) {
+ case LOCAL:
+ return DestinationOrder.LOCAL;
+ case RANDOM:
+ return DestinationOrder.RANDOM;
+ default:
+ return DestinationOrder.HASH;
+ }
+ }
+
+ private DestOrder convert(DestinationOrder order) {
+ switch (order) {
+ case LOCAL:
+ return DestOrder.LOCAL;
+ case RANDOM:
+ return DestOrder.RANDOM;
+ default:
+ return DestOrder.HASH;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/FederationProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/FederationProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/FederationProtocol.proto
index 487fe46..32a6250 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/FederationProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/FederationProtocol.proto
@@ -104,4 +104,63 @@ message NamenodeHeartbeatRequestProto {
message NamenodeHeartbeatResponseProto {
optional bool status = 1;
-}
\ No newline at end of file
+}
+
+
+/////////////////////////////////////////////////
+// Mount table
+/////////////////////////////////////////////////
+
+message RemoteLocationProto {
+ optional string nameserviceId = 1;
+ optional string path = 2;
+}
+
+message MountTableRecordProto {
+ optional string srcPath = 1;
+ repeated RemoteLocationProto destinations = 2;
+ optional uint64 dateCreated = 3;
+ optional uint64 dateModified = 4;
+ optional bool readOnly = 5 [default = false];
+
+ enum DestOrder {
+ HASH = 0;
+ LOCAL = 1;
+ RANDOM = 2;
+ }
+ optional DestOrder destOrder = 6 [default = HASH];
+}
+
+message AddMountTableEntryRequestProto {
+ optional MountTableRecordProto entry = 1;
+}
+
+message AddMountTableEntryResponseProto {
+ optional bool status = 1;
+}
+
+message UpdateMountTableEntryRequestProto {
+ optional MountTableRecordProto entry = 1;
+}
+
+message UpdateMountTableEntryResponseProto {
+ optional bool status = 1;
+}
+
+message RemoveMountTableEntryRequestProto {
+ optional string srcPath = 1;
+}
+
+message RemoveMountTableEntryResponseProto{
+ optional bool status = 1;
+}
+
+message GetMountTableEntriesRequestProto {
+ optional string srcPath = 1;
+}
+
+message GetMountTableEntriesResponseProto {
+ repeated MountTableRecordProto entries = 1;
+ optional uint64 timestamp = 2;
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
index 87427fd..a481553 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hdfs.server.federation.resolver.NamenodePriorityCompara
import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport;
import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.federation.router.Router;
import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
import org.apache.hadoop.util.Time;
@@ -68,6 +69,10 @@ public class MockResolver
this();
}
+ public MockResolver(Configuration conf, Router router) {
+ this();
+ }
+
public void addLocation(String mount, String nsId, String location) {
List<RemoteLocation> locationsList = this.locations.get(mount);
if (locationsList == null) {
@@ -258,7 +263,6 @@ public class MockResolver
@Override
public PathLocation getDestinationForPath(String path) throws IOException {
- Set<String> namespaceSet = new HashSet<>();
List<RemoteLocation> remoteLocations = new LinkedList<>();
for (String key : this.locations.keySet()) {
if (path.startsWith(key)) {
@@ -268,7 +272,6 @@ public class MockResolver
RemoteLocation remoteLocation =
new RemoteLocation(nameservice, finalPath);
remoteLocations.add(remoteLocation);
- namespaceSet.add(nameservice);
}
break;
}
@@ -277,7 +280,7 @@ public class MockResolver
// Path isn't supported, mimic resolver behavior.
return null;
}
- return new PathLocation(path, remoteLocations, namespaceSet);
+ return new PathLocation(path, remoteLocations);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
new file mode 100644
index 0000000..682d569
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
@@ -0,0 +1,396 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.router.Router;
+import org.apache.hadoop.hdfs.server.federation.store.MountTableStore;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test the {@link MountTableStore} from the {@link Router}.
+ */
+public class TestMountTableResolver {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestMountTableResolver.class);
+
+ private MountTableResolver mountTable;
+
+ private Map<String, String> getMountTableEntry(
+ String subcluster, String path) {
+ Map<String, String> ret = new HashMap<>();
+ ret.put(subcluster, path);
+ return ret;
+ }
+
+ /**
+ * Setup the mount table.
+ * / -> 1:/
+ * __tmp -> 2:/tmp
+ * __user -> 3:/user
+ * ____a -> 2:/user/test
+ * ______demo
+ * ________test
+ * __________a -> 1:/user/test
+ * __________b -> 3:/user/test
+ * ____b
+ * ______file1.txt -> 4:/user/file1.txt
+ * __usr
+ * ____bin -> 2:/bin
+ *
+ * @throws IOException If it cannot set the mount table.
+ */
+ private void setupMountTable() throws IOException {
+ Configuration conf = new Configuration();
+ mountTable = new MountTableResolver(conf);
+
+ // Root mount point
+ Map<String, String> map = getMountTableEntry("1", "/");
+ mountTable.addEntry(MountTable.newInstance("/", map));
+
+ // /tmp
+ map = getMountTableEntry("2", "/");
+ mountTable.addEntry(MountTable.newInstance("/tmp", map));
+
+ // /user
+ map = getMountTableEntry("3", "/user");
+ mountTable.addEntry(MountTable.newInstance("/user", map));
+
+ // /usr/bin
+ map = getMountTableEntry("2", "/bin");
+ mountTable.addEntry(MountTable.newInstance("/usr/bin", map));
+
+ // /user/a
+ map = getMountTableEntry("2", "/user/test");
+ mountTable.addEntry(MountTable.newInstance("/user/a", map));
+
+ // /user/b/file1.txt
+ map = getMountTableEntry("4", "/user/file1.txt");
+ mountTable.addEntry(MountTable.newInstance("/user/b/file1.txt", map));
+
+ // /user/a/demo/test/a
+ map = getMountTableEntry("1", "/user/test");
+ mountTable.addEntry(MountTable.newInstance("/user/a/demo/test/a", map));
+
+ // /user/a/demo/test/b
+ map = getMountTableEntry("3", "/user/test");
+ mountTable.addEntry(MountTable.newInstance("/user/a/demo/test/b", map));
+ }
+
+ @Before
+ public void setup() throws IOException {
+ setupMountTable();
+ }
+
+ @Test
+ public void testDestination() throws IOException {
+
+ // Check files
+ assertEquals("1->/tesfile1.txt",
+ mountTable.getDestinationForPath("/tesfile1.txt").toString());
+
+ assertEquals("3->/user/testfile2.txt",
+ mountTable.getDestinationForPath("/user/testfile2.txt").toString());
+
+ assertEquals("2->/user/test/testfile3.txt",
+ mountTable.getDestinationForPath("/user/a/testfile3.txt").toString());
+
+ assertEquals("3->/user/b/testfile4.txt",
+ mountTable.getDestinationForPath("/user/b/testfile4.txt").toString());
+
+ assertEquals("1->/share/file5.txt",
+ mountTable.getDestinationForPath("/share/file5.txt").toString());
+
+ assertEquals("2->/bin/file7.txt",
+ mountTable.getDestinationForPath("/usr/bin/file7.txt").toString());
+
+ assertEquals("1->/usr/file8.txt",
+ mountTable.getDestinationForPath("/usr/file8.txt").toString());
+
+ assertEquals("2->/user/test/demo/file9.txt",
+ mountTable.getDestinationForPath("/user/a/demo/file9.txt").toString());
+
+ // Check folders
+ assertEquals("3->/user/testfolder",
+ mountTable.getDestinationForPath("/user/testfolder").toString());
+
+ assertEquals("2->/user/test/b",
+ mountTable.getDestinationForPath("/user/a/b").toString());
+
+ assertEquals("3->/user/test/a",
+ mountTable.getDestinationForPath("/user/test/a").toString());
+
+ }
+
+ private void compareLists(List<String> list1, String[] list2) {
+ assertEquals(list1.size(), list2.length);
+ for (String item : list2) {
+ assertTrue(list1.contains(item));
+ }
+ }
+
+ @Test
+ public void testGetMountPoints() throws IOException {
+
+ // Check getting all mount points (virtual and real) beneath a path
+ List<String> mounts = mountTable.getMountPoints("/");
+ assertEquals(3, mounts.size());
+ compareLists(mounts, new String[] {"tmp", "user", "usr"});
+
+ mounts = mountTable.getMountPoints("/user");
+ assertEquals(2, mounts.size());
+ compareLists(mounts, new String[] {"a", "b"});
+
+ mounts = mountTable.getMountPoints("/user/a");
+ assertEquals(1, mounts.size());
+ compareLists(mounts, new String[] {"demo"});
+
+ mounts = mountTable.getMountPoints("/user/a/demo");
+ assertEquals(1, mounts.size());
+ compareLists(mounts, new String[] {"test"});
+
+ mounts = mountTable.getMountPoints("/user/a/demo/test");
+ assertEquals(2, mounts.size());
+ compareLists(mounts, new String[] {"a", "b"});
+
+ mounts = mountTable.getMountPoints("/tmp");
+ assertEquals(0, mounts.size());
+
+ mounts = mountTable.getMountPoints("/t");
+ assertNull(mounts);
+
+ mounts = mountTable.getMountPoints("/unknownpath");
+ assertNull(mounts);
+ }
+
+ private void compareRecords(List<MountTable> list1, String[] list2) {
+ assertEquals(list1.size(), list2.length);
+ for (String item : list2) {
+ for (MountTable record : list1) {
+ if (record.getSourcePath().equals(item)) {
+ return;
+ }
+ }
+ }
+ fail();
+ }
+
+ @Test
+ public void testGetMounts() throws IOException {
+
+ // Check listing the mount table records at or beneath a path
+ List<MountTable> records = mountTable.getMounts("/");
+ assertEquals(8, records.size());
+ compareRecords(records, new String[] {"/", "/tmp", "/user", "/usr/bin",
+ "user/a", "/user/a/demo/a", "/user/a/demo/b", "/user/b/file1.txt"});
+
+ records = mountTable.getMounts("/user");
+ assertEquals(5, records.size());
+ compareRecords(records, new String[] {"/user", "/user/a/demo/a",
+ "/user/a/demo/b", "user/a", "/user/b/file1.txt"});
+
+ records = mountTable.getMounts("/user/a");
+ assertEquals(3, records.size());
+ compareRecords(records,
+ new String[] {"/user/a/demo/a", "/user/a/demo/b", "/user/a"});
+
+ records = mountTable.getMounts("/tmp");
+ assertEquals(1, records.size());
+ compareRecords(records, new String[] {"/tmp"});
+ }
+
+ @Test
+ public void testRemoveSubTree()
+ throws UnsupportedOperationException, IOException {
+
+ // 3 mount points are present /tmp, /user, /usr
+ compareLists(mountTable.getMountPoints("/"),
+ new String[] {"user", "usr", "tmp"});
+
+ // /tmp currently points to namespace 2
+ assertEquals("2", mountTable.getDestinationForPath("/tmp/testfile.txt")
+ .getDefaultLocation().getNameserviceId());
+
+ // Remove tmp
+ mountTable.removeEntry("/tmp");
+
+ // Now 2 mount points are present /user, /usr
+ compareLists(mountTable.getMountPoints("/"),
+ new String[] {"user", "usr"});
+
+ // /tmp no longer exists, uses default namespace for mapping /
+ assertEquals("1", mountTable.getDestinationForPath("/tmp/testfile.txt")
+ .getDefaultLocation().getNameserviceId());
+ }
+
+ @Test
+ public void testRemoveVirtualNode()
+ throws UnsupportedOperationException, IOException {
+
+ // 3 mount points are present /tmp, /user, /usr
+ compareLists(mountTable.getMountPoints("/"),
+ new String[] {"user", "usr", "tmp"});
+
+ // /usr is virtual, uses namespace 1->/
+ assertEquals("1", mountTable.getDestinationForPath("/usr/testfile.txt")
+ .getDefaultLocation().getNameserviceId());
+
+ // Attempt to remove /usr
+ mountTable.removeEntry("/usr");
+
+ // Verify the remove failed
+ compareLists(mountTable.getMountPoints("/"),
+ new String[] {"user", "usr", "tmp"});
+ }
+
+ @Test
+ public void testRemoveLeafNode()
+ throws UnsupportedOperationException, IOException {
+
+ // /user/a/demo/test/a currently points to namespace 1
+ assertEquals("1", mountTable.getDestinationForPath("/user/a/demo/test/a")
+ .getDefaultLocation().getNameserviceId());
+
+ // Remove /user/a/demo/test/a
+ mountTable.removeEntry("/user/a/demo/test/a");
+
+ // Now /user/a/demo/test/a points to namespace 2 using the entry for /user/a
+ assertEquals("2", mountTable.getDestinationForPath("/user/a/demo/test/a")
+ .getDefaultLocation().getNameserviceId());
+
+ // Verify the virtual node at /user/a/demo still exists and was not deleted
+ compareLists(mountTable.getMountPoints("/user/a"), new String[] {"demo"});
+
+ // Verify the sibling node was unaffected and still points to ns 3
+ assertEquals("3", mountTable.getDestinationForPath("/user/a/demo/test/b")
+ .getDefaultLocation().getNameserviceId());
+ }
+
+ @Test
+ public void testRefreshEntries()
+ throws UnsupportedOperationException, IOException {
+
+ // Initial table loaded
+ testDestination();
+ assertEquals(8, mountTable.getMounts("/").size());
+
+ // Replace table with /1 and /2
+ List<MountTable> records = new ArrayList<>();
+ Map<String, String> map1 = getMountTableEntry("1", "/");
+ records.add(MountTable.newInstance("/1", map1));
+ Map<String, String> map2 = getMountTableEntry("2", "/");
+ records.add(MountTable.newInstance("/2", map2));
+ mountTable.refreshEntries(records);
+
+ // Verify addition
+ PathLocation destination1 = mountTable.getDestinationForPath("/1");
+ RemoteLocation defaultLoc1 = destination1.getDefaultLocation();
+ assertEquals("1", defaultLoc1.getNameserviceId());
+
+ PathLocation destination2 = mountTable.getDestinationForPath("/2");
+ RemoteLocation defaultLoc2 = destination2.getDefaultLocation();
+ assertEquals("2", defaultLoc2.getNameserviceId());
+
+ // Verify existing entries were removed
+ assertEquals(2, mountTable.getMounts("/").size());
+ boolean assertionThrown = false;
+ try {
+ testDestination();
+ fail();
+ } catch (AssertionError e) {
+ // The / entry was removed, so it triggers an exception
+ assertionThrown = true;
+ }
+ assertTrue(assertionThrown);
+ }
+
+ @Test
+ public void testMountTableScalability() throws IOException {
+
+ List<MountTable> emptyList = new ArrayList<>();
+ mountTable.refreshEntries(emptyList);
+
+ // Add 100,000 entries in flat list
+ for (int i = 0; i < 100000; i++) {
+ Map<String, String> map = getMountTableEntry("1", "/" + i);
+ MountTable record = MountTable.newInstance("/" + i, map);
+ mountTable.addEntry(record);
+ if (i % 10000 == 0) {
+ LOG.info("Adding flat mount record {}: {}", i, record);
+ }
+ }
+
+ assertEquals(100000, mountTable.getMountPoints("/").size());
+ assertEquals(100000, mountTable.getMounts("/").size());
+
+ // Add 1000 entries in deep list
+ mountTable.refreshEntries(emptyList);
+ String parent = "/";
+ for (int i = 0; i < 1000; i++) {
+ final int index = i;
+ Map<String, String> map = getMountTableEntry("1", "/" + index);
+ if (i > 0) {
+ parent = parent + "/";
+ }
+ parent = parent + i;
+ MountTable record = MountTable.newInstance(parent, map);
+ mountTable.addEntry(record);
+ }
+
+ assertEquals(1, mountTable.getMountPoints("/").size());
+ assertEquals(1000, mountTable.getMounts("/").size());
+
+ // Add 100,000 entries in deep and wide tree
+ mountTable.refreshEntries(emptyList);
+ Random rand = new Random();
+ parent = "/" + Integer.toString(rand.nextInt());
+ int numRootTrees = 1;
+ for (int i = 0; i < 100000; i++) {
+ final int index = i;
+ Map<String, String> map = getMountTableEntry("1", "/" + index);
+ parent = parent + "/" + i;
+ if (parent.length() > 2000) {
+ // Start new tree
+ parent = "/" + Integer.toString(rand.nextInt());
+ numRootTrees++;
+ }
+ MountTable record = MountTable.newInstance(parent, map);
+ mountTable.addEntry(record);
+ }
+
+ assertEquals(numRootTrees, mountTable.getMountPoints("/").size());
+ assertEquals(100000, mountTable.getMounts("/").size());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
index 598b9cf..dbb8f3f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
@@ -25,7 +25,9 @@ import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
@@ -40,6 +42,7 @@ import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFile
import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
import org.apache.hadoop.hdfs.server.federation.store.records.MembershipStats;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
import org.apache.hadoop.util.Time;
/**
@@ -234,6 +237,19 @@ public final class FederationStateStoreTestUtils {
return false;
}
+ public static List<MountTable> createMockMountTable(
+ List<String> nameservices) throws IOException {
+ // create table entries
+ List<MountTable> entries = new ArrayList<>();
+ for (String ns : nameservices) {
+ Map<String, String> destMap = new HashMap<>();
+ destMap.put(ns, "/target-" + ns);
+ MountTable entry = MountTable.newInstance("/" + ns, destMap);
+ entries.add(entry);
+ }
+ return entries;
+ }
+
public static MembershipState createMockRegistrationForNamenode(
String nameserviceId, String namenodeId,
FederationNamenodeServiceState state) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMountTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMountTable.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMountTable.java
new file mode 100644
index 0000000..d30d6ba
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMountTable.java
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMESERVICES;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyException;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.clearRecords;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.createMockMountTable;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.synchronizeRecords;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test the basic {@link StateStoreService}
+ * {@link MountTableStore} functionality.
+ */
+public class TestStateStoreMountTable extends TestStateStoreBase {
+
+ private static List<String> nameservices;
+ private static MountTableStore mountStore;
+
+ @BeforeClass
+ public static void create() throws IOException {
+ nameservices = new ArrayList<>();
+ nameservices.add(NAMESERVICES[0]);
+ nameservices.add(NAMESERVICES[1]);
+ }
+
+ @Before
+ public void setup() throws IOException, InterruptedException {
+ mountStore =
+ getStateStore().getRegisteredRecordStore(MountTableStore.class);
+ // Clear Mount table registrations
+ assertTrue(clearRecords(getStateStore(), MountTable.class));
+ }
+
+ @Test
+ public void testStateStoreDisconnected() throws Exception {
+
+ // Close the data store driver
+ getStateStore().closeDriver();
+ assertFalse(getStateStore().isDriverReady());
+
+ // Test APIs that access the store to check they throw the correct exception
+ AddMountTableEntryRequest addRequest =
+ AddMountTableEntryRequest.newInstance();
+ verifyException(mountStore, "addMountTableEntry",
+ StateStoreUnavailableException.class,
+ new Class[] {AddMountTableEntryRequest.class},
+ new Object[] {addRequest});
+
+ UpdateMountTableEntryRequest updateRequest =
+ UpdateMountTableEntryRequest.newInstance();
+ verifyException(mountStore, "updateMountTableEntry",
+ StateStoreUnavailableException.class,
+ new Class[] {UpdateMountTableEntryRequest.class},
+ new Object[] {updateRequest});
+
+ RemoveMountTableEntryRequest removeRequest =
+ RemoveMountTableEntryRequest.newInstance();
+ verifyException(mountStore, "removeMountTableEntry",
+ StateStoreUnavailableException.class,
+ new Class[] {RemoveMountTableEntryRequest.class},
+ new Object[] {removeRequest});
+
+ GetMountTableEntriesRequest getRequest =
+ GetMountTableEntriesRequest.newInstance();
+ mountStore.loadCache(true);
+ verifyException(mountStore, "getMountTableEntries",
+ StateStoreUnavailableException.class,
+ new Class[] {GetMountTableEntriesRequest.class},
+ new Object[] {getRequest});
+ }
+
+ @Test
+ public void testSynchronizeMountTable() throws IOException {
+ // Synchronize and get mount table entries
+ List<MountTable> entries = createMockMountTable(nameservices);
+ assertTrue(synchronizeRecords(getStateStore(), entries, MountTable.class));
+ for (MountTable e : entries) {
+ mountStore.loadCache(true);
+ MountTable entry = getMountTableEntry(e.getSourcePath());
+ assertNotNull(entry);
+ assertEquals(e.getDefaultLocation().getDest(),
+ entry.getDefaultLocation().getDest());
+ }
+ }
+
+ @Test
+ public void testAddMountTableEntry() throws IOException {
+
+ // Add 1
+ List<MountTable> entries = createMockMountTable(nameservices);
+ List<MountTable> entries1 = getMountTableEntries("/").getRecords();
+ assertEquals(0, entries1.size());
+ MountTable entry0 = entries.get(0);
+ AddMountTableEntryRequest request =
+ AddMountTableEntryRequest.newInstance(entry0);
+ AddMountTableEntryResponse response =
+ mountStore.addMountTableEntry(request);
+ assertTrue(response.getStatus());
+
+ mountStore.loadCache(true);
+ List<MountTable> entries2 = getMountTableEntries("/").getRecords();
+ assertEquals(1, entries2.size());
+ }
+
+ @Test
+ public void testRemoveMountTableEntry() throws IOException {
+
+ // Add many
+ List<MountTable> entries = createMockMountTable(nameservices);
+ synchronizeRecords(getStateStore(), entries, MountTable.class);
+ mountStore.loadCache(true);
+ List<MountTable> entries1 = getMountTableEntries("/").getRecords();
+ assertEquals(entries.size(), entries1.size());
+
+ // Remove 1
+ RemoveMountTableEntryRequest request =
+ RemoveMountTableEntryRequest.newInstance();
+ request.setSrcPath(entries.get(0).getSourcePath());
+ assertTrue(mountStore.removeMountTableEntry(request).getStatus());
+
+ // Verify remove
+ mountStore.loadCache(true);
+ List<MountTable> entries2 = getMountTableEntries("/").getRecords();
+ assertEquals(entries.size() - 1, entries2.size());
+ }
+
+ @Test
+ public void testUpdateMountTableEntry() throws IOException {
+
+ // Add 1
+ List<MountTable> entries = createMockMountTable(nameservices);
+ MountTable entry0 = entries.get(0);
+ String srcPath = entry0.getSourcePath();
+ String nsId = entry0.getDefaultLocation().getNameserviceId();
+ AddMountTableEntryRequest request =
+ AddMountTableEntryRequest.newInstance(entry0);
+ AddMountTableEntryResponse response =
+ mountStore.addMountTableEntry(request);
+ assertTrue(response.getStatus());
+
+ // Verify
+ mountStore.loadCache(true);
+ MountTable matchingEntry0 = getMountTableEntry(srcPath);
+ assertNotNull(matchingEntry0);
+ assertEquals(nsId, matchingEntry0.getDefaultLocation().getNameserviceId());
+
+ // Edit destination nameservice for source path
+ Map<String, String> destMap =
+ Collections.singletonMap("testnameservice", "/");
+ MountTable replacement =
+ MountTable.newInstance(srcPath, destMap);
+ UpdateMountTableEntryRequest updateRequest =
+ UpdateMountTableEntryRequest.newInstance(replacement);
+ UpdateMountTableEntryResponse updateResponse =
+ mountStore.updateMountTableEntry(updateRequest);
+ assertTrue(updateResponse.getStatus());
+
+ // Verify
+ mountStore.loadCache(true);
+ MountTable matchingEntry1 = getMountTableEntry(srcPath);
+ assertNotNull(matchingEntry1);
+ assertEquals("testnameservice",
+ matchingEntry1.getDefaultLocation().getNameserviceId());
+ }
+
+ /**
+ * Gets an existing mount table record in the state store.
+ *
+ * @param mount The mount point of the record to remove.
+ * @return The matching record if found, null if it is not found.
+ * @throws IOException If the state store could not be accessed.
+ */
+ private MountTable getMountTableEntry(String mount) throws IOException {
+ GetMountTableEntriesRequest request =
+ GetMountTableEntriesRequest.newInstance(mount);
+ GetMountTableEntriesResponse response =
+ mountStore.getMountTableEntries(request);
+ List<MountTable> results = response.getEntries();
+ if (results.size() > 0) {
+ // First result is sorted to have the shortest mount string length
+ return results.get(0);
+ }
+ return null;
+ }
+
+ /**
+ * Fetch all mount table records beneath a root path.
+ *
+ * @param store FederationMountTableStore instance to commit the data.
+ * @param mount The root search path, enter "/" to return all mount table
+ * records.
+ *
+ * @return A list of all mount table records found below the root mount.
+ *
+ * @throws IOException If the state store could not be accessed.
+ */
+ private QueryResult<MountTable> getMountTableEntries(String mount)
+ throws IOException {
+ if (mount == null) {
+ throw new IOException("Please specify a root search path");
+ }
+ GetMountTableEntriesRequest request =
+ GetMountTableEntriesRequest.newInstance();
+ request.setSrcPath(mount);
+ GetMountTableEntriesResponse response =
+ mountStore.getMountTableEntries(request);
+ List<MountTable> records = response.getEntries();
+ long timestamp = response.getTimestamp();
+ return new QueryResult<MountTable>(records, timestamp);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
index dc51ee9..8239fb1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
@@ -27,6 +27,7 @@ import java.io.IOException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -39,6 +40,7 @@ import org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUt
import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
import org.apache.hadoop.hdfs.server.federation.store.records.Query;
import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
import org.junit.AfterClass;
@@ -109,6 +111,11 @@ public class TestStateStoreDriverBase {
generateRandomString(), generateRandomString(),
generateRandomString(), generateRandomString(),
generateRandomEnum(FederationNamenodeServiceState.class), false);
+ } else if (recordClass == MountTable.class) {
+ String src = "/" + generateRandomString();
+ Map<String, String> destMap = Collections.singletonMap(
+ generateRandomString(), "/" + generateRandomString());
+ return (T) MountTable.newInstance(src, destMap);
}
return null;
@@ -155,6 +162,7 @@ public class TestStateStoreDriverBase {
public static void removeAll(StateStoreDriver driver) throws IOException {
driver.removeAll(MembershipState.class);
+ driver.removeAll(MountTable.class);
}
public <T extends BaseRecord> void testInsert(
@@ -347,22 +355,26 @@ public class TestStateStoreDriverBase {
public void testInsert(StateStoreDriver driver)
throws IllegalArgumentException, IllegalAccessException, IOException {
testInsert(driver, MembershipState.class);
+ testInsert(driver, MountTable.class);
}
public void testPut(StateStoreDriver driver)
throws IllegalArgumentException, ReflectiveOperationException,
IOException, SecurityException {
testPut(driver, MembershipState.class);
+ testPut(driver, MountTable.class);
}
public void testRemove(StateStoreDriver driver)
throws IllegalArgumentException, IllegalAccessException, IOException {
testRemove(driver, MembershipState.class);
+ testRemove(driver, MountTable.class);
}
public void testFetchErrors(StateStoreDriver driver)
throws IllegalArgumentException, IllegalAccessException, IOException {
testFetchErrors(driver, MembershipState.class);
+ testFetchErrors(driver, MountTable.class);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f2aa35/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMountTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMountTable.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMountTable.java
new file mode 100644
index 0000000..b6f91cf
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMountTable.java
@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.junit.Test;
+
+/**
+ * Test the Mount Table entry in the State Store.
+ */
+public class TestMountTable {
+
+ private static final String SRC = "/test";
+ private static final String DST_NS_0 = "ns0";
+ private static final String DST_NS_1 = "ns1";
+ private static final String DST_PATH_0 = "/path1";
+ private static final String DST_PATH_1 = "/path/path2";
+ private static final List<RemoteLocation> DST = new LinkedList<>();
+ static {
+ DST.add(new RemoteLocation(DST_NS_0, DST_PATH_0));
+ DST.add(new RemoteLocation(DST_NS_1, DST_PATH_1));
+ }
+ private static final Map<String, String> DST_MAP = new LinkedHashMap<>();
+ static {
+ DST_MAP.put(DST_NS_0, DST_PATH_0);
+ DST_MAP.put(DST_NS_1, DST_PATH_1);
+ }
+
+ private static final long DATE_CREATED = 100;
+ private static final long DATE_MOD = 200;
+
+
+ @Test
+ public void testGetterSetter() throws IOException {
+
+ MountTable record = MountTable.newInstance(SRC, DST_MAP);
+
+ validateDestinations(record);
+ assertEquals(SRC, record.getSourcePath());
+ assertEquals(DST, record.getDestinations());
+ assertTrue(DATE_CREATED > 0);
+ assertTrue(DATE_MOD > 0);
+
+ MountTable record2 =
+ MountTable.newInstance(SRC, DST_MAP, DATE_CREATED, DATE_MOD);
+
+ validateDestinations(record2);
+ assertEquals(SRC, record2.getSourcePath());
+ assertEquals(DST, record2.getDestinations());
+ assertEquals(DATE_CREATED, record2.getDateCreated());
+ assertEquals(DATE_MOD, record2.getDateModified());
+ assertFalse(record.isReadOnly());
+ assertEquals(DestinationOrder.HASH, record.getDestOrder());
+ }
+
+ @Test
+ public void testSerialization() throws IOException {
+ testSerialization(DestinationOrder.RANDOM);
+ testSerialization(DestinationOrder.HASH);
+ testSerialization(DestinationOrder.LOCAL);
+ }
+
+ private void testSerialization(final DestinationOrder order)
+ throws IOException {
+
+ MountTable record = MountTable.newInstance(
+ SRC, DST_MAP, DATE_CREATED, DATE_MOD);
+ record.setReadOnly(true);
+ record.setDestOrder(order);
+
+ StateStoreSerializer serializer = StateStoreSerializer.getSerializer();
+ String serializedString = serializer.serializeString(record);
+ MountTable record2 =
+ serializer.deserialize(serializedString, MountTable.class);
+
+ validateDestinations(record2);
+ assertEquals(SRC, record2.getSourcePath());
+ assertEquals(DST, record2.getDestinations());
+ assertEquals(DATE_CREATED, record2.getDateCreated());
+ assertEquals(DATE_MOD, record2.getDateModified());
+ assertTrue(record2.isReadOnly());
+ assertEquals(order, record2.getDestOrder());
+ }
+
+ @Test
+ public void testReadOnly() throws IOException {
+
+ Map<String, String> dest = new HashMap<>();
+ dest.put(DST_NS_0, DST_PATH_0);
+ dest.put(DST_NS_1, DST_PATH_1);
+ MountTable record1 = MountTable.newInstance(SRC, dest);
+ record1.setReadOnly(true);
+
+ validateDestinations(record1);
+ assertEquals(SRC, record1.getSourcePath());
+ assertEquals(DST, record1.getDestinations());
+ assertTrue(DATE_CREATED > 0);
+ assertTrue(DATE_MOD > 0);
+ assertTrue(record1.isReadOnly());
+
+ MountTable record2 = MountTable.newInstance(
+ SRC, DST_MAP, DATE_CREATED, DATE_MOD);
+ record2.setReadOnly(true);
+
+ validateDestinations(record2);
+ assertEquals(SRC, record2.getSourcePath());
+ assertEquals(DST, record2.getDestinations());
+ assertEquals(DATE_CREATED, record2.getDateCreated());
+ assertEquals(DATE_MOD, record2.getDateModified());
+ assertTrue(record2.isReadOnly());
+ }
+
+ @Test
+ public void testOrder() throws IOException {
+ testOrder(DestinationOrder.HASH);
+ testOrder(DestinationOrder.LOCAL);
+ testOrder(DestinationOrder.RANDOM);
+ }
+
+ private void testOrder(final DestinationOrder order)
+ throws IOException {
+
+ MountTable record = MountTable.newInstance(
+ SRC, DST_MAP, DATE_CREATED, DATE_MOD);
+ record.setDestOrder(order);
+
+ validateDestinations(record);
+ assertEquals(SRC, record.getSourcePath());
+ assertEquals(DST, record.getDestinations());
+ assertEquals(DATE_CREATED, record.getDateCreated());
+ assertEquals(DATE_MOD, record.getDateModified());
+ assertEquals(order, record.getDestOrder());
+ }
+
+ private void validateDestinations(MountTable record) {
+
+ assertEquals(SRC, record.getSourcePath());
+ assertEquals(2, record.getDestinations().size());
+
+ RemoteLocation location1 = record.getDestinations().get(0);
+ assertEquals(DST_NS_0, location1.getNameserviceId());
+ assertEquals(DST_PATH_0, location1.getDest());
+
+ RemoteLocation location2 = record.getDestinations().get(1);
+ assertEquals(DST_NS_1, location2.getNameserviceId());
+ assertEquals(DST_PATH_1, location2.getDest());
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/40] hadoop git commit: HDFS-12357. Let NameNode to bypass
external attribute provider for configured users. Contributed by Yongjun
Zhang, Arun Suresh.
Posted by in...@apache.org.
HDFS-12357. Let NameNode to bypass external attribute provider for configured users. Contributed by Yongjun Zhang, Arun Suresh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d77ed238
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d77ed238
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d77ed238
Branch: refs/heads/HDFS-10467
Commit: d77ed238a911fc85d6f4bbce606cac7ec44f557f
Parents: 5ff7416
Author: Yongjun Zhang <yz...@cloudera.com>
Authored: Thu Sep 7 09:45:45 2017 -0700
Committer: Yongjun Zhang <yz...@cloudera.com>
Committed: Thu Sep 7 09:50:36 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 +
.../hdfs/server/namenode/FSDirectory.java | 58 +++++++++-
.../src/main/resources/hdfs-default.xml | 12 +++
.../namenode/TestINodeAttributeProvider.java | 105 +++++++++++++++++--
4 files changed, 167 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d77ed238/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index bc7b716..d06e378 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -641,6 +641,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_DATANODE_MIN_SUPPORTED_NAMENODE_VERSION_KEY = "dfs.datanode.min.supported.namenode.version";
public static final String DFS_DATANODE_MIN_SUPPORTED_NAMENODE_VERSION_DEFAULT = "2.1.0-beta";
public static final String DFS_NAMENODE_INODE_ATTRIBUTES_PROVIDER_KEY = "dfs.namenode.inode.attributes.provider.class";
+ public static final String DFS_NAMENODE_INODE_ATTRIBUTES_PROVIDER_BYPASS_USERS_KEY = "dfs.namenode.inode.attributes.provider.bypass.users";
+ public static final String DFS_NAMENODE_INODE_ATTRIBUTES_PROVIDER_BYPASS_USERS_DEFAULT = "";
+
public static final String DFS_DATANODE_BP_READY_TIMEOUT_KEY = "dfs.datanode.bp-ready.timeout";
public static final long DFS_DATANODE_BP_READY_TIMEOUT_DEFAULT = 20;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d77ed238/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index e6aa533..6604b5a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -74,6 +74,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.SortedSet;
@@ -202,6 +203,10 @@ public class FSDirectory implements Closeable {
private INodeAttributeProvider attributeProvider;
+ // A HashSet of principals of users for whom the external attribute provider
+ // will be bypassed
+ private HashSet<String> usersToBypassExtAttrProvider = null;
+
public void setINodeAttributeProvider(INodeAttributeProvider provider) {
attributeProvider = provider;
}
@@ -357,6 +362,49 @@ public class FSDirectory implements Closeable {
this.quotaInitThreads = conf.getInt(
DFSConfigKeys.DFS_NAMENODE_QUOTA_INIT_THREADS_KEY,
DFSConfigKeys.DFS_NAMENODE_QUOTA_INIT_THREADS_DEFAULT);
+
+ initUsersToBypassExtProvider(conf);
+ }
+
+ private void initUsersToBypassExtProvider(Configuration conf) {
+ String[] bypassUsers = conf.getTrimmedStrings(
+ DFSConfigKeys.DFS_NAMENODE_INODE_ATTRIBUTES_PROVIDER_BYPASS_USERS_KEY,
+ DFSConfigKeys.DFS_NAMENODE_INODE_ATTRIBUTES_PROVIDER_BYPASS_USERS_DEFAULT);
+ for(int i = 0; i < bypassUsers.length; i++) {
+ String tmp = bypassUsers[i].trim();
+ if (!tmp.isEmpty()) {
+ if (usersToBypassExtAttrProvider == null) {
+ usersToBypassExtAttrProvider = new HashSet<String>();
+ }
+ LOG.info("Add user " + tmp + " to the list that will bypass external"
+ + " attribute provider.");
+ usersToBypassExtAttrProvider.add(tmp);
+ }
+ }
+ }
+
+ /**
+ * Check if a given user is configured to bypass external attribute provider.
+ * @param user user principal
+ * @return true if the user is to bypass external attribute provider
+ */
+ private boolean isUserBypassingExtAttrProvider(final String user) {
+ return (usersToBypassExtAttrProvider != null) &&
+ usersToBypassExtAttrProvider.contains(user);
+ }
+
+ /**
+ * Return attributeProvider or null if ugi is to bypass attributeProvider.
+ * @param ugi
+ * @return configured attributeProvider or null
+ */
+ private INodeAttributeProvider getUserFilteredAttributeProvider(
+ UserGroupInformation ugi) {
+ if (attributeProvider == null ||
+ (ugi != null && isUserBypassingExtAttrProvider(ugi.getUserName()))) {
+ return null;
+ }
+ return attributeProvider;
}
/**
@@ -1711,7 +1759,7 @@ public class FSDirectory implements Closeable {
FSPermissionChecker getPermissionChecker(String fsOwner, String superGroup,
UserGroupInformation ugi) throws AccessControlException {
return new FSPermissionChecker(
- fsOwner, superGroup, ugi, attributeProvider);
+ fsOwner, superGroup, ugi, getUserFilteredAttributeProvider(ugi));
}
void checkOwner(FSPermissionChecker pc, INodesInPath iip)
@@ -1896,18 +1944,20 @@ public class FSDirectory implements Closeable {
}
INodeAttributes getAttributes(INodesInPath iip)
- throws FileNotFoundException {
+ throws IOException {
INode node = FSDirectory.resolveLastINode(iip);
int snapshot = iip.getPathSnapshotId();
INodeAttributes nodeAttrs = node.getSnapshotINode(snapshot);
- if (attributeProvider != null) {
+ UserGroupInformation ugi = NameNode.getRemoteUser();
+ INodeAttributeProvider ap = this.getUserFilteredAttributeProvider(ugi);
+ if (ap != null) {
// permission checking sends the full components array including the
// first empty component for the root. however file status
// related calls are expected to strip out the root component according
// to TestINodeAttributeProvider.
byte[][] components = iip.getPathComponents();
components = Arrays.copyOfRange(components, 1, components.length);
- nodeAttrs = attributeProvider.getAttributes(components, nodeAttrs);
+ nodeAttrs = ap.getAttributes(components, nodeAttrs);
}
return nodeAttrs;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d77ed238/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index d58e54e..36c74f6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4160,6 +4160,18 @@
</property>
<property>
+ <name>dfs.namenode.authorization.provider.bypass.users</name>
+ <value></value>
+ <description>
+ A list of user principals (in secure cluster) or user names (in insecure
+ cluster) for whom the external attribute provider will be bypassed for all
+ operations. This means file attributes stored in HDFS instead of the
+ external provider will be used for permission checking and be returned when
+ requested.
+ </description>
+</property>
+
+<property>
<name>dfs.namenode.max-num-blocks-to-log</name>
<value>1000</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d77ed238/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java
index ffdc535..bbc5fa0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java
@@ -24,6 +24,7 @@ import java.util.Map;
import java.util.Set;
import com.google.common.collect.ImmutableList;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -33,19 +34,25 @@ import org.apache.hadoop.fs.permission.*;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider.AccessControlEnforcer;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
public class TestINodeAttributeProvider {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestINodeAttributeProvider.class);
+
private MiniDFSCluster miniDFS;
private static final Set<String> CALLED = new HashSet<String>();
+ private static final short HDFS_PERMISSION = 0777;
+ private static final short PROVIDER_PERMISSION = 0770;
public static class MyAuthorizationProvider extends INodeAttributeProvider {
@@ -112,7 +119,8 @@ public class TestINodeAttributeProvider {
@Override
public long getPermissionLong() {
- return (useDefault) ? inode.getPermissionLong() : 0770;
+ return (useDefault) ? inode.getPermissionLong() :
+ (long)PROVIDER_PERMISSION;
}
@Override
@@ -177,6 +185,9 @@ public class TestINodeAttributeProvider {
conf.set(DFSConfigKeys.DFS_NAMENODE_INODE_ATTRIBUTES_PROVIDER_KEY,
MyAuthorizationProvider.class.getName());
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
+ conf.set(
+ DFSConfigKeys.DFS_NAMENODE_INODE_ATTRIBUTES_PROVIDER_BYPASS_USERS_KEY,
+ " u2,, ,u3, ");
EditLogFileOutputStream.setShouldSkipFsyncForTesting(true);
miniDFS = new MiniDFSCluster.Builder(conf).build();
}
@@ -195,8 +206,11 @@ public class TestINodeAttributeProvider {
public void testDelegationToProvider() throws Exception {
Assert.assertTrue(CALLED.contains("start"));
FileSystem fs = FileSystem.get(miniDFS.getConfiguration(0));
- fs.mkdirs(new Path("/tmp"));
- fs.setPermission(new Path("/tmp"), new FsPermission((short) 0777));
+ final Path tmpPath = new Path("/tmp");
+ final Path fooPath = new Path("/tmp/foo");
+
+ fs.mkdirs(tmpPath);
+ fs.setPermission(tmpPath, new FsPermission(HDFS_PERMISSION));
UserGroupInformation ugi = UserGroupInformation.createUserForTesting("u1",
new String[]{"g1"});
ugi.doAs(new PrivilegedExceptionAction<Void>() {
@@ -204,17 +218,19 @@ public class TestINodeAttributeProvider {
public Void run() throws Exception {
FileSystem fs = FileSystem.get(miniDFS.getConfiguration(0));
CALLED.clear();
- fs.mkdirs(new Path("/tmp/foo"));
+ fs.mkdirs(fooPath);
Assert.assertTrue(CALLED.contains("getAttributes"));
Assert.assertTrue(CALLED.contains("checkPermission|null|null|null"));
Assert.assertTrue(CALLED.contains("checkPermission|WRITE|null|null"));
+
CALLED.clear();
- fs.listStatus(new Path("/tmp/foo"));
+ fs.listStatus(fooPath);
Assert.assertTrue(CALLED.contains("getAttributes"));
Assert.assertTrue(
CALLED.contains("checkPermission|null|null|READ_EXECUTE"));
+
CALLED.clear();
- fs.getAclStatus(new Path("/tmp/foo"));
+ fs.getAclStatus(fooPath);
Assert.assertTrue(CALLED.contains("getAttributes"));
Assert.assertTrue(CALLED.contains("checkPermission|null|null|null"));
return null;
@@ -222,6 +238,81 @@ public class TestINodeAttributeProvider {
});
}
+ private class AssertHelper {
+ private boolean bypass = true;
+ AssertHelper(boolean bp) {
+ bypass = bp;
+ }
+ public void doAssert(boolean x) {
+ if (bypass) {
+ Assert.assertFalse(x);
+ } else {
+ Assert.assertTrue(x);
+ }
+ }
+ }
+
+ private void testBypassProviderHelper(final String[] users,
+ final short expectedPermission, final boolean bypass) throws Exception {
+ final AssertHelper asserter = new AssertHelper(bypass);
+
+ Assert.assertTrue(CALLED.contains("start"));
+
+ FileSystem fs = FileSystem.get(miniDFS.getConfiguration(0));
+ final Path userPath = new Path("/user");
+ final Path authz = new Path("/user/authz");
+ final Path authzChild = new Path("/user/authz/child2");
+
+ fs.mkdirs(userPath);
+ fs.setPermission(userPath, new FsPermission(HDFS_PERMISSION));
+ fs.mkdirs(authz);
+ fs.setPermission(authz, new FsPermission(HDFS_PERMISSION));
+ fs.mkdirs(authzChild);
+ fs.setPermission(authzChild, new FsPermission(HDFS_PERMISSION));
+ for(String user : users) {
+ UserGroupInformation ugiBypass =
+ UserGroupInformation.createUserForTesting(user,
+ new String[]{"g1"});
+ ugiBypass.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ FileSystem fs = FileSystem.get(miniDFS.getConfiguration(0));
+ Assert.assertEquals(expectedPermission,
+ fs.getFileStatus(authzChild).getPermission().toShort());
+ asserter.doAssert(CALLED.contains("getAttributes"));
+ asserter.doAssert(CALLED.contains("checkPermission|null|null|null"));
+
+ CALLED.clear();
+ Assert.assertEquals(expectedPermission,
+ fs.listStatus(userPath)[0].getPermission().toShort());
+ asserter.doAssert(CALLED.contains("getAttributes"));
+ asserter.doAssert(
+ CALLED.contains("checkPermission|null|null|READ_EXECUTE"));
+
+ CALLED.clear();
+ fs.getAclStatus(authzChild);
+ asserter.doAssert(CALLED.contains("getAttributes"));
+ asserter.doAssert(CALLED.contains("checkPermission|null|null|null"));
+ return null;
+ }
+ });
+ }
+ }
+
+ @Test
+ public void testAuthzDelegationToProvider() throws Exception {
+ LOG.info("Test not bypassing provider");
+ String[] users = {"u1"};
+ testBypassProviderHelper(users, PROVIDER_PERMISSION, false);
+ }
+
+ @Test
+ public void testAuthzBypassingProvider() throws Exception {
+ LOG.info("Test bypassing provider");
+ String[] users = {"u2", "u3"};
+ testBypassProviderHelper(users, HDFS_PERMISSION, true);
+ }
+
@Test
public void testCustomProvider() throws Exception {
FileSystem fs = FileSystem.get(miniDFS.getConfiguration(0));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/40] hadoop git commit: HDFS-10629. Federation Roter. Contributed
by Jason Kace and Inigo Goiri.
Posted by in...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffdfd2e6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
new file mode 100644
index 0000000..ee6f57d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
@@ -0,0 +1,290 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.NamenodePriorityComparator;
+import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport;
+import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.util.Time;
+
+/**
+ * In-memory cache/mock of a namenode and file resolver. Stores the most
+ * recently updated NN information for each nameservice and block pool. Also
+ * stores a virtual mount table for resolving global namespace paths to local NN
+ * paths.
+ */
+public class MockResolver
+ implements ActiveNamenodeResolver, FileSubclusterResolver {
+
+ private Map<String, List<? extends FederationNamenodeContext>> resolver =
+ new HashMap<String, List<? extends FederationNamenodeContext>>();
+ private Map<String, List<RemoteLocation>> locations =
+ new HashMap<String, List<RemoteLocation>>();
+ private Set<FederationNamespaceInfo> namespaces =
+ new HashSet<FederationNamespaceInfo>();
+ private String defaultNamespace = null;
+
+ public MockResolver(Configuration conf, StateStoreService store) {
+ this.cleanRegistrations();
+ }
+
+ public void addLocation(String mount, String nameservice, String location) {
+ RemoteLocation remoteLocation = new RemoteLocation(nameservice, location);
+ List<RemoteLocation> locationsList = locations.get(mount);
+ if (locationsList == null) {
+ locationsList = new LinkedList<RemoteLocation>();
+ locations.put(mount, locationsList);
+ }
+ if (!locationsList.contains(remoteLocation)) {
+ locationsList.add(remoteLocation);
+ }
+
+ if (this.defaultNamespace == null) {
+ this.defaultNamespace = nameservice;
+ }
+ }
+
+ public synchronized void cleanRegistrations() {
+ this.resolver =
+ new HashMap<String, List<? extends FederationNamenodeContext>>();
+ this.namespaces = new HashSet<FederationNamespaceInfo>();
+ }
+
+ @Override
+ public void updateActiveNamenode(
+ String ns, InetSocketAddress successfulAddress) {
+
+ String address = successfulAddress.getHostName() + ":" +
+ successfulAddress.getPort();
+ String key = ns;
+ if (key != null) {
+ // Update the active entry
+ @SuppressWarnings("unchecked")
+ List<FederationNamenodeContext> iterator =
+ (List<FederationNamenodeContext>) resolver.get(key);
+ for (FederationNamenodeContext namenode : iterator) {
+ if (namenode.getRpcAddress().equals(address)) {
+ MockNamenodeContext nn = (MockNamenodeContext) namenode;
+ nn.setState(FederationNamenodeServiceState.ACTIVE);
+ break;
+ }
+ }
+ Collections.sort(iterator, new NamenodePriorityComparator());
+ }
+ }
+
+ @Override
+ public List<? extends FederationNamenodeContext>
+ getNamenodesForNameserviceId(String nameserviceId) {
+ return resolver.get(nameserviceId);
+ }
+
+ @Override
+ public List<? extends FederationNamenodeContext> getNamenodesForBlockPoolId(
+ String blockPoolId) {
+ return resolver.get(blockPoolId);
+ }
+
+ private static class MockNamenodeContext
+ implements FederationNamenodeContext {
+ private String webAddress;
+ private String rpcAddress;
+ private String serviceAddress;
+ private String lifelineAddress;
+ private String namenodeId;
+ private String nameserviceId;
+ private FederationNamenodeServiceState state;
+ private long dateModified;
+
+ MockNamenodeContext(
+ String rpc, String service, String lifeline, String web,
+ String ns, String nn, FederationNamenodeServiceState state) {
+ this.rpcAddress = rpc;
+ this.serviceAddress = service;
+ this.lifelineAddress = lifeline;
+ this.webAddress = web;
+ this.namenodeId = nn;
+ this.nameserviceId = ns;
+ this.state = state;
+ this.dateModified = Time.now();
+ }
+
+ public void setState(FederationNamenodeServiceState newState) {
+ this.state = newState;
+ this.dateModified = Time.now();
+ }
+
+ @Override
+ public String getRpcAddress() {
+ return rpcAddress;
+ }
+
+ @Override
+ public String getServiceAddress() {
+ return serviceAddress;
+ }
+
+ @Override
+ public String getLifelineAddress() {
+ return lifelineAddress;
+ }
+
+ @Override
+ public String getWebAddress() {
+ return webAddress;
+ }
+
+ @Override
+ public String getNamenodeKey() {
+ return nameserviceId + " " + namenodeId + " " + rpcAddress;
+ }
+
+ @Override
+ public String getNameserviceId() {
+ return nameserviceId;
+ }
+
+ @Override
+ public String getNamenodeId() {
+ return namenodeId;
+ }
+
+ @Override
+ public FederationNamenodeServiceState getState() {
+ return state;
+ }
+
+ @Override
+ public long getDateModified() {
+ return dateModified;
+ }
+ }
+
+ @Override
+ public synchronized boolean registerNamenode(NamenodeStatusReport report)
+ throws IOException {
+ MockNamenodeContext context = new MockNamenodeContext(
+ report.getRpcAddress(), report.getServiceAddress(),
+ report.getLifelineAddress(), report.getWebAddress(),
+ report.getNameserviceId(), report.getNamenodeId(), report.getState());
+
+ String nsId = report.getNameserviceId();
+ String bpId = report.getBlockPoolId();
+ String cId = report.getClusterId();
+ @SuppressWarnings("unchecked")
+ List<MockNamenodeContext> existingItems =
+ (List<MockNamenodeContext>) resolver.get(nsId);
+ if (existingItems == null) {
+ existingItems = new ArrayList<MockNamenodeContext>();
+ resolver.put(bpId, existingItems);
+ resolver.put(nsId, existingItems);
+ }
+ boolean added = false;
+ for (int i=0; i<existingItems.size() && !added; i++) {
+ MockNamenodeContext existing = existingItems.get(i);
+ if (existing.getNamenodeKey().equals(context.getNamenodeKey())) {
+ existingItems.set(i, context);
+ added = true;
+ }
+ }
+ if (!added) {
+ existingItems.add(context);
+ }
+ Collections.sort(existingItems, new NamenodePriorityComparator());
+
+ FederationNamespaceInfo info = new FederationNamespaceInfo(bpId, cId, nsId);
+ namespaces.add(info);
+ return true;
+ }
+
+ @Override
+ public Set<FederationNamespaceInfo> getNamespaces() throws IOException {
+ return this.namespaces;
+ }
+
+ @Override
+ public PathLocation getDestinationForPath(String path) throws IOException {
+ String finalPath = null;
+ String nameservice = null;
+ Set<String> namespaceSet = new HashSet<String>();
+ LinkedList<RemoteLocation> remoteLocations =
+ new LinkedList<RemoteLocation>();
+ for(String key : this.locations.keySet()) {
+ if(path.startsWith(key)) {
+ for (RemoteLocation location : this.locations.get(key)) {
+ finalPath = location.getDest() + path.substring(key.length());
+ nameservice = location.getNameserviceId();
+ RemoteLocation remoteLocation =
+ new RemoteLocation(nameservice, finalPath);
+ remoteLocations.add(remoteLocation);
+ namespaceSet.add(nameservice);
+ }
+ break;
+ }
+ }
+ if (remoteLocations.isEmpty()) {
+ // Path isn't supported, mimic resolver behavior.
+ return null;
+ }
+ return new PathLocation(path, remoteLocations, namespaceSet);
+ }
+
+ @Override
+ public List<String> getMountPoints(String path) throws IOException {
+ List<String> mounts = new ArrayList<String>();
+ if (path.equals("/")) {
+ // Mounts only supported under root level
+ for (String mount : this.locations.keySet()) {
+ if (mount.length() > 1) {
+ // Remove leading slash, this is the behavior of the mount tree,
+ // return only names.
+ mounts.add(mount.replace("/", ""));
+ }
+ }
+ }
+ return mounts;
+ }
+
+ @Override
+ public void setRouterId(String router) {
+ }
+
+ @Override
+ public String getDefaultNamespace() {
+ return defaultNamespace;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffdfd2e6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
new file mode 100644
index 0000000..16d624c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Constructs a router configuration with individual features enabled/disabled.
+ */
+public class RouterConfigBuilder {
+
+ private Configuration conf;
+
+ public RouterConfigBuilder(Configuration configuration) {
+ this.conf = configuration;
+ }
+
+ public RouterConfigBuilder() {
+ this.conf = new Configuration();
+ }
+
+ public Configuration build() {
+ return conf;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffdfd2e6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
new file mode 100644
index 0000000..55d04ad
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
@@ -0,0 +1,767 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology.NNConf;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology.NSConf;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport;
+import org.apache.hadoop.hdfs.server.federation.router.Router;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.Service.STATE;
+
+/**
+ * Test utility to mimic a federated HDFS cluster with a router.
+ */
+public class RouterDFSCluster {
+ /**
+ * Router context.
+ */
+ public class RouterContext {
+ private Router router;
+ private FileContext fileContext;
+ private String nameserviceId;
+ private String namenodeId;
+ private int rpcPort;
+ private DFSClient client;
+ private Configuration conf;
+ private URI fileSystemUri;
+
+ public RouterContext(Configuration conf, String ns, String nn)
+ throws URISyntaxException {
+ this.namenodeId = nn;
+ this.nameserviceId = ns;
+ this.conf = conf;
+ router = new Router();
+ router.init(conf);
+ }
+
+ public Router getRouter() {
+ return this.router;
+ }
+
+ public String getNameserviceId() {
+ return this.nameserviceId;
+ }
+
+ public String getNamenodeId() {
+ return this.namenodeId;
+ }
+
+ public int getRpcPort() {
+ return this.rpcPort;
+ }
+
+ public FileContext getFileContext() {
+ return this.fileContext;
+ }
+
+ public void initRouter() throws URISyntaxException {
+ }
+
+ public DistributedFileSystem getFileSystem() throws IOException {
+ DistributedFileSystem fs =
+ (DistributedFileSystem) DistributedFileSystem.get(conf);
+ return fs;
+ }
+
+ public DFSClient getClient(UserGroupInformation user)
+ throws IOException, URISyntaxException, InterruptedException {
+
+ LOG.info("Connecting to router at " + fileSystemUri);
+ return user.doAs(new PrivilegedExceptionAction<DFSClient>() {
+ @Override
+ public DFSClient run() throws IOException {
+ return new DFSClient(fileSystemUri, conf);
+ }
+ });
+ }
+
+ public DFSClient getClient() throws IOException, URISyntaxException {
+
+ if (client == null) {
+ LOG.info("Connecting to router at " + fileSystemUri);
+ client = new DFSClient(fileSystemUri, conf);
+ }
+ return client;
+ }
+ }
+
+ /**
+ * Namenode context.
+ */
+ public class NamenodeContext {
+ private NameNode namenode;
+ private String nameserviceId;
+ private String namenodeId;
+ private FileContext fileContext;
+ private int rpcPort;
+ private int servicePort;
+ private int lifelinePort;
+ private int httpPort;
+ private URI fileSystemUri;
+ private int index;
+ private Configuration conf;
+ private DFSClient client;
+
+ public NamenodeContext(Configuration conf, String ns, String nn,
+ int index) {
+ this.conf = conf;
+ this.namenodeId = nn;
+ this.nameserviceId = ns;
+ this.index = index;
+ }
+
+ public NameNode getNamenode() {
+ return this.namenode;
+ }
+
+ public String getNameserviceId() {
+ return this.nameserviceId;
+ }
+
+ public String getNamenodeId() {
+ return this.namenodeId;
+ }
+
+ public FileContext getFileContext() {
+ return this.fileContext;
+ }
+
+ public void setNamenode(NameNode n) throws URISyntaxException {
+ namenode = n;
+
+ // Store the bound ports and override the default FS with the local NN's
+ // RPC
+ rpcPort = n.getNameNodeAddress().getPort();
+ servicePort = n.getServiceRpcAddress().getPort();
+ lifelinePort = n.getServiceRpcAddress().getPort();
+ httpPort = n.getHttpAddress().getPort();
+ fileSystemUri = new URI("hdfs://" + namenode.getHostAndPort());
+ DistributedFileSystem.setDefaultUri(conf, fileSystemUri);
+
+ try {
+ this.fileContext = FileContext.getFileContext(conf);
+ } catch (UnsupportedFileSystemException e) {
+ this.fileContext = null;
+ }
+ }
+
+ public String getRpcAddress() {
+ return namenode.getNameNodeAddress().getHostName() + ":" + rpcPort;
+ }
+
+ public String getServiceAddress() {
+ return namenode.getServiceRpcAddress().getHostName() + ":" + servicePort;
+ }
+
+ public String getLifelineAddress() {
+ return namenode.getServiceRpcAddress().getHostName() + ":" + lifelinePort;
+ }
+
+ public String getHttpAddress() {
+ return namenode.getHttpAddress().getHostName() + ":" + httpPort;
+ }
+
+ public DistributedFileSystem getFileSystem() throws IOException {
+ DistributedFileSystem fs =
+ (DistributedFileSystem) DistributedFileSystem.get(conf);
+ return fs;
+ }
+
+ public void resetClient() {
+ client = null;
+ }
+
+ public DFSClient getClient(UserGroupInformation user)
+ throws IOException, URISyntaxException, InterruptedException {
+
+ LOG.info("Connecting to namenode at " + fileSystemUri);
+ return user.doAs(new PrivilegedExceptionAction<DFSClient>() {
+ @Override
+ public DFSClient run() throws IOException {
+ return new DFSClient(fileSystemUri, conf);
+ }
+ });
+ }
+
+ public DFSClient getClient() throws IOException, URISyntaxException {
+ if (client == null) {
+ LOG.info("Connecting to namenode at " + fileSystemUri);
+ client = new DFSClient(fileSystemUri, conf);
+ }
+ return client;
+ }
+
+ public String getConfSuffix() {
+ String suffix = nameserviceId;
+ if (highAvailability) {
+ suffix += "." + namenodeId;
+ }
+ return suffix;
+ }
+ }
+
+ public static final String NAMENODE1 = "nn0";
+ public static final String NAMENODE2 = "nn1";
+ public static final String NAMENODE3 = "nn2";
+ public static final String TEST_STRING = "teststring";
+ public static final String TEST_DIR = "testdir";
+ public static final String TEST_FILE = "testfile";
+
+ private List<String> nameservices;
+ private List<RouterContext> routers;
+ private List<NamenodeContext> namenodes;
+ private static final Log LOG = LogFactory.getLog(RouterDFSCluster.class);
+ private MiniDFSCluster cluster;
+ private boolean highAvailability;
+
+ protected static final int DEFAULT_HEARTBEAT_INTERVAL = 5;
+ protected static final int DEFAULT_CACHE_INTERVAL_SEC = 5;
+ private Configuration routerOverrides;
+ private Configuration namenodeOverrides;
+
+ private static final String NAMENODES = NAMENODE1 + "," + NAMENODE2;
+
+ public RouterDFSCluster(boolean ha, int numNameservices) {
+ this(ha, numNameservices, 2);
+ }
+
+ public RouterDFSCluster(boolean ha, int numNameservices, int numNamenodes) {
+ this.highAvailability = ha;
+ configureNameservices(numNameservices, numNamenodes);
+ }
+
+ public void addRouterOverrides(Configuration conf) {
+ if (this.routerOverrides == null) {
+ this.routerOverrides = conf;
+ } else {
+ this.routerOverrides.addResource(conf);
+ }
+ }
+
+ public void addNamenodeOverrides(Configuration conf) {
+ if (this.namenodeOverrides == null) {
+ this.namenodeOverrides = conf;
+ } else {
+ this.namenodeOverrides.addResource(conf);
+ }
+ }
+
+ public Configuration generateNamenodeConfiguration(
+ String defaultNameserviceId) {
+ Configuration c = new HdfsConfiguration();
+
+ c.set(DFSConfigKeys.DFS_NAMESERVICES, getNameservicesKey());
+ c.set("fs.defaultFS", "hdfs://" + defaultNameserviceId);
+
+ for (String ns : nameservices) {
+ if (highAvailability) {
+ c.set(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "." + ns, NAMENODES);
+ }
+
+ for (NamenodeContext context : getNamenodes(ns)) {
+ String suffix = context.getConfSuffix();
+
+ c.set(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + suffix,
+ "127.0.0.1:" + context.rpcPort);
+ c.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY + "." + suffix,
+ "127.0.0.1:" + context.httpPort);
+ c.set(DFSConfigKeys.DFS_NAMENODE_RPC_BIND_HOST_KEY + "." + suffix,
+ "0.0.0.0");
+ }
+ }
+
+ if (namenodeOverrides != null) {
+ c.addResource(namenodeOverrides);
+ }
+ return c;
+ }
+
+ public Configuration generateClientConfiguration() {
+ Configuration conf = new HdfsConfiguration();
+ conf.addResource(generateNamenodeConfiguration(getNameservices().get(0)));
+ return conf;
+ }
+
+ public Configuration generateRouterConfiguration(String localNameserviceId,
+ String localNamenodeId) throws IOException {
+ Configuration conf = new HdfsConfiguration();
+ conf.addResource(generateNamenodeConfiguration(localNameserviceId));
+
+ // Use mock resolver classes
+ conf.set(DFSConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS,
+ MockResolver.class.getCanonicalName());
+ conf.set(DFSConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
+ MockResolver.class.getCanonicalName());
+
+ // Set the nameservice ID for the default NN monitor
+ conf.set(DFSConfigKeys.DFS_NAMESERVICE_ID, localNameserviceId);
+
+ if (localNamenodeId != null) {
+ conf.set(DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY, localNamenodeId);
+ }
+
+ StringBuilder routerBuilder = new StringBuilder();
+ for (String ns : nameservices) {
+ for (NamenodeContext context : getNamenodes(ns)) {
+ String suffix = context.getConfSuffix();
+
+ if (routerBuilder.length() != 0) {
+ routerBuilder.append(",");
+ }
+ routerBuilder.append(suffix);
+ }
+ }
+
+ return conf;
+ }
+
+ public void configureNameservices(int numNameservices, int numNamenodes) {
+ nameservices = new ArrayList<String>();
+ for (int i = 0; i < numNameservices; i++) {
+ nameservices.add("ns" + i);
+ }
+ namenodes = new ArrayList<NamenodeContext>();
+ int index = 0;
+ for (String ns : nameservices) {
+ Configuration nnConf = generateNamenodeConfiguration(ns);
+ if (highAvailability) {
+ NamenodeContext context =
+ new NamenodeContext(nnConf, ns, NAMENODE1, index);
+ namenodes.add(context);
+ index++;
+
+ if (numNamenodes > 1) {
+ context = new NamenodeContext(nnConf, ns, NAMENODE2, index + 1);
+ namenodes.add(context);
+ index++;
+ }
+
+ if (numNamenodes > 2) {
+ context = new NamenodeContext(nnConf, ns, NAMENODE3, index + 1);
+ namenodes.add(context);
+ index++;
+ }
+
+ } else {
+ NamenodeContext context = new NamenodeContext(nnConf, ns, null, index);
+ namenodes.add(context);
+ index++;
+ }
+ }
+ }
+
+ public String getNameservicesKey() {
+ StringBuilder ns = new StringBuilder();
+ for (int i = 0; i < nameservices.size(); i++) {
+ if (i > 0) {
+ ns.append(",");
+ }
+ ns.append(nameservices.get(i));
+ }
+ return ns.toString();
+ }
+
+ public String getRandomNameservice() {
+ Random r = new Random();
+ return nameservices.get(r.nextInt(nameservices.size()));
+ }
+
+ public List<String> getNameservices() {
+ return nameservices;
+ }
+
+ public List<NamenodeContext> getNamenodes(String nameservice) {
+ ArrayList<NamenodeContext> nns = new ArrayList<NamenodeContext>();
+ for (NamenodeContext c : namenodes) {
+ if (c.nameserviceId.equals(nameservice)) {
+ nns.add(c);
+ }
+ }
+ return nns;
+ }
+
+ public NamenodeContext getRandomNamenode() {
+ Random rand = new Random();
+ return namenodes.get(rand.nextInt(namenodes.size()));
+ }
+
+ public List<NamenodeContext> getNamenodes() {
+ return namenodes;
+ }
+
+ public boolean isHighAvailability() {
+ return highAvailability;
+ }
+
+ public NamenodeContext getNamenode(String nameservice,
+ String namenode) {
+ for (NamenodeContext c : namenodes) {
+ if (c.nameserviceId.equals(nameservice)) {
+ if (namenode == null || c.namenodeId == null || namenode.isEmpty()
+ || c.namenodeId.isEmpty()) {
+ return c;
+ } else if (c.namenodeId.equals(namenode)) {
+ return c;
+ }
+ }
+ }
+ return null;
+ }
+
+ public List<RouterContext> getRouters(String nameservice) {
+ ArrayList<RouterContext> nns = new ArrayList<RouterContext>();
+ for (RouterContext c : routers) {
+ if (c.nameserviceId.equals(nameservice)) {
+ nns.add(c);
+ }
+ }
+ return nns;
+ }
+
+ public RouterContext getRouterContext(String nameservice,
+ String namenode) {
+ for (RouterContext c : routers) {
+ if (namenode == null) {
+ return c;
+ }
+ if (c.namenodeId.equals(namenode)
+ && c.nameserviceId.equals(nameservice)) {
+ return c;
+ }
+ }
+ return null;
+ }
+
+ public RouterContext getRandomRouter() {
+ Random rand = new Random();
+ return routers.get(rand.nextInt(routers.size()));
+ }
+
+ public List<RouterContext> getRouters() {
+ return routers;
+ }
+
+ public RouterContext buildRouter(String nameservice, String namenode)
+ throws URISyntaxException, IOException {
+ Configuration config = generateRouterConfiguration(nameservice, namenode);
+ RouterContext rc = new RouterContext(config, nameservice, namenode);
+ return rc;
+ }
+
+ public void startCluster() {
+ startCluster(null);
+ }
+
+ public void startCluster(Configuration overrideConf) {
+ try {
+ MiniDFSNNTopology topology = new MiniDFSNNTopology();
+ for (String ns : nameservices) {
+
+ NSConf conf = new MiniDFSNNTopology.NSConf(ns);
+ if (highAvailability) {
+ for(int i = 0; i < namenodes.size()/nameservices.size(); i++) {
+ NNConf nnConf = new MiniDFSNNTopology.NNConf("nn" + i);
+ conf.addNN(nnConf);
+ }
+ } else {
+ NNConf nnConf = new MiniDFSNNTopology.NNConf(null);
+ conf.addNN(nnConf);
+ }
+ topology.addNameservice(conf);
+ }
+ topology.setFederation(true);
+
+ // Start mini DFS cluster
+ Configuration nnConf = generateNamenodeConfiguration(nameservices.get(0));
+ if (overrideConf != null) {
+ nnConf.addResource(overrideConf);
+ }
+ cluster = new MiniDFSCluster.Builder(nnConf).nnTopology(topology).build();
+ cluster.waitActive();
+
+ // Store NN pointers
+ for (int i = 0; i < namenodes.size(); i++) {
+ NameNode nn = cluster.getNameNode(i);
+ namenodes.get(i).setNamenode(nn);
+ }
+
+ } catch (Exception e) {
+ LOG.error("Cannot start Router DFS cluster: " + e.getMessage(), e);
+ cluster.shutdown();
+ }
+ }
+
+ public void startRouters()
+ throws InterruptedException, URISyntaxException, IOException {
+ // Create routers
+ routers = new ArrayList<RouterContext>();
+ for (String ns : nameservices) {
+
+ for (NamenodeContext context : getNamenodes(ns)) {
+ routers.add(buildRouter(ns, context.namenodeId));
+ }
+ }
+
+ // Start all routers
+ for (RouterContext router : routers) {
+ router.router.start();
+ }
+ // Wait until all routers are active and record their ports
+ for (RouterContext router : routers) {
+ waitActive(router);
+ router.initRouter();
+ }
+ }
+
+ public void waitActive(NamenodeContext nn) throws IOException {
+ cluster.waitActive(nn.index);
+ }
+
+ public void waitActive(RouterContext router)
+ throws InterruptedException {
+ for (int loopCount = 0; loopCount < 20; loopCount++) {
+ // Validate connection of routers to NNs
+ if (router.router.getServiceState() == STATE.STARTED) {
+ return;
+ }
+ Thread.sleep(1000);
+ }
+ assertFalse(
+ "Timeout waiting for " + router.router.toString() + " to activate.",
+ true);
+ }
+
+
+ public void registerNamenodes() throws IOException {
+ for (RouterContext r : routers) {
+ ActiveNamenodeResolver resolver = r.router.getNamenodeResolver();
+ for (NamenodeContext nn : namenodes) {
+ // Generate a report
+ NamenodeStatusReport report = new NamenodeStatusReport(nn.nameserviceId,
+ nn.namenodeId, nn.getRpcAddress(), nn.getServiceAddress(),
+ nn.getLifelineAddress(), nn.getHttpAddress());
+ report.setNamespaceInfo(nn.namenode.getNamesystem().getFSImage()
+ .getStorage().getNamespaceInfo());
+
+ // Determine HA state from nn public state string
+ String nnState = nn.namenode.getState();
+ HAServiceState haState = HAServiceState.ACTIVE;
+ for (HAServiceState state : HAServiceState.values()) {
+ if (nnState.equalsIgnoreCase(state.name())) {
+ haState = state;
+ break;
+ }
+ }
+ report.setHAServiceState(haState);
+
+ // Register with the resolver
+ resolver.registerNamenode(report);
+ }
+ }
+ }
+
+ public void waitNamenodeRegistration()
+ throws InterruptedException, IllegalStateException, IOException {
+ for (RouterContext r : routers) {
+ for (NamenodeContext nn : namenodes) {
+ FederationTestUtils.waitNamenodeRegistered(
+ r.router.getNamenodeResolver(), nn.nameserviceId, nn.namenodeId,
+ null);
+ }
+ }
+ }
+
+ public void waitRouterRegistrationQuorum(RouterContext router,
+ FederationNamenodeServiceState state, String nameservice, String namenode)
+ throws InterruptedException, IOException {
+ LOG.info("Waiting for NN - " + nameservice + ":" + namenode
+ + " to transition to state - " + state);
+ FederationTestUtils.waitNamenodeRegistered(
+ router.router.getNamenodeResolver(), nameservice, namenode, state);
+ }
+
+ public String getFederatedPathForNameservice(String ns) {
+ return "/" + ns;
+ }
+
+ public String getNamenodePathForNameservice(String ns) {
+ return "/target-" + ns;
+ }
+
+ /**
+ * @return example:
+ * <ul>
+ * <li>/ns0/testdir which maps to ns0->/target-ns0/testdir
+ * </ul>
+ */
+ public String getFederatedTestDirectoryForNameservice(String ns) {
+ return getFederatedPathForNameservice(ns) + "/" + TEST_DIR;
+ }
+
+ /**
+ * @return example:
+ * <ul>
+ * <li>/target-ns0/testdir
+ * </ul>
+ */
+ public String getNamenodeTestDirectoryForNameservice(String ns) {
+ return getNamenodePathForNameservice(ns) + "/" + TEST_DIR;
+ }
+
+ /**
+ * @return example:
+ * <ul>
+ * <li>/ns0/testfile which maps to ns0->/target-ns0/testfile
+ * </ul>
+ */
+ public String getFederatedTestFileForNameservice(String ns) {
+ return getFederatedPathForNameservice(ns) + "/" + TEST_FILE;
+ }
+
+ /**
+ * @return example:
+ * <ul>
+ * <li>/target-ns0/testfile
+ * </ul>
+ */
+ public String getNamenodeTestFileForNameservice(String ns) {
+ return getNamenodePathForNameservice(ns) + "/" + TEST_FILE;
+ }
+
+ public void shutdown() {
+ cluster.shutdown();
+ if (routers != null) {
+ for (RouterContext context : routers) {
+ stopRouter(context);
+ }
+ }
+ }
+
+ public void stopRouter(RouterContext router) {
+ try {
+
+ router.router.shutDown();
+
+ int loopCount = 0;
+ while (router.router.getServiceState() != STATE.STOPPED) {
+ loopCount++;
+ Thread.sleep(1000);
+ if (loopCount > 20) {
+ LOG.error("Unable to shutdown router - " + router.rpcPort);
+ break;
+ }
+ }
+ } catch (InterruptedException e) {
+ }
+ }
+
+ /////////////////////////////////////////////////////////////////////////////
+ // Namespace Test Fixtures
+ /////////////////////////////////////////////////////////////////////////////
+
+ /**
+ * Creates test directories via the namenode.
+ * 1) /target-ns0/testfile
+ * 2) /target-ns1/testfile
+ * @throws IOException
+ */
+ public void createTestDirectoriesNamenode() throws IOException {
+ // Add a test dir to each NS and verify
+ for (String ns : getNameservices()) {
+ NamenodeContext context = getNamenode(ns, null);
+ if (!createTestDirectoriesNamenode(context)) {
+ throw new IOException("Unable to create test directory for ns - " + ns);
+ }
+ }
+ }
+
+ public boolean createTestDirectoriesNamenode(NamenodeContext nn)
+ throws IOException {
+ return FederationTestUtils.addDirectory(nn.getFileSystem(),
+ getNamenodeTestDirectoryForNameservice(nn.nameserviceId));
+ }
+
+ public void deleteAllFiles() throws IOException {
+ // Delete all files via the NNs and verify
+ for (NamenodeContext context : getNamenodes()) {
+ FileStatus[] status = context.getFileSystem().listStatus(new Path("/"));
+ for(int i = 0; i <status.length; i++) {
+ Path p = status[i].getPath();
+ context.getFileSystem().delete(p, true);
+ }
+ status = context.getFileSystem().listStatus(new Path("/"));
+ assertEquals(status.length, 0);
+ }
+ }
+
+ /////////////////////////////////////////////////////////////////////////////
+ // MockRouterResolver Test Fixtures
+ /////////////////////////////////////////////////////////////////////////////
+
+ /**
+ * <ul>
+ * <li>/ -> [ns0->/].
+ * <li>/nso -> ns0->/target-ns0.
+ * <li>/ns1 -> ns1->/target-ns1.
+ * </ul>
+ */
+ public void installMockLocations() {
+ for (RouterContext r : routers) {
+ MockResolver resolver =
+ (MockResolver) r.router.getSubclusterResolver();
+ // create table entries
+ for (String ns : nameservices) {
+ // Direct path
+ resolver.addLocation(getFederatedPathForNameservice(ns), ns,
+ getNamenodePathForNameservice(ns));
+ }
+
+ // Root path goes to both NS1
+ resolver.addLocation("/", nameservices.get(0), "/");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffdfd2e6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
new file mode 100644
index 0000000..8c720c7
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.MockResolver;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.service.Service.STATE;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * The the safe mode for the {@link Router} controlled by
+ * {@link SafeModeTimer}.
+ */
+public class TestRouter {
+
+ private static Configuration conf;
+
+ @BeforeClass
+ public static void create() throws IOException {
+ // Basic configuration without the state store
+ conf = new Configuration();
+ // Mock resolver classes
+ conf.set(DFSConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS,
+ MockResolver.class.getCanonicalName());
+ conf.set(DFSConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
+ MockResolver.class.getCanonicalName());
+
+ // Simulate a co-located NN
+ conf.set(DFSConfigKeys.DFS_NAMESERVICES, "ns0");
+ conf.set("fs.defaultFS", "hdfs://" + "ns0");
+ conf.set(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + "ns0",
+ "127.0.0.1:0" + 0);
+ conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY + "." + "ns0",
+ "127.0.0.1:" + 0);
+ conf.set(DFSConfigKeys.DFS_NAMENODE_RPC_BIND_HOST_KEY + "." + "ns0",
+ "0.0.0.0");
+ }
+
+ @AfterClass
+ public static void destroy() {
+ }
+
+ @Before
+ public void setup() throws IOException, URISyntaxException {
+ }
+
+ @After
+ public void cleanup() {
+ }
+
+ private static void testRouterStartup(Configuration routerConfig)
+ throws InterruptedException, IOException {
+ Router router = new Router();
+ assertEquals(STATE.NOTINITED, router.getServiceState());
+ router.init(routerConfig);
+ assertEquals(STATE.INITED, router.getServiceState());
+ router.start();
+ assertEquals(STATE.STARTED, router.getServiceState());
+ router.stop();
+ assertEquals(STATE.STOPPED, router.getServiceState());
+ router.close();
+ }
+
+ @Test
+ public void testRouterService() throws InterruptedException, IOException {
+
+ // Run with all services
+ testRouterStartup((new RouterConfigBuilder(conf)).build());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/40] hadoop git commit: HDFS-11546. Federation Router RPC server.
Contributed by Jason Kace and Inigo Goiri.
Posted by in...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e96a2c7f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
new file mode 100644
index 0000000..3a32be1
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
@@ -0,0 +1,856 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.net.InetSocketAddress;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadFactory;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.NameNodeProxiesClient.ProxyAndInfo;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.io.retry.RetryPolicies;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.io.retry.RetryPolicy.RetryAction.RetryDecision;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.ipc.StandbyException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+/**
+ * A client proxy for Router -> NN communication using the NN ClientProtocol.
+ * <p>
+ * Provides routers to invoke remote ClientProtocol methods and handle
+ * retries/failover.
+ * <ul>
+ * <li>invokeSingle Make a single request to a single namespace
+ * <li>invokeSequential Make a sequential series of requests to multiple
+ * ordered namespaces until a condition is met.
+ * <li>invokeConcurrent Make concurrent requests to multiple namespaces and
+ * return all of the results.
+ * </ul>
+ * Also maintains a cached pool of connections to NNs. Connections are managed
+ * by the ConnectionManager and are unique to each user + NN. The size of the
+ * connection pool can be configured. Larger pools allow for more simultaneous
+ * requests to a single NN from a single user.
+ */
+public class RouterRpcClient {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RouterRpcClient.class);
+
+
+ /** Router identifier. */
+ private final String routerId;
+
+ /** Interface to identify the active NN for a nameservice or blockpool ID. */
+ private final ActiveNamenodeResolver namenodeResolver;
+
+ /** Connection pool to the Namenodes per user for performance. */
+ private final ConnectionManager connectionManager;
+ /** Service to run asynchronous calls. */
+ private final ExecutorService executorService;
+ /** Retry policy for router -> NN communication. */
+ private final RetryPolicy retryPolicy;
+
+ /** Pattern to parse a stack trace line. */
+ private static final Pattern STACK_TRACE_PATTERN =
+ Pattern.compile("\\tat (.*)\\.(.*)\\((.*):(\\d*)\\)");
+
+
+ /**
+ * Create a router RPC client to manage remote procedure calls to NNs.
+ *
+ * @param conf Hdfs Configuation.
+ * @param resolver A NN resolver to determine the currently active NN in HA.
+ * @param monitor Optional performance monitor.
+ */
+ public RouterRpcClient(Configuration conf, String identifier,
+ ActiveNamenodeResolver resolver) {
+
+ this.routerId = identifier;
+
+ this.namenodeResolver = resolver;
+
+ this.connectionManager = new ConnectionManager(conf);
+ this.connectionManager.start();
+
+ ThreadFactory threadFactory = new ThreadFactoryBuilder()
+ .setNameFormat("RPC Router Client-%d")
+ .build();
+ this.executorService = Executors.newCachedThreadPool(threadFactory);
+
+ int maxFailoverAttempts = conf.getInt(
+ HdfsClientConfigKeys.Failover.MAX_ATTEMPTS_KEY,
+ HdfsClientConfigKeys.Failover.MAX_ATTEMPTS_DEFAULT);
+ int maxRetryAttempts = conf.getInt(
+ HdfsClientConfigKeys.Retry.MAX_ATTEMPTS_KEY,
+ HdfsClientConfigKeys.Retry.MAX_ATTEMPTS_DEFAULT);
+ int failoverSleepBaseMillis = conf.getInt(
+ HdfsClientConfigKeys.Failover.SLEEPTIME_BASE_KEY,
+ HdfsClientConfigKeys.Failover.SLEEPTIME_BASE_DEFAULT);
+ int failoverSleepMaxMillis = conf.getInt(
+ HdfsClientConfigKeys.Failover.SLEEPTIME_MAX_KEY,
+ HdfsClientConfigKeys.Failover.SLEEPTIME_MAX_DEFAULT);
+ this.retryPolicy = RetryPolicies.failoverOnNetworkException(
+ RetryPolicies.TRY_ONCE_THEN_FAIL, maxFailoverAttempts, maxRetryAttempts,
+ failoverSleepBaseMillis, failoverSleepMaxMillis);
+ }
+
+ /**
+ * Shutdown the client.
+ */
+ public void shutdown() {
+ if (this.connectionManager != null) {
+ this.connectionManager.close();
+ }
+ if (this.executorService != null) {
+ this.executorService.shutdownNow();
+ }
+ }
+
+ /**
+ * Total number of available sockets between the router and NNs.
+ *
+ * @return Number of namenode clients.
+ */
+ public int getNumConnections() {
+ return this.connectionManager.getNumConnections();
+ }
+
+ /**
+ * Total number of available sockets between the router and NNs.
+ *
+ * @return Number of namenode clients.
+ */
+ public int getNumActiveConnections() {
+ return this.connectionManager.getNumActiveConnections();
+ }
+
+ /**
+ * Total number of open connection pools to a NN. Each connection pool.
+ * represents one user + one NN.
+ *
+ * @return Number of connection pools.
+ */
+ public int getNumConnectionPools() {
+ return this.connectionManager.getNumConnectionPools();
+ }
+
+ /**
+ * Number of connections between the router and NNs being created sockets.
+ *
+ * @return Number of connections waiting to be created.
+ */
+ public int getNumCreatingConnections() {
+ return this.connectionManager.getNumCreatingConnections();
+ }
+
+ /**
+ * Get ClientProtocol proxy client for a NameNode. Each combination of user +
+ * NN must use a unique proxy client. Previously created clients are cached
+ * and stored in a connection pool by the ConnectionManager.
+ *
+ * @param ugi User group information.
+ * @param nsId Nameservice identifier.
+ * @param rpcAddress ClientProtocol RPC server address of the NN.
+ * @return ConnectionContext containing a ClientProtocol proxy client for the
+ * NN + current user.
+ * @throws IOException If we cannot get a connection to the NameNode.
+ */
+ private ConnectionContext getConnection(
+ UserGroupInformation ugi, String nsId, String rpcAddress)
+ throws IOException {
+ ConnectionContext connection = null;
+ try {
+ // Each proxy holds the UGI info for the current user when it is created.
+ // This cache does not scale very well, one entry per user per namenode,
+ // and may need to be adjusted and/or selectively pruned. The cache is
+ // important due to the excessive overhead of creating a new proxy wrapper
+ // for each individual request.
+
+ // TODO Add tokens from the federated UGI
+ connection = this.connectionManager.getConnection(ugi, rpcAddress);
+ LOG.debug("User {} NN {} is using connection {}",
+ ugi.getUserName(), rpcAddress, connection);
+ } catch (Exception ex) {
+ LOG.error("Cannot open NN client to address: {}", rpcAddress, ex);
+ }
+
+ if (connection == null) {
+ throw new IOException("Cannot get a connection to " + rpcAddress);
+ }
+ return connection;
+ }
+
+ /**
+ * Convert an exception to an IOException.
+ *
+ * For a non-IOException, wrap it with IOException. For a RemoteException,
+ * unwrap it. For an IOException which is not a RemoteException, return it.
+ *
+ * @param e Exception to convert into an exception.
+ * @return Created IO exception.
+ */
+ private static IOException toIOException(Exception e) {
+ if (e instanceof RemoteException) {
+ return ((RemoteException) e).unwrapRemoteException();
+ }
+ if (e instanceof IOException) {
+ return (IOException)e;
+ }
+ return new IOException(e);
+ }
+
+ /**
+ * If we should retry the RPC call.
+ *
+ * @param ex Exception reported.
+ * @param retryCount Number of retries.
+ * @return Retry decision.
+ * @throws IOException Original exception if the retry policy generates one.
+ */
+ private RetryDecision shouldRetry(final IOException ioe, final int retryCount)
+ throws IOException {
+ try {
+ final RetryPolicy.RetryAction a =
+ this.retryPolicy.shouldRetry(ioe, retryCount, 0, true);
+ return a.action;
+ } catch (Exception ex) {
+ LOG.error("Re-throwing API exception, no more retries", ex);
+ throw toIOException(ex);
+ }
+ }
+
+ /**
+ * Invokes a method against the ClientProtocol proxy server. If a standby
+ * exception is generated by the call to the client, retries using the
+ * alternate server.
+ *
+ * Re-throws exceptions generated by the remote RPC call as either
+ * RemoteException or IOException.
+ *
+ * @param ugi User group information.
+ * @param namenodes A prioritized list of namenodes within the same
+ * nameservice.
+ * @param method Remote ClientProtcol method to invoke.
+ * @param params Variable list of parameters matching the method.
+ * @return The result of invoking the method.
+ * @throws IOException
+ */
+ private Object invokeMethod(
+ final UserGroupInformation ugi,
+ final List<? extends FederationNamenodeContext> namenodes,
+ final Method method, final Object... params) throws IOException {
+
+ if (namenodes == null || namenodes.isEmpty()) {
+ throw new IOException("No namenodes to invoke " + method.getName() +
+ " with params " + Arrays.toString(params) + " from " + this.routerId);
+ }
+
+ Object ret = null;
+ boolean failover = false;
+ Map<FederationNamenodeContext, IOException> ioes = new LinkedHashMap<>();
+ for (FederationNamenodeContext namenode : namenodes) {
+ ConnectionContext connection = null;
+ try {
+ String nsId = namenode.getNameserviceId();
+ String rpcAddress = namenode.getRpcAddress();
+ connection = this.getConnection(ugi, nsId, rpcAddress);
+ ProxyAndInfo<ClientProtocol> client = connection.getClient();
+ ClientProtocol proxy = client.getProxy();
+ ret = invoke(0, method, proxy, params);
+ if (failover) {
+ // Success on alternate server, update
+ InetSocketAddress address = client.getAddress();
+ namenodeResolver.updateActiveNamenode(nsId, address);
+ }
+ return ret;
+ } catch (IOException ioe) {
+ ioes.put(namenode, ioe);
+ if (ioe instanceof StandbyException) {
+ // Fail over indicated by retry policy and/or NN
+ failover = true;
+ } else if (ioe instanceof RemoteException) {
+ // RemoteException returned by NN
+ throw (RemoteException) ioe;
+ } else {
+ // Other communication error, this is a failure
+ // Communication retries are handled by the retry policy
+ throw ioe;
+ }
+ } finally {
+ if (connection != null) {
+ connection.release();
+ }
+ }
+ }
+
+ // All namenodes were unavailable or in standby
+ String msg = "No namenode available to invoke " + method.getName() + " " +
+ Arrays.toString(params);
+ LOG.error(msg);
+ for (Entry<FederationNamenodeContext, IOException> entry :
+ ioes.entrySet()) {
+ FederationNamenodeContext namenode = entry.getKey();
+ String nsId = namenode.getNameserviceId();
+ String nnId = namenode.getNamenodeId();
+ String addr = namenode.getRpcAddress();
+ IOException ioe = entry.getValue();
+ if (ioe instanceof StandbyException) {
+ LOG.error("{} {} at {} is in Standby", nsId, nnId, addr);
+ } else {
+ LOG.error("{} {} at {} error: \"{}\"",
+ nsId, nnId, addr, ioe.getMessage());
+ }
+ }
+ throw new StandbyException(msg);
+ }
+
+ /**
+ * Invokes a method on the designated object. Catches exceptions specific to
+ * the invocation.
+ *
+ * Re-throws exceptions generated by the remote RPC call as either
+ * RemoteException or IOException.
+ *
+ * @param method Method to invoke
+ * @param obj Target object for the method
+ * @param params Variable parameters
+ * @return Response from the remote server
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ private Object invoke(int retryCount, final Method method, final Object obj,
+ final Object... params) throws IOException {
+ try {
+ return method.invoke(obj, params);
+ } catch (IllegalAccessException e) {
+ LOG.error("Unexpected exception while proxying API", e);
+ return null;
+ } catch (IllegalArgumentException e) {
+ LOG.error("Unexpected exception while proxying API", e);
+ return null;
+ } catch (InvocationTargetException e) {
+ Throwable cause = e.getCause();
+ if (cause instanceof IOException) {
+ IOException ioe = (IOException) cause;
+ // Check if we should retry.
+ RetryDecision decision = shouldRetry(ioe, retryCount);
+ if (decision == RetryDecision.RETRY) {
+ // retry
+ return invoke(++retryCount, method, obj, params);
+ } else if (decision == RetryDecision.FAILOVER_AND_RETRY) {
+ // failover, invoker looks for standby exceptions for failover.
+ if (ioe instanceof StandbyException) {
+ throw ioe;
+ } else {
+ throw new StandbyException(ioe.getMessage());
+ }
+ } else {
+ if (ioe instanceof RemoteException) {
+ RemoteException re = (RemoteException) ioe;
+ ioe = re.unwrapRemoteException();
+ ioe = getCleanException(ioe);
+ }
+ throw ioe;
+ }
+ } else {
+ throw new IOException(e);
+ }
+ }
+ }
+
+ /**
+ * Get a clean copy of the exception. Sometimes the exceptions returned by the
+ * server contain the full stack trace in the message.
+ *
+ * @param ioe Exception to clean up.
+ * @return Copy of the original exception with a clean message.
+ */
+ private static IOException getCleanException(IOException ioe) {
+ IOException ret = null;
+
+ String msg = ioe.getMessage();
+ Throwable cause = ioe.getCause();
+ StackTraceElement[] stackTrace = ioe.getStackTrace();
+
+ // Clean the message by removing the stack trace
+ int index = msg.indexOf("\n");
+ if (index > 0) {
+ String[] msgSplit = msg.split("\n");
+ msg = msgSplit[0];
+
+ // Parse stack trace from the message
+ List<StackTraceElement> elements = new LinkedList<>();
+ for (int i=1; i<msgSplit.length; i++) {
+ String line = msgSplit[i];
+ Matcher matcher = STACK_TRACE_PATTERN.matcher(line);
+ if (matcher.find()) {
+ String declaringClass = matcher.group(1);
+ String methodName = matcher.group(2);
+ String fileName = matcher.group(3);
+ int lineNumber = Integer.parseInt(matcher.group(4));
+ StackTraceElement element = new StackTraceElement(
+ declaringClass, methodName, fileName, lineNumber);
+ elements.add(element);
+ }
+ }
+ stackTrace = elements.toArray(new StackTraceElement[elements.size()]);
+ }
+
+ // Create the new output exception
+ if (ioe instanceof RemoteException) {
+ RemoteException re = (RemoteException)ioe;
+ ret = new RemoteException(re.getClassName(), msg);
+ } else {
+ // Try the simple constructor and initialize the fields
+ Class<? extends IOException> ioeClass = ioe.getClass();
+ try {
+ Constructor<? extends IOException> constructor =
+ ioeClass.getDeclaredConstructor(String.class);
+ ret = constructor.newInstance(msg);
+ } catch (ReflectiveOperationException e) {
+ // If there are errors, just use the input one
+ LOG.error("Could not create exception {}", ioeClass.getSimpleName(), e);
+ ret = ioe;
+ }
+ }
+ if (ret != null) {
+ ret.initCause(cause);
+ ret.setStackTrace(stackTrace);
+ }
+
+ return ret;
+ }
+
+ /**
+ * Invokes a ClientProtocol method. Determines the target nameservice via a
+ * provided block.
+ *
+ * Re-throws exceptions generated by the remote RPC call as either
+ * RemoteException or IOException.
+ *
+ * @param block Block used to determine appropriate nameservice.
+ * @param method The remote method and parameters to invoke.
+ * @return The result of invoking the method.
+ * @throws IOException
+ */
+ public Object invokeSingle(final ExtendedBlock block, RemoteMethod method)
+ throws IOException {
+ String bpId = block.getBlockPoolId();
+ return invokeSingleBlockPool(bpId, method);
+ }
+
+ /**
+ * Invokes a ClientProtocol method. Determines the target nameservice using
+ * the block pool id.
+ *
+ * Re-throws exceptions generated by the remote RPC call as either
+ * RemoteException or IOException.
+ *
+ * @param bpId Block pool identifier.
+ * @param method The remote method and parameters to invoke.
+ * @return The result of invoking the method.
+ * @throws IOException
+ */
+ public Object invokeSingleBlockPool(final String bpId, RemoteMethod method)
+ throws IOException {
+ String nsId = getNameserviceForBlockPoolId(bpId);
+ return invokeSingle(nsId, method);
+ }
+
+ /**
+ * Invokes a ClientProtocol method against the specified namespace.
+ *
+ * Re-throws exceptions generated by the remote RPC call as either
+ * RemoteException or IOException.
+ *
+ * @param nsId Target namespace for the method.
+ * @param method The remote method and parameters to invoke.
+ * @return The result of invoking the method.
+ * @throws IOException
+ */
+ public Object invokeSingle(final String nsId, RemoteMethod method)
+ throws IOException {
+ UserGroupInformation ugi = RouterRpcServer.getRemoteUser();
+ List<? extends FederationNamenodeContext> nns =
+ getNamenodesForNameservice(nsId);
+ RemoteLocationContext loc = new RemoteLocation(nsId, "/");
+ return invokeMethod(ugi, nns, method.getMethod(), method.getParams(loc));
+ }
+
+ /**
+ * Invokes a single proxy call for a single location.
+ *
+ * Re-throws exceptions generated by the remote RPC call as either
+ * RemoteException or IOException.
+ *
+ * @param location RemoteLocation to invoke.
+ * @param remoteMethod The remote method and parameters to invoke.
+ * @return The result of invoking the method if successful.
+ * @throws IOException
+ */
+ public Object invokeSingle(final RemoteLocationContext location,
+ RemoteMethod remoteMethod) throws IOException {
+ List<RemoteLocationContext> locations = Collections.singletonList(location);
+ return invokeSequential(locations, remoteMethod);
+ }
+
+ /**
+ * Invokes sequential proxy calls to different locations. Continues to invoke
+ * calls until a call returns without throwing a remote exception.
+ *
+ * @param locations List of locations/nameservices to call concurrently.
+ * @param remoteMethod The remote method and parameters to invoke.
+ * @return The result of the first successful call, or if no calls are
+ * successful, the result of the last RPC call executed.
+ * @throws IOException if the success condition is not met and one of the RPC
+ * calls generated a remote exception.
+ */
+ public Object invokeSequential(
+ final List<? extends RemoteLocationContext> locations,
+ final RemoteMethod remoteMethod) throws IOException {
+ return invokeSequential(locations, remoteMethod, null, null);
+ }
+
+ /**
+ * Invokes sequential proxy calls to different locations. Continues to invoke
+ * calls until the success condition is met, or until all locations have been
+ * attempted.
+ *
+ * The success condition may be specified by:
+ * <ul>
+ * <li>An expected result class
+ * <li>An expected result value
+ * </ul>
+ *
+ * If no expected result class/values are specified, the success condition is
+ * a call that does not throw a remote exception.
+ *
+ * @param locations List of locations/nameservices to call concurrently.
+ * @param remoteMethod The remote method and parameters to invoke.
+ * @param expectedResultClass In order to be considered a positive result, the
+ * return type must be of this class.
+ * @param expectedResultValue In order to be considered a positive result, the
+ * return value must equal the value of this object.
+ * @return The result of the first successful call, or if no calls are
+ * successful, the result of the first RPC call executed.
+ * @throws IOException if the success condition is not met, return the first
+ * remote exception generated.
+ */
+ public Object invokeSequential(
+ final List<? extends RemoteLocationContext> locations,
+ final RemoteMethod remoteMethod, Class<?> expectedResultClass,
+ Object expectedResultValue) throws IOException {
+
+ final UserGroupInformation ugi = RouterRpcServer.getRemoteUser();
+ final Method m = remoteMethod.getMethod();
+ IOException firstThrownException = null;
+ IOException lastThrownException = null;
+ Object firstResult = null;
+ // Invoke in priority order
+ for (final RemoteLocationContext loc : locations) {
+ String ns = loc.getNameserviceId();
+ List<? extends FederationNamenodeContext> namenodes =
+ getNamenodesForNameservice(ns);
+ try {
+ Object[] params = remoteMethod.getParams(loc);
+ Object result = invokeMethod(ugi, namenodes, m, params);
+ // Check if the result is what we expected
+ if (isExpectedClass(expectedResultClass, result) &&
+ isExpectedValue(expectedResultValue, result)) {
+ // Valid result, stop here
+ return result;
+ }
+ if (firstResult == null) {
+ firstResult = result;
+ }
+ } catch (IOException ioe) {
+ // Record it and move on
+ lastThrownException = (IOException) ioe;
+ if (firstThrownException == null) {
+ firstThrownException = lastThrownException;
+ }
+ } catch (Exception e) {
+ // Unusual error, ClientProtocol calls always use IOException (or
+ // RemoteException). Re-wrap in IOException for compatibility with
+ // ClientProtcol.
+ LOG.error("Unexpected exception {} proxying {} to {}",
+ e.getClass(), m.getName(), ns, e);
+ lastThrownException = new IOException(
+ "Unexpected exception proxying API " + e.getMessage(), e);
+ if (firstThrownException == null) {
+ firstThrownException = lastThrownException;
+ }
+ }
+ }
+
+ if (firstThrownException != null) {
+ // re-throw the last exception thrown for compatibility
+ throw firstThrownException;
+ }
+ // Return the last result, whether it is the value we are looking for or a
+ return firstResult;
+ }
+
+ /**
+ * Checks if a result matches the required result class.
+ *
+ * @param expectedResultClass Required result class, null to skip the check.
+ * @param result The result to check.
+ * @return True if the result is an instance of the required class or if the
+ * expected class is null.
+ */
+ private static boolean isExpectedClass(Class<?> expectedClass, Object clazz) {
+ if (expectedClass == null) {
+ return true;
+ } else if (clazz == null) {
+ return false;
+ } else {
+ return expectedClass.isInstance(clazz);
+ }
+ }
+
+ /**
+ * Checks if a result matches the expected value.
+ *
+ * @param expectedResultValue The expected value, null to skip the check.
+ * @param result The result to check.
+ * @return True if the result is equals to the expected value or if the
+ * expected value is null.
+ */
+ private static boolean isExpectedValue(Object expectedValue, Object value) {
+ if (expectedValue == null) {
+ return true;
+ } else if (value == null) {
+ return false;
+ } else {
+ return value.equals(expectedValue);
+ }
+ }
+
+ /**
+ * Invokes multiple concurrent proxy calls to different clients. Returns an
+ * array of results.
+ *
+ * Re-throws exceptions generated by the remote RPC call as either
+ * RemoteException or IOException.
+ *
+ * @param locations List of remote locations to call concurrently.
+ * @param remoteMethod The remote method and parameters to invoke.
+ * @param requireResponse If true an exception will be thrown if all calls do
+ * not complete. If false exceptions are ignored and all data results
+ * successfully received are returned.
+ * @param standby If the requests should go to the standby namenodes too.
+ * @return Result of invoking the method per subcluster: nsId -> result.
+ * @throws IOException If requiredResponse=true and any of the calls throw an
+ * exception.
+ */
+ @SuppressWarnings("unchecked")
+ public <T extends RemoteLocationContext> Map<T, Object> invokeConcurrent(
+ final Collection<T> locations, final RemoteMethod method,
+ boolean requireResponse, boolean standby) throws IOException {
+
+ final UserGroupInformation ugi = RouterRpcServer.getRemoteUser();
+ final Method m = method.getMethod();
+
+ if (locations.size() == 1) {
+ // Shortcut, just one call
+ T location = locations.iterator().next();
+ String ns = location.getNameserviceId();
+ final List<? extends FederationNamenodeContext> namenodes =
+ getNamenodesForNameservice(ns);
+ Object[] paramList = method.getParams(location);
+ Object result = invokeMethod(ugi, namenodes, m, paramList);
+ return Collections.singletonMap(location, result);
+ }
+
+ List<T> orderedLocations = new LinkedList<>();
+ Set<Callable<Object>> callables = new HashSet<>();
+ for (final T location : locations) {
+ String nsId = location.getNameserviceId();
+ final List<? extends FederationNamenodeContext> namenodes =
+ getNamenodesForNameservice(nsId);
+ final Object[] paramList = method.getParams(location);
+ if (standby) {
+ // Call the objectGetter to all NNs (including standby)
+ for (final FederationNamenodeContext nn : namenodes) {
+ String nnId = nn.getNamenodeId();
+ final List<FederationNamenodeContext> nnList =
+ Collections.singletonList(nn);
+ T nnLocation = location;
+ if (location instanceof RemoteLocation) {
+ nnLocation = (T)new RemoteLocation(nsId, nnId, location.getDest());
+ }
+ orderedLocations.add(nnLocation);
+ callables.add(new Callable<Object>() {
+ public Object call() throws Exception {
+ return invokeMethod(ugi, nnList, m, paramList);
+ }
+ });
+ }
+ } else {
+ // Call the objectGetter in order of nameservices in the NS list
+ orderedLocations.add(location);
+ callables.add(new Callable<Object>() {
+ public Object call() throws Exception {
+ return invokeMethod(ugi, namenodes, m, paramList);
+ }
+ });
+ }
+ }
+
+ try {
+ List<Future<Object>> futures = executorService.invokeAll(callables);
+ Map<T, Object> results = new TreeMap<>();
+ Map<T, IOException> exceptions = new TreeMap<>();
+ for (int i=0; i<futures.size(); i++) {
+ T location = orderedLocations.get(i);
+ try {
+ Future<Object> future = futures.get(i);
+ Object result = future.get();
+ results.put(location, result);
+ } catch (ExecutionException ex) {
+ Throwable cause = ex.getCause();
+ LOG.debug("Canot execute {} in {}: {}",
+ m.getName(), location, cause.getMessage());
+
+ // Convert into IOException if needed
+ IOException ioe = null;
+ if (cause instanceof IOException) {
+ ioe = (IOException) cause;
+ } else {
+ ioe = new IOException("Unhandled exception while proxying API " +
+ m.getName() + ": " + cause.getMessage(), cause);
+ }
+
+ // Response from all servers required, use this error.
+ if (requireResponse) {
+ throw ioe;
+ }
+
+ // Store the exceptions
+ exceptions.put(location, ioe);
+ }
+ }
+
+ // Throw the exception for the first location if there are no results
+ if (results.isEmpty()) {
+ T location = orderedLocations.get(0);
+ IOException ioe = exceptions.get(location);
+ if (ioe != null) {
+ throw ioe;
+ }
+ }
+
+ return results;
+ } catch (InterruptedException ex) {
+ LOG.error("Unexpected error while invoking API: {}", ex.getMessage());
+ throw new IOException(
+ "Unexpected error while invoking API " + ex.getMessage(), ex);
+ }
+ }
+
+ /**
+ * Get a prioritized list of NNs that share the same nameservice ID (in the
+ * same namespace). NNs that are reported as ACTIVE will be first in the list.
+ *
+ * @param nameserviceId The nameservice ID for the namespace.
+ * @return A prioritized list of NNs to use for communication.
+ * @throws IOException If a NN cannot be located for the nameservice ID.
+ */
+ private List<? extends FederationNamenodeContext> getNamenodesForNameservice(
+ final String nsId) throws IOException {
+
+ final List<? extends FederationNamenodeContext> namenodes =
+ namenodeResolver.getNamenodesForNameserviceId(nsId);
+
+ if (namenodes == null || namenodes.isEmpty()) {
+ throw new IOException("Cannot locate a registered namenode for " + nsId +
+ " from " + this.routerId);
+ }
+ return namenodes;
+ }
+
+ /**
+ * Get a prioritized list of NNs that share the same block pool ID (in the
+ * same namespace). NNs that are reported as ACTIVE will be first in the list.
+ *
+ * @param blockPoolId The blockpool ID for the namespace.
+ * @return A prioritized list of NNs to use for communication.
+ * @throws IOException If a NN cannot be located for the block pool ID.
+ */
+ private List<? extends FederationNamenodeContext> getNamenodesForBlockPoolId(
+ final String bpId) throws IOException {
+
+ List<? extends FederationNamenodeContext> namenodes =
+ namenodeResolver.getNamenodesForBlockPoolId(bpId);
+
+ if (namenodes == null || namenodes.isEmpty()) {
+ throw new IOException("Cannot locate a registered namenode for " + bpId +
+ " from " + this.routerId);
+ }
+ return namenodes;
+ }
+
+ /**
+ * Get the nameservice identifier for a block pool.
+ *
+ * @param bpId Identifier of the block pool.
+ * @return Nameservice identifier.
+ * @throws IOException If a NN cannot be located for the block pool ID.
+ */
+ private String getNameserviceForBlockPoolId(final String bpId)
+ throws IOException {
+ List<? extends FederationNamenodeContext> namenodes =
+ getNamenodesForBlockPoolId(bpId);
+ FederationNamenodeContext namenode = namenodes.get(0);
+ return namenode.getNameserviceId();
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/40] hadoop git commit: HDFS-12376. Enable JournalNode Sync by
default. Contributed by Hanisha Koneru.
Posted by in...@apache.org.
HDFS-12376. Enable JournalNode Sync by default. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5ff74167
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5ff74167
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5ff74167
Branch: refs/heads/HDFS-10467
Commit: 5ff74167ddd378a738898425c4462f3432f9c31d
Parents: 2adf8be
Author: Arpit Agarwal <ar...@apache.org>
Authored: Thu Sep 7 08:18:28 2017 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu Sep 7 08:18:28 2017 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +-
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ff74167/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 17cabad..bc7b716 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -924,7 +924,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_JOURNALNODE_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY = "dfs.journalnode.kerberos.internal.spnego.principal";
public static final String DFS_JOURNALNODE_ENABLE_SYNC_KEY =
"dfs.journalnode.enable.sync";
- public static final boolean DFS_JOURNALNODE_ENABLE_SYNC_DEFAULT = false;
+ public static final boolean DFS_JOURNALNODE_ENABLE_SYNC_DEFAULT = true;
public static final String DFS_JOURNALNODE_SYNC_INTERVAL_KEY =
"dfs.journalnode.sync.interval";
public static final long DFS_JOURNALNODE_SYNC_INTERVAL_DEFAULT = 2*60*1000L;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ff74167/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index bbe2eca..d58e54e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -3933,7 +3933,7 @@
<property>
<name>dfs.journalnode.enable.sync</name>
- <value>false</value>
+ <value>true</value>
<description>
If true, the journal nodes wil sync with each other. The journal nodes
will periodically gossip with other journal nodes to compare edit log
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/40] hadoop git commit: YARN-5328. Plan/ResourceAllocation data
structure enhancements required to support recurring reservations in
ReservationSystem.
Posted by in...@apache.org.
YARN-5328. Plan/ResourceAllocation data structure enhancements required to support recurring reservations in ReservationSystem.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b6e7d136
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b6e7d136
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b6e7d136
Branch: refs/heads/HDFS-10467
Commit: b6e7d1369690eaf50ce9ea7968f91a72ecb74de0
Parents: e3345e9
Author: Subru Krishnan <su...@apache.org>
Authored: Wed Sep 6 16:46:01 2017 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Wed Sep 6 16:46:01 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 6 +
.../yarn/conf/TestYarnConfigurationFields.java | 4 +-
.../reservation/AbstractReservationSystem.java | 90 ++--
.../AbstractSchedulerPlanFollower.java | 183 ++++----
.../reservation/InMemoryPlan.java | 400 ++++++++++++-----
.../InMemoryReservationAllocation.java | 36 +-
.../reservation/NoOverCommitPolicy.java | 2 +-
.../PeriodicRLESparseResourceAllocation.java | 130 ++++--
.../resourcemanager/reservation/PlanEdit.java | 24 +-
.../resourcemanager/reservation/PlanView.java | 94 ++--
.../RLESparseResourceAllocation.java | 112 ++---
.../reservation/ReservationAllocation.java | 60 ++-
.../reservation/ReservationInputValidator.java | 134 +++---
.../reservation/ReservationSystem.java | 6 +-
.../reservation/SharingPolicy.java | 13 +-
.../reservation/planning/Planner.java | 2 +-
.../reservation/planning/PlanningAlgorithm.java | 2 +-
.../reservation/planning/StageAllocator.java | 2 +-
.../planning/StageAllocatorGreedy.java | 2 +-
.../planning/StageAllocatorGreedyRLE.java | 5 +-
.../planning/StageAllocatorLowCostAligned.java | 4 +-
.../reservation/ReservationSystemTestUtil.java | 135 +++---
.../reservation/TestInMemoryPlan.java | 431 ++++++++++++-------
...TestPeriodicRLESparseResourceAllocation.java | 109 +++--
.../TestRLESparseResourceAllocation.java | 122 +++---
.../planning/TestSimpleCapacityReplanner.java | 8 +-
26 files changed, 1340 insertions(+), 776 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e7d136/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 4944821..27ca957 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -262,6 +262,12 @@ public class YarnConfiguration extends Configuration {
public static final long DEFAULT_RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP =
1000L;
+ /** The maximum periodicity for the Reservation System. */
+ public static final String RM_RESERVATION_SYSTEM_MAX_PERIODICITY =
+ RM_PREFIX + "reservation-system.max-periodicity";
+ public static final long DEFAULT_RM_RESERVATION_SYSTEM_MAX_PERIODICITY =
+ 86400000L;
+
/**
* Enable periodic monitor threads.
* @see #RM_SCHEDULER_MONITOR_POLICIES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e7d136/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index bd7bf93..1d3111c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.conf.TestConfigurationFieldsBase;
*/
public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
- @SuppressWarnings("deprecation")
+ @SuppressWarnings({"deprecation", "methodlength"})
@Override
public void initializeMemberVariables() {
xmlFilename = new String("yarn-default.xml");
@@ -69,6 +69,8 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
configurationPropsToSkipCompare.add(YarnConfiguration
.YARN_SECURITY_SERVICE_AUTHORIZATION_COLLECTOR_NODEMANAGER_PROTOCOL);
configurationPropsToSkipCompare.add(YarnConfiguration.CURATOR_LEADER_ELECTOR);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.RM_RESERVATION_SYSTEM_MAX_PERIODICITY);
// Federation default configs to be ignored
configurationPropsToSkipCompare
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e7d136/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
index 5ef4912..5b8772c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
@@ -18,6 +18,17 @@
package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
@@ -46,17 +57,6 @@ import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
/**
* This is the implementation of {@link ReservationSystem} based on the
* {@link ResourceScheduler}
@@ -66,8 +66,8 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
public abstract class AbstractReservationSystem extends AbstractService
implements ReservationSystem {
- private static final Logger LOG = LoggerFactory
- .getLogger(AbstractReservationSystem.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AbstractReservationSystem.class);
// private static final String DEFAULT_CAPACITY_SCHEDULER_PLAN
@@ -103,6 +103,8 @@ public abstract class AbstractReservationSystem extends AbstractService
private boolean isRecoveryEnabled = false;
+ private long maxPeriodicity;
+
/**
* Construct the service.
*
@@ -143,36 +145,41 @@ public abstract class AbstractReservationSystem extends AbstractService
this.conf = conf;
scheduler = rmContext.getScheduler();
// Get the plan step size
- planStepSize =
- conf.getTimeDuration(
- YarnConfiguration.RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP,
- YarnConfiguration.DEFAULT_RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP,
- TimeUnit.MILLISECONDS);
+ planStepSize = conf.getTimeDuration(
+ YarnConfiguration.RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP,
+ YarnConfiguration.DEFAULT_RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP,
+ TimeUnit.MILLISECONDS);
if (planStepSize < 0) {
planStepSize =
YarnConfiguration.DEFAULT_RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP;
}
+ maxPeriodicity =
+ conf.getLong(YarnConfiguration.RM_RESERVATION_SYSTEM_MAX_PERIODICITY,
+ YarnConfiguration.DEFAULT_RM_RESERVATION_SYSTEM_MAX_PERIODICITY);
+ if (maxPeriodicity <= 0) {
+ maxPeriodicity =
+ YarnConfiguration.DEFAULT_RM_RESERVATION_SYSTEM_MAX_PERIODICITY;
+ }
// Create a plan corresponding to every reservable queue
Set<String> planQueueNames = scheduler.getPlanQueues();
for (String planQueueName : planQueueNames) {
Plan plan = initializePlan(planQueueName);
plans.put(planQueueName, plan);
}
- isRecoveryEnabled = conf.getBoolean(
- YarnConfiguration.RECOVERY_ENABLED,
+ isRecoveryEnabled = conf.getBoolean(YarnConfiguration.RECOVERY_ENABLED,
YarnConfiguration.DEFAULT_RM_RECOVERY_ENABLED);
if (conf.getBoolean(YarnConfiguration.YARN_RESERVATION_ACL_ENABLE,
- YarnConfiguration.DEFAULT_YARN_RESERVATION_ACL_ENABLE) &&
- conf.getBoolean(YarnConfiguration.YARN_ACL_ENABLE,
- YarnConfiguration.DEFAULT_YARN_ACL_ENABLE)) {
+ YarnConfiguration.DEFAULT_YARN_RESERVATION_ACL_ENABLE)
+ && conf.getBoolean(YarnConfiguration.YARN_ACL_ENABLE,
+ YarnConfiguration.DEFAULT_YARN_ACL_ENABLE)) {
reservationsACLsManager = new ReservationsACLsManager(scheduler, conf);
}
}
private void loadPlan(String planName,
Map<ReservationId, ReservationAllocationStateProto> reservations)
- throws PlanningException {
+ throws PlanningException {
Plan plan = plans.get(planName);
Resource minAllocation = getMinAllocation();
ResourceCalculator rescCalculator = getResourceCalculator();
@@ -248,8 +255,8 @@ public abstract class AbstractReservationSystem extends AbstractService
Class<?> planFollowerPolicyClazz =
conf.getClassByName(planFollowerPolicyClassName);
if (PlanFollower.class.isAssignableFrom(planFollowerPolicyClazz)) {
- return (PlanFollower) ReflectionUtils.newInstance(
- planFollowerPolicyClazz, conf);
+ return (PlanFollower) ReflectionUtils
+ .newInstance(planFollowerPolicyClazz, conf);
} else {
throw new YarnRuntimeException("Class: " + planFollowerPolicyClassName
+ " not instance of " + PlanFollower.class.getCanonicalName());
@@ -257,7 +264,8 @@ public abstract class AbstractReservationSystem extends AbstractService
} catch (ClassNotFoundException e) {
throw new YarnRuntimeException(
"Could not instantiate PlanFollowerPolicy: "
- + planFollowerPolicyClassName, e);
+ + planFollowerPolicyClassName,
+ e);
}
}
@@ -371,9 +379,8 @@ public abstract class AbstractReservationSystem extends AbstractService
public ReservationId getNewReservationId() {
writeLock.lock();
try {
- ReservationId resId =
- ReservationId.newInstance(ResourceManager.getClusterTimeStamp(),
- resCounter.incrementAndGet());
+ ReservationId resId = ReservationId.newInstance(
+ ResourceManager.getClusterTimeStamp(), resCounter.incrementAndGet());
LOG.info("Allocated new reservationId: " + resId);
return resId;
} finally {
@@ -390,8 +397,11 @@ public abstract class AbstractReservationSystem extends AbstractService
* Get the default reservation system corresponding to the scheduler
*
* @param scheduler the scheduler for which the reservation system is required
+ *
+ * @return the {@link ReservationSystem} based on the configured scheduler
*/
- public static String getDefaultReservationSystem(ResourceScheduler scheduler) {
+ public static String getDefaultReservationSystem(
+ ResourceScheduler scheduler) {
if (scheduler instanceof CapacityScheduler) {
return CapacityReservationSystem.class.getName();
} else if (scheduler instanceof FairScheduler) {
@@ -409,12 +419,11 @@ public abstract class AbstractReservationSystem extends AbstractService
Resource maxAllocation = getMaxAllocation();
ResourceCalculator rescCalc = getResourceCalculator();
Resource totCap = getPlanQueueCapacity(planQueueName);
- Plan plan =
- new InMemoryPlan(getRootQueueMetrics(), adPolicy,
- getAgent(planQueuePath), totCap, planStepSize, rescCalc,
- minAllocation, maxAllocation, planQueueName,
- getReplanner(planQueuePath), getReservationSchedulerConfiguration()
- .getMoveOnExpiry(planQueuePath), rmContext);
+ Plan plan = new InMemoryPlan(getRootQueueMetrics(), adPolicy,
+ getAgent(planQueuePath), totCap, planStepSize, rescCalc, minAllocation,
+ maxAllocation, planQueueName, getReplanner(planQueuePath),
+ getReservationSchedulerConfiguration().getMoveOnExpiry(planQueuePath),
+ maxPeriodicity, rmContext);
LOG.info("Initialized plan {} based on reservable queue {}",
plan.toString(), planQueueName);
return plan;
@@ -477,8 +486,8 @@ public abstract class AbstractReservationSystem extends AbstractService
Class<?> admissionPolicyClazz =
conf.getClassByName(admissionPolicyClassName);
if (SharingPolicy.class.isAssignableFrom(admissionPolicyClazz)) {
- return (SharingPolicy) ReflectionUtils.newInstance(
- admissionPolicyClazz, conf);
+ return (SharingPolicy) ReflectionUtils.newInstance(admissionPolicyClazz,
+ conf);
} else {
throw new YarnRuntimeException("Class: " + admissionPolicyClassName
+ " not instance of " + SharingPolicy.class.getCanonicalName());
@@ -493,8 +502,7 @@ public abstract class AbstractReservationSystem extends AbstractService
return this.reservationsACLsManager;
}
- protected abstract ReservationSchedulerConfiguration
- getReservationSchedulerConfiguration();
+ protected abstract ReservationSchedulerConfiguration getReservationSchedulerConfiguration();
protected abstract String getPlanQueuePath(String planQueueName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e7d136/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java
index 90357e3..9b6a0b0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java
@@ -18,6 +18,14 @@
package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -33,24 +41,17 @@ import org.apache.hadoop.yarn.util.resource.Resources;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
- private static final Logger LOG = LoggerFactory
- .getLogger(AbstractSchedulerPlanFollower.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AbstractSchedulerPlanFollower.class);
protected Collection<Plan> plans = new ArrayList<Plan>();
protected YarnScheduler scheduler;
protected Clock clock;
@Override
- public void init(Clock clock, ResourceScheduler sched, Collection<Plan> plans) {
+ public void init(Clock clock, ResourceScheduler sched,
+ Collection<Plan> plans) {
this.clock = clock;
this.scheduler = sched;
this.plans.addAll(plans);
@@ -71,7 +72,7 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
@Override
public synchronized void synchronizePlan(Plan plan, boolean shouldReplan) {
- String planQueueName = plan.getQueueName();
+ String planQueueName = plan.getQueueName();
if (LOG.isDebugEnabled()) {
LOG.debug("Running plan follower edit policy for plan: " + planQueueName);
}
@@ -82,12 +83,14 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
now += step - (now % step);
}
Queue planQueue = getPlanQueue(planQueueName);
- if (planQueue == null) return;
+ if (planQueue == null) {
+ return;
+ }
// first we publish to the plan the current availability of resources
Resource clusterResources = scheduler.getClusterResource();
- Resource planResources = getPlanResources(plan, planQueue,
- clusterResources);
+ Resource planResources =
+ getPlanResources(plan, planQueue, clusterResources);
Set<ReservationAllocation> currentReservations =
plan.getReservationsAtTime(now);
Set<String> curReservationNames = new HashSet<String>();
@@ -95,12 +98,11 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
int numRes = getReservedResources(now, currentReservations,
curReservationNames, reservedResources);
// create the default reservation queue if it doesnt exist
- String defReservationId = getReservationIdFromQueueName(planQueueName) +
- ReservationConstants.DEFAULT_QUEUE_SUFFIX;
- String defReservationQueue = getReservationQueueName(planQueueName,
- defReservationId);
- createDefaultReservationQueue(planQueueName, planQueue,
- defReservationId);
+ String defReservationId = getReservationIdFromQueueName(planQueueName)
+ + ReservationConstants.DEFAULT_QUEUE_SUFFIX;
+ String defReservationQueue =
+ getReservationQueueName(planQueueName, defReservationId);
+ createDefaultReservationQueue(planQueueName, planQueue, defReservationId);
curReservationNames.add(defReservationId);
// if the resources dedicated to this plan has shrunk invoke replanner
boolean shouldResize = false;
@@ -149,10 +151,8 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
// sort allocations from the one giving up the most resources, to the
// one asking for the most avoid order-of-operation errors that
// temporarily violate 100% capacity bound
- List<ReservationAllocation> sortedAllocations =
- sortByDelta(
- new ArrayList<ReservationAllocation>(currentReservations), now,
- plan);
+ List<ReservationAllocation> sortedAllocations = sortByDelta(
+ new ArrayList<ReservationAllocation>(currentReservations), now, plan);
for (ReservationAllocation res : sortedAllocations) {
String currResId = res.getReservationId().toString();
if (curReservationNames.contains(currResId)) {
@@ -163,10 +163,9 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
if (planResources.getMemorySize() > 0
&& planResources.getVirtualCores() > 0) {
if (shouldResize) {
- capToAssign =
- calculateReservationToPlanProportion(
- plan.getResourceCalculator(), planResources,
- reservedResources, capToAssign);
+ capToAssign = calculateReservationToPlanProportion(
+ plan.getResourceCalculator(), planResources, reservedResources,
+ capToAssign);
}
targetCapacity =
calculateReservationToPlanRatio(plan.getResourceCalculator(),
@@ -185,7 +184,8 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
maxCapacity = targetCapacity;
}
try {
- setQueueEntitlement(planQueueName, currResId, targetCapacity, maxCapacity);
+ setQueueEntitlement(planQueueName, currResId, targetCapacity,
+ maxCapacity);
} catch (YarnException e) {
LOG.warn("Exception while trying to size reservation for plan: {}",
currResId, planQueueName, e);
@@ -196,9 +196,10 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
// compute the default queue capacity
float defQCap = 1.0f - totalAssignedCapacity;
if (LOG.isDebugEnabled()) {
- LOG.debug("PlanFollowerEditPolicyTask: total Plan Capacity: {} "
- + "currReservation: {} default-queue capacity: {}", planResources,
- numRes, defQCap);
+ LOG.debug(
+ "PlanFollowerEditPolicyTask: total Plan Capacity: {} "
+ + "currReservation: {} default-queue capacity: {}",
+ planResources, numRes, defQCap);
}
// set the default queue to eat-up all remaining capacity
try {
@@ -225,12 +226,11 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
}
protected void setQueueEntitlement(String planQueueName, String currResId,
- float targetCapacity,
- float maxCapacity) throws YarnException {
- String reservationQueueName = getReservationQueueName(planQueueName,
- currResId);
- scheduler.setEntitlement(reservationQueueName, new QueueEntitlement(
- targetCapacity, maxCapacity));
+ float targetCapacity, float maxCapacity) throws YarnException {
+ String reservationQueueName =
+ getReservationQueueName(planQueueName, currResId);
+ scheduler.setEntitlement(reservationQueueName,
+ new QueueEntitlement(targetCapacity, maxCapacity));
}
// Schedulers have different ways of naming queues. See YARN-2773
@@ -244,14 +244,21 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
* Then move all apps in the set of queues to the parent plan queue's default
* reservation queue if move is enabled. Finally cleanups the queue by killing
* any apps (if move is disabled or move failed) and removing the queue
+ *
+ * @param planQueueName the name of {@code PlanQueue}
+ * @param shouldMove flag to indicate if any running apps should be moved or
+ * killed
+ * @param toRemove the remnant apps to clean up
+ * @param defReservationQueue the default {@code ReservationQueue} of the
+ * {@link Plan}
*/
- protected void cleanupExpiredQueues(String planQueueName,
- boolean shouldMove, Set<String> toRemove, String defReservationQueue) {
+ protected void cleanupExpiredQueues(String planQueueName, boolean shouldMove,
+ Set<String> toRemove, String defReservationQueue) {
for (String expiredReservationId : toRemove) {
try {
// reduce entitlement to 0
- String expiredReservation = getReservationQueueName(planQueueName,
- expiredReservationId);
+ String expiredReservation =
+ getReservationQueueName(planQueueName, expiredReservationId);
setQueueEntitlement(planQueueName, expiredReservation, 0.0f, 0.0f);
if (shouldMove) {
moveAppsInQueueSync(expiredReservation, defReservationQueue);
@@ -275,7 +282,7 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
* reservation queue in a synchronous fashion
*/
private void moveAppsInQueueSync(String expiredReservation,
- String defReservationQueue) {
+ String defReservationQueue) {
List<ApplicationAttemptId> activeApps =
scheduler.getAppsInQueue(expiredReservation);
if (activeApps.isEmpty()) {
@@ -287,16 +294,16 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
scheduler.moveApplication(app.getApplicationId(), defReservationQueue);
} catch (YarnException e) {
LOG.warn(
- "Encountered unexpected error during migration of application: {}" +
- " from reservation: {}",
+ "Encountered unexpected error during migration of application: {}"
+ + " from reservation: {}",
app, expiredReservation, e);
}
}
}
- protected int getReservedResources(long now, Set<ReservationAllocation>
- currentReservations, Set<String> curReservationNames,
- Resource reservedResources) {
+ protected int getReservedResources(long now,
+ Set<ReservationAllocation> currentReservations,
+ Set<String> curReservationNames, Resource reservedResources) {
int numRes = 0;
if (currentReservations != null) {
numRes = currentReservations.size();
@@ -312,23 +319,30 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
* Sort in the order from the least new amount of resources asked (likely
* negative) to the highest. This prevents "order-of-operation" errors related
* to exceeding 100% capacity temporarily.
+ *
+ * @param currentReservations the currently active reservations
+ * @param now the current time
+ * @param plan the {@link Plan} that is being considered
+ *
+ * @return the sorted list of {@link ReservationAllocation}s
*/
protected List<ReservationAllocation> sortByDelta(
List<ReservationAllocation> currentReservations, long now, Plan plan) {
- Collections.sort(currentReservations, new ReservationAllocationComparator(
- now, this, plan));
+ Collections.sort(currentReservations,
+ new ReservationAllocationComparator(now, this, plan));
return currentReservations;
}
/**
- * Get queue associated with reservable queue named
- * @param planQueueName Name of the reservable queue
+ * Get queue associated with reservable queue named.
+ *
+ * @param planQueueName name of the reservable queue
* @return queue associated with the reservable queue
*/
protected abstract Queue getPlanQueue(String planQueueName);
/**
- * Resizes reservations based on currently available resources
+ * Resizes reservations based on currently available resources.
*/
private Resource calculateReservationToPlanProportion(
ResourceCalculator rescCalculator, Resource availablePlanResources,
@@ -338,7 +352,7 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
}
/**
- * Calculates ratio of reservationResources to planResources
+ * Calculates ratio of reservationResources to planResources.
*/
private float calculateReservationToPlanRatio(
ResourceCalculator rescCalculator, Resource clusterResources,
@@ -348,7 +362,7 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
}
/**
- * Check if plan resources are less than expected reservation resources
+ * Check if plan resources are less than expected reservation resources.
*/
private boolean arePlanResourcesLessThanReservations(
ResourceCalculator rescCalculator, Resource clusterResources,
@@ -358,38 +372,56 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
}
/**
- * Get a list of reservation queues for this planQueue
+ * Get a list of reservation queues for this planQueue.
+ *
+ * @param planQueue the queue for the current {@link Plan}
+ *
+ * @return the queues corresponding to the reservations
*/
protected abstract List<? extends Queue> getChildReservationQueues(
Queue planQueue);
/**
- * Add a new reservation queue for reservation currResId for this planQueue
+ * Add a new reservation queue for reservation currResId for this planQueue.
*/
- protected abstract void addReservationQueue(
- String planQueueName, Queue queue, String currResId);
+ protected abstract void addReservationQueue(String planQueueName, Queue queue,
+ String currResId);
/**
- * Creates the default reservation queue for use when no reservation is
- * used for applications submitted to this planQueue
+ * Creates the default reservation queue for use when no reservation is used
+ * for applications submitted to this planQueue.
+ *
+ * @param planQueueName name of the reservable queue
+ * @param queue the queue for the current {@link Plan}
+ * @param defReservationQueue name of the default {@code ReservationQueue}
*/
- protected abstract void createDefaultReservationQueue(
- String planQueueName, Queue queue, String defReservationQueue);
+ protected abstract void createDefaultReservationQueue(String planQueueName,
+ Queue queue, String defReservationQueue);
/**
- * Get plan resources for this planQueue
+ * Get plan resources for this planQueue.
+ *
+ * @param plan the current {@link Plan} being considered
+ * @param clusterResources the resources available in the cluster
+ *
+ * @return the resources allocated to the specified {@link Plan}
*/
- protected abstract Resource getPlanResources(
- Plan plan, Queue queue, Resource clusterResources);
+ protected abstract Resource getPlanResources(Plan plan, Queue queue,
+ Resource clusterResources);
/**
* Get reservation queue resources if it exists otherwise return null.
+ *
+ * @param plan the current {@link Plan} being considered
+ * @param reservationId the identifier of the reservation
+ *
+ * @return the resources allocated to the specified reservation
*/
protected abstract Resource getReservationQueueResourceIfExists(Plan plan,
ReservationId reservationId);
- private static class ReservationAllocationComparator implements
- Comparator<ReservationAllocation> {
+ private static class ReservationAllocationComparator
+ implements Comparator<ReservationAllocation> {
AbstractSchedulerPlanFollower planFollower;
long now;
Plan plan;
@@ -404,14 +436,12 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
private Resource getUnallocatedReservedResources(
ReservationAllocation reservation) {
Resource resResource;
- Resource reservationResource = planFollower
- .getReservationQueueResourceIfExists
- (plan, reservation.getReservationId());
+ Resource reservationResource =
+ planFollower.getReservationQueueResourceIfExists(plan,
+ reservation.getReservationId());
if (reservationResource != null) {
- resResource =
- Resources.subtract(
- reservation.getResourcesAtTime(now),
- reservationResource);
+ resResource = Resources.subtract(reservation.getResourcesAtTime(now),
+ reservationResource);
} else {
resResource = reservation.getResourcesAtTime(now);
}
@@ -428,4 +458,3 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
}
}
}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e7d136/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java
index 783fd09..9eb1820 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -33,9 +33,10 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation.RLEOperator;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation.RLEOperator;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.planning.Planner;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.planning.ReservationAgent;
@@ -64,9 +65,14 @@ public class InMemoryPlan implements Plan {
private RLESparseResourceAllocation rleSparseVector;
+ private PeriodicRLESparseResourceAllocation periodicRle;
+
private Map<String, RLESparseResourceAllocation> userResourceAlloc =
new HashMap<String, RLESparseResourceAllocation>();
+ private Map<String, RLESparseResourceAllocation> userPeriodicResourceAlloc =
+ new HashMap<String, RLESparseResourceAllocation>();
+
private Map<String, RLESparseResourceAllocation> userActiveReservationCount =
new HashMap<String, RLESparseResourceAllocation>();
@@ -96,15 +102,27 @@ public class InMemoryPlan implements Plan {
String queueName, Planner replanner, boolean getMoveOnExpiry,
RMContext rmContext) {
this(queueMetrics, policy, agent, totalCapacity, step, resCalc, minAlloc,
- maxAlloc, queueName, replanner, getMoveOnExpiry, rmContext,
- new UTCClock());
+ maxAlloc, queueName, replanner, getMoveOnExpiry,
+ YarnConfiguration.DEFAULT_RM_RESERVATION_SYSTEM_MAX_PERIODICITY,
+ rmContext);
}
public InMemoryPlan(QueueMetrics queueMetrics, SharingPolicy policy,
ReservationAgent agent, Resource totalCapacity, long step,
ResourceCalculator resCalc, Resource minAlloc, Resource maxAlloc,
String queueName, Planner replanner, boolean getMoveOnExpiry,
- RMContext rmContext, Clock clock) {
+ long maxPeriodicty, RMContext rmContext) {
+ this(queueMetrics, policy, agent, totalCapacity, step, resCalc, minAlloc,
+ maxAlloc, queueName, replanner, getMoveOnExpiry, maxPeriodicty,
+ rmContext, new UTCClock());
+ }
+
+ @SuppressWarnings("checkstyle:parameternumber")
+ public InMemoryPlan(QueueMetrics queueMetrics, SharingPolicy policy,
+ ReservationAgent agent, Resource totalCapacity, long step,
+ ResourceCalculator resCalc, Resource minAlloc, Resource maxAlloc,
+ String queueName, Planner replanner, boolean getMoveOnExpiry,
+ long maxPeriodicty, RMContext rmContext, Clock clock) {
this.queueMetrics = queueMetrics;
this.policy = policy;
this.agent = agent;
@@ -114,6 +132,8 @@ public class InMemoryPlan implements Plan {
this.minAlloc = minAlloc;
this.maxAlloc = maxAlloc;
this.rleSparseVector = new RLESparseResourceAllocation(resCalc);
+ this.periodicRle =
+ new PeriodicRLESparseResourceAllocation(resCalc, maxPeriodicty);
this.queueName = queueName;
this.replanner = replanner;
this.getMoveOnExpiry = getMoveOnExpiry;
@@ -126,6 +146,39 @@ public class InMemoryPlan implements Plan {
return queueMetrics;
}
+ private RLESparseResourceAllocation getUserRLEResourceAllocation(String user,
+ long period) {
+ RLESparseResourceAllocation resAlloc = null;
+ if (period > 0) {
+ if (userPeriodicResourceAlloc.containsKey(user)) {
+ resAlloc = userPeriodicResourceAlloc.get(user);
+ } else {
+ resAlloc = new PeriodicRLESparseResourceAllocation(resCalc,
+ periodicRle.getTimePeriod());
+ userPeriodicResourceAlloc.put(user, resAlloc);
+ }
+ } else {
+ if (userResourceAlloc.containsKey(user)) {
+ resAlloc = userResourceAlloc.get(user);
+ } else {
+ resAlloc = new RLESparseResourceAllocation(resCalc);
+ userResourceAlloc.put(user, resAlloc);
+ }
+ }
+ return resAlloc;
+ }
+
+ private void gcUserRLEResourceAllocation(String user, long period) {
+ if (period > 0) {
+ if (userPeriodicResourceAlloc.get(user).isEmpty()) {
+ userPeriodicResourceAlloc.remove(user);
+ }
+ } else {
+ if (userResourceAlloc.get(user).isEmpty()) {
+ userResourceAlloc.remove(user);
+ }
+ }
+ }
private void incrementAllocation(ReservationAllocation reservation) {
assert (readWriteLock.isWriteLockedByCurrentThread());
@@ -133,11 +186,10 @@ public class InMemoryPlan implements Plan {
reservation.getAllocationRequests();
// check if we have encountered the user earlier and if not add an entry
String user = reservation.getUser();
- RLESparseResourceAllocation resAlloc = userResourceAlloc.get(user);
- if (resAlloc == null) {
- resAlloc = new RLESparseResourceAllocation(resCalc);
- userResourceAlloc.put(user, resAlloc);
- }
+ long period = reservation.getPeriodicity();
+ RLESparseResourceAllocation resAlloc =
+ getUserRLEResourceAllocation(user, period);
+
RLESparseResourceAllocation resCount = userActiveReservationCount.get(user);
if (resCount == null) {
resCount = new RLESparseResourceAllocation(resCalc);
@@ -149,14 +201,43 @@ public class InMemoryPlan implements Plan {
for (Map.Entry<ReservationInterval, Resource> r : allocationRequests
.entrySet()) {
- resAlloc.addInterval(r.getKey(), r.getValue());
- rleSparseVector.addInterval(r.getKey(), r.getValue());
- if (Resources.greaterThan(resCalc, totalCapacity, r.getValue(),
- ZERO_RESOURCE)) {
- earliestActive = Math.min(earliestActive, r.getKey().getStartTime());
- latestActive = Math.max(latestActive, r.getKey().getEndTime());
+
+ if (period > 0L) {
+ for (int i = 0; i < periodicRle.getTimePeriod() / period; i++) {
+
+ long rStart = r.getKey().getStartTime() + i * period;
+ long rEnd = r.getKey().getEndTime() + i * period;
+
+ // handle wrap-around
+ if (rEnd > periodicRle.getTimePeriod()) {
+ long diff = rEnd - periodicRle.getTimePeriod();
+ rEnd = periodicRle.getTimePeriod();
+ ReservationInterval newInterval = new ReservationInterval(0, diff);
+ periodicRle.addInterval(newInterval, r.getValue());
+ resAlloc.addInterval(newInterval, r.getValue());
+ }
+
+ ReservationInterval newInterval =
+ new ReservationInterval(rStart, rEnd);
+ periodicRle.addInterval(newInterval, r.getValue());
+ resAlloc.addInterval(newInterval, r.getValue());
+ }
+
+ } else {
+ rleSparseVector.addInterval(r.getKey(), r.getValue());
+ resAlloc.addInterval(r.getKey(), r.getValue());
+ if (Resources.greaterThan(resCalc, totalCapacity, r.getValue(),
+ ZERO_RESOURCE)) {
+ earliestActive = Math.min(earliestActive, r.getKey().getStartTime());
+ latestActive = Math.max(latestActive, r.getKey().getEndTime());
+ }
}
}
+ // periodic reservations are active from start time and good till cancelled
+ if (period > 0L) {
+ earliestActive = reservation.getStartTime();
+ latestActive = Long.MAX_VALUE;
+ }
resCount.addInterval(new ReservationInterval(earliestActive, latestActive),
Resource.newInstance(1, 1));
}
@@ -166,27 +247,55 @@ public class InMemoryPlan implements Plan {
Map<ReservationInterval, Resource> allocationRequests =
reservation.getAllocationRequests();
String user = reservation.getUser();
- RLESparseResourceAllocation resAlloc = userResourceAlloc.get(user);
+ long period = reservation.getPeriodicity();
+ RLESparseResourceAllocation resAlloc =
+ getUserRLEResourceAllocation(user, period);
long earliestActive = Long.MAX_VALUE;
long latestActive = Long.MIN_VALUE;
for (Map.Entry<ReservationInterval, Resource> r : allocationRequests
.entrySet()) {
- resAlloc.removeInterval(r.getKey(), r.getValue());
- rleSparseVector.removeInterval(r.getKey(), r.getValue());
- if (Resources.greaterThan(resCalc, totalCapacity, r.getValue(),
- ZERO_RESOURCE)) {
- earliestActive = Math.min(earliestActive, r.getKey().getStartTime());
- latestActive = Math.max(latestActive, r.getKey().getEndTime());
+ if (period > 0L) {
+ for (int i = 0; i < periodicRle.getTimePeriod() / period; i++) {
+
+ long rStart = r.getKey().getStartTime() + i * period;
+ long rEnd = r.getKey().getEndTime() + i * period;
+
+ // handle wrap-around
+ if (rEnd > periodicRle.getTimePeriod()) {
+ long diff = rEnd - periodicRle.getTimePeriod();
+ rEnd = periodicRle.getTimePeriod();
+ ReservationInterval newInterval = new ReservationInterval(0, diff);
+ periodicRle.removeInterval(newInterval, r.getValue());
+ resAlloc.removeInterval(newInterval, r.getValue());
+ }
+
+ ReservationInterval newInterval =
+ new ReservationInterval(rStart, rEnd);
+ periodicRle.removeInterval(newInterval, r.getValue());
+ resAlloc.removeInterval(newInterval, r.getValue());
+ }
+ } else {
+ rleSparseVector.removeInterval(r.getKey(), r.getValue());
+ resAlloc.removeInterval(r.getKey(), r.getValue());
+ if (Resources.greaterThan(resCalc, totalCapacity, r.getValue(),
+ ZERO_RESOURCE)) {
+ earliestActive = Math.min(earliestActive, r.getKey().getStartTime());
+ latestActive = Math.max(latestActive, r.getKey().getEndTime());
+ }
}
}
- if (resAlloc.isEmpty()) {
- userResourceAlloc.remove(user);
- }
+ gcUserRLEResourceAllocation(user, period);
RLESparseResourceAllocation resCount = userActiveReservationCount.get(user);
- resCount.removeInterval(new ReservationInterval(earliestActive,
- latestActive), Resource.newInstance(1, 1));
+ // periodic reservations are active from start time and good till cancelled
+ if (period > 0L) {
+ earliestActive = reservation.getStartTime();
+ latestActive = Long.MAX_VALUE;
+ }
+ resCount.removeInterval(
+ new ReservationInterval(earliestActive, latestActive),
+ Resource.newInstance(1, 1));
if (resCount.isEmpty()) {
userActiveReservationCount.remove(user);
}
@@ -198,9 +307,9 @@ public class InMemoryPlan implements Plan {
if (currentReservations != null) {
Set<ReservationAllocation> flattenedReservations =
new TreeSet<ReservationAllocation>();
- for (Set<InMemoryReservationAllocation> reservationEntries :
- currentReservations.values()) {
- flattenedReservations.addAll(reservationEntries);
+ for (Set<InMemoryReservationAllocation> res : currentReservations
+ .values()) {
+ flattenedReservations.addAll(res);
}
return flattenedReservations;
} else {
@@ -218,19 +327,16 @@ public class InMemoryPlan implements Plan {
InMemoryReservationAllocation inMemReservation =
(InMemoryReservationAllocation) reservation;
if (inMemReservation.getUser() == null) {
- String errMsg =
- "The specified Reservation with ID "
- + inMemReservation.getReservationId()
- + " is not mapped to any user";
+ String errMsg = "The specified Reservation with ID "
+ + inMemReservation.getReservationId() + " is not mapped to any user";
LOG.error(errMsg);
throw new IllegalArgumentException(errMsg);
}
writeLock.lock();
try {
if (reservationTable.containsKey(inMemReservation.getReservationId())) {
- String errMsg =
- "The specified Reservation with ID "
- + inMemReservation.getReservationId() + " already exists";
+ String errMsg = "The specified Reservation with ID "
+ + inMemReservation.getReservationId() + " already exists";
LOG.error(errMsg);
throw new IllegalArgumentException(errMsg);
}
@@ -246,9 +352,8 @@ public class InMemoryPlan implements Plan {
getQueueName(), inMemReservation.getReservationId().toString());
}
}
- ReservationInterval searchInterval =
- new ReservationInterval(inMemReservation.getStartTime(),
- inMemReservation.getEndTime());
+ ReservationInterval searchInterval = new ReservationInterval(
+ inMemReservation.getStartTime(), inMemReservation.getEndTime());
Set<InMemoryReservationAllocation> reservations =
currentReservations.get(searchInterval);
if (reservations == null) {
@@ -280,9 +385,8 @@ public class InMemoryPlan implements Plan {
ReservationId resId = reservation.getReservationId();
ReservationAllocation currReservation = getReservationById(resId);
if (currReservation == null) {
- String errMsg =
- "The specified Reservation with ID " + resId
- + " does not exist in the plan";
+ String errMsg = "The specified Reservation with ID " + resId
+ + " does not exist in the plan";
LOG.error(errMsg);
throw new IllegalArgumentException(errMsg);
}
@@ -318,9 +422,8 @@ public class InMemoryPlan implements Plan {
private boolean removeReservation(ReservationAllocation reservation) {
assert (readWriteLock.isWriteLockedByCurrentThread());
- ReservationInterval searchInterval =
- new ReservationInterval(reservation.getStartTime(),
- reservation.getEndTime());
+ ReservationInterval searchInterval = new ReservationInterval(
+ reservation.getStartTime(), reservation.getEndTime());
Set<InMemoryReservationAllocation> reservations =
currentReservations.get(searchInterval);
if (reservations != null) {
@@ -337,16 +440,15 @@ public class InMemoryPlan implements Plan {
currentReservations.remove(searchInterval);
}
} else {
- String errMsg =
- "The specified Reservation with ID " + reservation.getReservationId()
- + " does not exist in the plan";
+ String errMsg = "The specified Reservation with ID "
+ + reservation.getReservationId() + " does not exist in the plan";
LOG.error(errMsg);
throw new IllegalArgumentException(errMsg);
}
reservationTable.remove(reservation.getReservationId());
decrementAllocation(reservation);
LOG.info("Sucessfully deleted reservation: {} in plan.",
- reservation.getReservationId());
+ reservation.getReservationId());
return true;
}
@@ -356,9 +458,8 @@ public class InMemoryPlan implements Plan {
try {
ReservationAllocation reservation = getReservationById(reservationID);
if (reservation == null) {
- String errMsg =
- "The specified Reservation with ID " + reservationID
- + " does not exist in the plan";
+ String errMsg = "The specified Reservation with ID " + reservationID
+ + " does not exist in the plan";
LOG.error(errMsg);
throw new IllegalArgumentException(errMsg);
}
@@ -453,66 +554,90 @@ public class InMemoryPlan implements Plan {
long start, long end) {
readLock.lock();
try {
+ // merge periodic and non-periodic allocations
RLESparseResourceAllocation userResAlloc = userResourceAlloc.get(user);
+ RLESparseResourceAllocation userPeriodicResAlloc =
+ userPeriodicResourceAlloc.get(user);
+ if (userResAlloc != null && userPeriodicResAlloc != null) {
+ return RLESparseResourceAllocation.merge(resCalc, totalCapacity,
+ userResAlloc, userPeriodicResAlloc, RLEOperator.add, start, end);
+ }
if (userResAlloc != null) {
return userResAlloc.getRangeOverlapping(start, end);
- } else {
- return new RLESparseResourceAllocation(resCalc);
}
+ if (userPeriodicResAlloc != null) {
+ return userPeriodicResAlloc.getRangeOverlapping(start, end);
+ }
+ } catch (PlanningException e) {
+ LOG.warn("Exception while trying to merge periodic"
+ + " and non-periodic user allocations: {}", e.getMessage(), e);
} finally {
readLock.unlock();
}
+ return new RLESparseResourceAllocation(resCalc);
}
@Override
public Resource getTotalCommittedResources(long t) {
readLock.lock();
try {
- return rleSparseVector.getCapacityAtTime(t);
+ return Resources.add(rleSparseVector.getCapacityAtTime(t),
+ periodicRle.getCapacityAtTime(t));
} finally {
readLock.unlock();
}
}
@Override
- public Set<ReservationAllocation> getReservations(ReservationId
- reservationID, ReservationInterval interval) {
+ public Set<ReservationAllocation> getReservations(ReservationId reservationID,
+ ReservationInterval interval) {
return getReservations(reservationID, interval, null);
}
@Override
- public Set<ReservationAllocation> getReservations(ReservationId
- reservationID, ReservationInterval interval, String user) {
+ public Set<ReservationAllocation> getReservations(ReservationId reservationID,
+ ReservationInterval interval, String user) {
if (reservationID != null) {
ReservationAllocation allocation = getReservationById(reservationID);
- if (allocation == null){
+ if (allocation == null) {
return Collections.emptySet();
}
return Collections.singleton(allocation);
}
- long startTime = interval == null? 0 : interval.getStartTime();
- long endTime = interval == null? Long.MAX_VALUE : interval.getEndTime();
+ long startTime = interval == null ? 0 : interval.getStartTime();
+ long endTime = interval == null ? Long.MAX_VALUE : interval.getEndTime();
ReservationInterval searchInterval =
- new ReservationInterval(endTime, Long.MAX_VALUE);
+ new ReservationInterval(endTime, Long.MAX_VALUE);
readLock.lock();
try {
- SortedMap<ReservationInterval, Set<InMemoryReservationAllocation>>
- reservations = currentReservations.headMap(searchInterval, true);
- if (!reservations.isEmpty()) {
- Set<ReservationAllocation> flattenedReservations =
- new HashSet<>();
- for (Set<InMemoryReservationAllocation> reservationEntries :
- reservations.values()) {
- for (InMemoryReservationAllocation res : reservationEntries) {
- if (res.getEndTime() > startTime) {
- if (user != null && !user.isEmpty()
- && !res.getUser().equals(user)) {
- continue;
+ SortedMap<ReservationInterval, Set<InMemoryReservationAllocation>> res =
+ currentReservations.headMap(searchInterval, true);
+ if (!res.isEmpty()) {
+ Set<ReservationAllocation> flattenedReservations = new HashSet<>();
+ for (Set<InMemoryReservationAllocation> resEntries : res.values()) {
+ for (InMemoryReservationAllocation reservation : resEntries) {
+ // validate user
+ if (user != null && !user.isEmpty()
+ && !reservation.getUser().equals(user)) {
+ continue;
+ }
+ // handle periodic reservations
+ long period = reservation.getPeriodicity();
+ if (period > 0) {
+ long t = endTime % period;
+ // check for both contained and wrap-around reservations
+ if ((t - startTime) * (t - endTime)
+ * (startTime - endTime) >= 0) {
+ flattenedReservations.add(reservation);
+ }
+ } else {
+ // check for non-periodic reservations
+ if (reservation.getEndTime() > startTime) {
+ flattenedReservations.add(reservation);
}
- flattenedReservations.add(res);
}
}
}
@@ -550,36 +675,82 @@ public class InMemoryPlan implements Plan {
@Override
public RLESparseResourceAllocation getAvailableResourceOverTime(String user,
- ReservationId oldId, long start, long end) throws PlanningException {
+ ReservationId oldId, long start, long end, long period)
+ throws PlanningException {
readLock.lock();
try {
- // create RLE of totCapacity
- TreeMap<Long, Resource> totAvailable = new TreeMap<Long, Resource>();
- totAvailable.put(start, Resources.clone(totalCapacity));
- RLESparseResourceAllocation totRLEAvail =
- new RLESparseResourceAllocation(totAvailable, resCalc);
-
- // subtract used from available
- RLESparseResourceAllocation netAvailable;
-
- netAvailable =
- RLESparseResourceAllocation.merge(resCalc,
- Resources.clone(totalCapacity), totRLEAvail, rleSparseVector,
- RLEOperator.subtractTestNonNegative, start, end);
-
- // add back in old reservation used resources if any
- ReservationAllocation old = reservationTable.get(oldId);
- if (old != null) {
- netAvailable =
- RLESparseResourceAllocation.merge(resCalc,
- Resources.clone(totalCapacity), netAvailable,
- old.getResourcesOverTime(), RLEOperator.add, start, end);
+
+ // for non-periodic return simple available resources
+ if (period == 0) {
+
+ // create RLE of totCapacity
+ TreeMap<Long, Resource> totAvailable = new TreeMap<Long, Resource>();
+ totAvailable.put(start, Resources.clone(totalCapacity));
+ RLESparseResourceAllocation totRLEAvail =
+ new RLESparseResourceAllocation(totAvailable, resCalc);
+
+ // subtract used from available
+ RLESparseResourceAllocation netAvailable;
+
+ netAvailable = RLESparseResourceAllocation.merge(resCalc,
+ Resources.clone(totalCapacity), totRLEAvail, rleSparseVector,
+ RLEOperator.subtractTestNonNegative, start, end);
+
+ // remove periodic component
+ netAvailable = RLESparseResourceAllocation.merge(resCalc,
+ Resources.clone(totalCapacity), netAvailable, periodicRle,
+ RLEOperator.subtractTestNonNegative, start, end);
+
+ // add back in old reservation used resources if any
+ ReservationAllocation old = reservationTable.get(oldId);
+ if (old != null) {
+
+ RLESparseResourceAllocation addBackPrevious =
+ old.getResourcesOverTime(start, end);
+ netAvailable = RLESparseResourceAllocation.merge(resCalc,
+ Resources.clone(totalCapacity), netAvailable, addBackPrevious,
+ RLEOperator.add, start, end);
+ }
+ // lower it if this is needed by the sharing policy
+ netAvailable = getSharingPolicy().availableResources(netAvailable, this,
+ user, oldId, start, end);
+ return netAvailable;
+ } else {
+
+ if (periodicRle.getTimePeriod() % period != 0) {
+ throw new PlanningException("The reservation periodicity (" + period
+ + ") must be" + "an exact divider of the system maxPeriod ("
+ + periodicRle.getTimePeriod() + ")");
+ }
+
+ // find the minimum resources available among all the instances that fit
+ // in the LCM
+ long numInstInLCM = periodicRle.getTimePeriod() / period;
+
+ RLESparseResourceAllocation minOverLCM =
+ getAvailableResourceOverTime(user, oldId, start, end, 0);
+ for (int i = 1; i < numInstInLCM; i++) {
+
+ long rStart = start + i * period;
+ long rEnd = end + i * period;
+
+ // recursive invocation of non-periodic range (to pick raw-info)
+ RLESparseResourceAllocation snapShot =
+ getAvailableResourceOverTime(user, oldId, rStart, rEnd, 0);
+
+ // time-align on start
+ snapShot.shift(-(i * period));
+
+ // pick the minimum amount of resources in each time interval
+ minOverLCM =
+ RLESparseResourceAllocation.merge(resCalc, getTotalCapacity(),
+ minOverLCM, snapShot, RLEOperator.min, start, end);
+
+ }
+
+ return minOverLCM;
+
}
- // lower it if this is needed by the sharing policy
- netAvailable =
- getSharingPolicy().availableResources(netAvailable, this, user,
- oldId, start, end);
- return netAvailable;
} finally {
readLock.unlock();
}
@@ -637,7 +808,7 @@ public class InMemoryPlan implements Plan {
public String toCumulativeString() {
readLock.lock();
try {
- return rleSparseVector.toString();
+ return rleSparseVector.toString() + "\n" + periodicRle.toString();
} finally {
readLock.unlock();
}
@@ -689,11 +860,18 @@ public class InMemoryPlan implements Plan {
}
@Override
- public RLESparseResourceAllocation getCumulativeLoadOverTime(
- long start, long end) {
+ public RLESparseResourceAllocation getCumulativeLoadOverTime(long start,
+ long end) throws PlanningException {
readLock.lock();
try {
- return rleSparseVector.getRangeOverlapping(start, end);
+
+ RLESparseResourceAllocation ret =
+ rleSparseVector.getRangeOverlapping(start, end);
+ ret = RLESparseResourceAllocation.merge(resCalc, totalCapacity, ret,
+ periodicRle.getRangeOverlapping(start, end), RLEOperator.add, start,
+ end);
+
+ return ret;
} finally {
readLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e7d136/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryReservationAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryReservationAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryReservationAllocation.java
index 69fd43f..00c8e44 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryReservationAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryReservationAllocation.java
@@ -42,6 +42,7 @@ public class InMemoryReservationAllocation implements ReservationAllocation {
private final Map<ReservationInterval, Resource> allocationRequests;
private boolean hasGang = false;
private long acceptedAt = -1;
+ private long periodicity = 0;
private RLESparseResourceAllocation resourcesOverTime;
@@ -67,9 +68,16 @@ public class InMemoryReservationAllocation implements ReservationAllocation {
this.allocationRequests = allocations;
this.planName = planName;
this.hasGang = hasGang;
- resourcesOverTime = new RLESparseResourceAllocation(calculator);
- for (Map.Entry<ReservationInterval, Resource> r : allocations
- .entrySet()) {
+ if (contract != null && contract.getRecurrenceExpression() != null) {
+ this.periodicity = Long.parseLong(contract.getRecurrenceExpression());
+ }
+ if (periodicity > 0) {
+ resourcesOverTime =
+ new PeriodicRLESparseResourceAllocation(calculator, periodicity);
+ } else {
+ resourcesOverTime = new RLESparseResourceAllocation(calculator);
+ }
+ for (Map.Entry<ReservationInterval, Resource> r : allocations.entrySet()) {
resourcesOverTime.addInterval(r.getKey(), r.getValue());
}
}
@@ -133,17 +141,33 @@ public class InMemoryReservationAllocation implements ReservationAllocation {
}
@Override
- public RLESparseResourceAllocation getResourcesOverTime(){
+ public RLESparseResourceAllocation getResourcesOverTime() {
return resourcesOverTime;
}
@Override
+ public RLESparseResourceAllocation getResourcesOverTime(long start,
+ long end) {
+ return resourcesOverTime.getRangeOverlapping(start, end);
+ }
+
+ @Override
+ public long getPeriodicity() {
+ return periodicity;
+ }
+
+ @Override
+ public void setPeriodicity(long period) {
+ periodicity = period;
+ }
+
+ @Override
public String toString() {
StringBuilder sBuf = new StringBuilder();
sBuf.append(getReservationId()).append(" user:").append(getUser())
.append(" startTime: ").append(getStartTime()).append(" endTime: ")
- .append(getEndTime()).append(" alloc:\n[")
- .append(resourcesOverTime.toString()).append("] ");
+ .append(getEndTime()).append(" Periodiciy: ").append(periodicity)
+ .append(" alloc:\n[").append(resourcesOverTime.toString()).append("] ");
return sBuf.toString();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e7d136/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java
index 55f1d00..49d4702 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java
@@ -40,7 +40,7 @@ public class NoOverCommitPolicy implements SharingPolicy {
RLESparseResourceAllocation available = plan.getAvailableResourceOverTime(
reservation.getUser(), reservation.getReservationId(),
- reservation.getStartTime(), reservation.getEndTime());
+ reservation.getStartTime(), reservation.getEndTime(), 0);
// test the reservation does not exceed what is available
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e7d136/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PeriodicRLESparseResourceAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PeriodicRLESparseResourceAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PeriodicRLESparseResourceAllocation.java
index 8e3be8b3..7bc44f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PeriodicRLESparseResourceAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PeriodicRLESparseResourceAllocation.java
@@ -18,47 +18,94 @@
package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.annotations.VisibleForTesting;
+
/**
- * This data structure stores a periodic RLESparseResourceAllocation.
+ * This data structure stores a periodic {@link RLESparseResourceAllocation}.
* Default period is 1 day (86400000ms).
*/
-public class PeriodicRLESparseResourceAllocation extends
- RLESparseResourceAllocation {
+public class PeriodicRLESparseResourceAllocation
+ extends RLESparseResourceAllocation {
// Log
- private static final Logger LOG = LoggerFactory
- .getLogger(PeriodicRLESparseResourceAllocation.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(PeriodicRLESparseResourceAllocation.class);
private long timePeriod;
/**
* Constructor.
*
- * @param rleVector {@link RLESparseResourceAllocation} with the run-length
- encoded data.
+ * @param resourceCalculator {@link ResourceCalculator} the resource
+ * calculator to use.
* @param timePeriod Time period in milliseconds.
*/
public PeriodicRLESparseResourceAllocation(
- RLESparseResourceAllocation rleVector, Long timePeriod) {
- super(rleVector.getCumulative(), rleVector.getResourceCalculator());
+ ResourceCalculator resourceCalculator, Long timePeriod) {
+ super(resourceCalculator);
this.timePeriod = timePeriod;
}
/**
* Constructor. Default time period set to 1 day.
*
+ * @param resourceCalculator {@link ResourceCalculator} the resource
+ * calculator to use..
+ */
+ public PeriodicRLESparseResourceAllocation(
+ ResourceCalculator resourceCalculator) {
+ this(resourceCalculator,
+ YarnConfiguration.DEFAULT_RM_RESERVATION_SYSTEM_MAX_PERIODICITY);
+ }
+
+ /**
+ * Constructor.
+ *
* @param rleVector {@link RLESparseResourceAllocation} with the run-length
- encoded data.
+ * encoded data.
+ * @param timePeriod Time period in milliseconds.
*/
+ @VisibleForTesting
public PeriodicRLESparseResourceAllocation(
- RLESparseResourceAllocation rleVector) {
- this(rleVector, 86400000L);
+ RLESparseResourceAllocation rleVector, Long timePeriod) {
+ super(rleVector.getCumulative(), rleVector.getResourceCalculator());
+ this.timePeriod = timePeriod;
+
+ // make sure the PeriodicRLE is zero-based, and handles wrap-around
+ long delta = (getEarliestStartTime() % timePeriod - getEarliestStartTime());
+ shift(delta);
+
+ List<Long> toRemove = new ArrayList<>();
+ Map<Long, Resource> toAdd = new TreeMap<>();
+
+ for (Map.Entry<Long, Resource> entry : cumulativeCapacity.entrySet()) {
+ if (entry.getKey() > timePeriod) {
+ toRemove.add(entry.getKey());
+ if (entry.getValue() != null) {
+ toAdd.put(timePeriod, entry.getValue());
+ long prev = entry.getKey() % timePeriod;
+ toAdd.put(prev, this.getCapacityAtTime(prev));
+ toAdd.put(0L, entry.getValue());
+ }
+ }
+ }
+ for (Long l : toRemove) {
+ cumulativeCapacity.remove(l);
+ }
+ cumulativeCapacity.putAll(toAdd);
}
/**
@@ -78,24 +125,25 @@ public class PeriodicRLESparseResourceAllocation extends
* The interval may include 0, but the end time must be strictly less than
* timePeriod.
*
- * @param interval {@link ReservationInterval} to which the specified
- * resource is to be added.
+ * @param interval {@link ReservationInterval} to which the specified resource
+ * is to be added.
* @param resource {@link Resource} to be added to the interval specified.
* @return true if addition is successful, false otherwise
*/
- public boolean addInterval(ReservationInterval interval,
- Resource resource) {
+ public boolean addInterval(ReservationInterval interval, Resource resource) {
long startTime = interval.getStartTime();
long endTime = interval.getEndTime();
+
if (startTime >= 0 && endTime > startTime && endTime <= timePeriod) {
return super.addInterval(interval, resource);
} else {
- LOG.info("Cannot set capacity beyond end time: " + timePeriod);
+ LOG.info("Cannot set capacity beyond end time: " + timePeriod + " was ("
+ + interval.toString() + ")");
return false;
}
}
- /**
+ /**
* Removes a resource for the specified interval.
*
* @param interval the {@link ReservationInterval} for which the resource is
@@ -103,14 +151,15 @@ public class PeriodicRLESparseResourceAllocation extends
* @param resource the {@link Resource} to be removed.
* @return true if removal is successful, false otherwise
*/
- public boolean removeInterval(
- ReservationInterval interval, Resource resource) {
+ public boolean removeInterval(ReservationInterval interval,
+ Resource resource) {
long startTime = interval.getStartTime();
long endTime = interval.getEndTime();
// If the resource to be subtracted is less than the minimum resource in
// the range, abort removal to avoid negative capacity.
- if (!Resources.fitsIn(
- resource, super.getMinimumCapacityInInterval(interval))) {
+ // TODO revesit decrementing endTime
+ if (!Resources.fitsIn(resource, getMinimumCapacityInInterval(
+ new ReservationInterval(startTime, endTime - 1)))) {
LOG.info("Request to remove more resources than what is available");
return false;
}
@@ -125,17 +174,16 @@ public class PeriodicRLESparseResourceAllocation extends
/**
* Get maximum capacity at periodic offsets from the specified time.
*
- * @param tick UTC time base from which offsets are specified for finding
- * the maximum capacity.
- * @param period periodic offset at which capacities are evaluted.
+ * @param tick UTC time base from which offsets are specified for finding the
+ * maximum capacity.
+ * @param period periodic offset at which capacities are evaluated.
* @return the maximum {@link Resource} across the specified time instants.
* @return true if removal is successful, false otherwise
*/
public Resource getMaximumPeriodicCapacity(long tick, long period) {
Resource maxResource;
if (period < timePeriod) {
- maxResource =
- super.getMaximumPeriodicCapacity(tick % timePeriod, period);
+ maxResource = super.getMaximumPeriodicCapacity(tick % timePeriod, period);
} else {
// if period is greater than the length of PeriodicRLESparseAllocation,
// only a single value exists in this interval.
@@ -164,4 +212,30 @@ public class PeriodicRLESparseResourceAllocation extends
return ret.toString();
}
+ @Override
+ public RLESparseResourceAllocation getRangeOverlapping(long start, long end) {
+ NavigableMap<Long, Resource> unrolledMap = new TreeMap<>();
+ readLock.lock();
+ try {
+ long relativeStart = (start >= 0) ? start % timePeriod : 0;
+ NavigableMap<Long, Resource> cumulativeMap = this.getCumulative();
+ Long previous = cumulativeMap.floorKey(relativeStart);
+ previous = (previous != null) ? previous : 0;
+ for (long i = 0; i <= (end - start) / timePeriod; i++) {
+ for (Map.Entry<Long, Resource> e : cumulativeMap.entrySet()) {
+ long curKey = e.getKey() + (i * timePeriod);
+ if (curKey >= previous && (start + curKey - relativeStart) <= end) {
+ unrolledMap.put(curKey, e.getValue());
+ }
+ }
+ }
+ RLESparseResourceAllocation rle =
+ new RLESparseResourceAllocation(unrolledMap, getResourceCalculator());
+ rle.shift(start - relativeStart);
+ return rle;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e7d136/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanEdit.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanEdit.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanEdit.java
index 504a250..9afa324 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanEdit.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanEdit.java
@@ -28,54 +28,58 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.Plan
public interface PlanEdit extends PlanContext, PlanView {
/**
- * Add a new {@link ReservationAllocation} to the plan
+ * Add a new {@link ReservationAllocation} to the plan.
*
* @param reservation the {@link ReservationAllocation} to be added to the
* plan
* @param isRecovering flag to indicate if reservation is being added as part
* of failover or not
* @return true if addition is successful, false otherwise
+ * @throws PlanningException if addition is unsuccessful
*/
- public boolean addReservation(ReservationAllocation reservation,
+ boolean addReservation(ReservationAllocation reservation,
boolean isRecovering) throws PlanningException;
/**
* Updates an existing {@link ReservationAllocation} in the plan. This is
- * required for re-negotiation
+ * required for re-negotiation.
*
* @param reservation the {@link ReservationAllocation} to be updated the plan
* @return true if update is successful, false otherwise
+ * @throws PlanningException if update is unsuccessful
*/
- public boolean updateReservation(ReservationAllocation reservation)
+ boolean updateReservation(ReservationAllocation reservation)
throws PlanningException;
/**
* Delete an existing {@link ReservationAllocation} from the plan identified
* uniquely by its {@link ReservationId}. This will generally be used for
- * garbage collection
+ * garbage collection.
*
* @param reservationID the {@link ReservationAllocation} to be deleted from
* the plan identified uniquely by its {@link ReservationId}
* @return true if delete is successful, false otherwise
+ * @throws PlanningException if deletion is unsuccessful
*/
- public boolean deleteReservation(ReservationId reservationID)
+ boolean deleteReservation(ReservationId reservationID)
throws PlanningException;
/**
* Method invoked to garbage collect old reservations. It cleans up expired
- * reservations that have fallen out of the sliding archival window
+ * reservations that have fallen out of the sliding archival window.
*
* @param tick the current time from which the archival window is computed
+ * @throws PlanningException if archival is unsuccessful
*/
- public void archiveCompletedReservations(long tick) throws PlanningException;
+ void archiveCompletedReservations(long tick) throws PlanningException;
/**
* Sets the overall capacity in terms of {@link Resource} assigned to this
- * plan
+ * plan.
*
* @param capacity the overall capacity in terms of {@link Resource} assigned
* to this plan
*/
- public void setTotalCapacity(Resource capacity);
+ void setTotalCapacity(Resource capacity);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e7d136/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanView.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanView.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanView.java
index 2767993..4035f68 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanView.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PlanView.java
@@ -17,50 +17,50 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+import java.util.Set;
+
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
-import java.util.Set;
-
/**
* This interface provides a read-only view on the allocations made in this
* plan. This methods are used for example by {@code ReservationAgent}s to
* determine the free resources in a certain point in time, and by
* PlanFollowerPolicy to publish this plan to the scheduler.
*/
-public interface PlanView extends PlanContext {
+interface PlanView extends PlanContext {
/**
* Return a set of {@link ReservationAllocation} identified by the user who
* made the reservation.
*
* @param reservationID the unqiue id to identify the
- * {@link ReservationAllocation}
+ * {@link ReservationAllocation}
* @param interval the time interval used to retrieve the reservation
- * allocations from. Only reservations with start time no
- * greater than the interval end time, and end time no less
- * than the interval start time will be selected.
+ * allocations from. Only reservations with start time no greater
+ * than the interval end time, and end time no less than the interval
+ * start time will be selected.
* @param user the user to retrieve the reservation allocation from.
* @return a set of {@link ReservationAllocation} identified by the user who
- * made the reservation
+ * made the reservation
*/
- Set<ReservationAllocation> getReservations(ReservationId
- reservationID, ReservationInterval interval, String user);
+ Set<ReservationAllocation> getReservations(ReservationId reservationID,
+ ReservationInterval interval, String user);
/**
* Return a set of {@link ReservationAllocation} identified by any user.
*
* @param reservationID the unqiue id to identify the
- * {@link ReservationAllocation}
+ * {@link ReservationAllocation}
* @param interval the time interval used to retrieve the reservation
- * allocations from. Only reservations with start time no
- * greater than the interval end time, and end time no less
- * than the interval start time will be selected.
+ * allocations from. Only reservations with start time no greater
+ * than the interval end time, and end time no less than the interval
+ * start time will be selected.
* @return a set of {@link ReservationAllocation} identified by any user
*/
Set<ReservationAllocation> getReservations(ReservationId reservationID,
- ReservationInterval interval);
+ ReservationInterval interval);
/**
* Return a {@link ReservationAllocation} identified by its
@@ -70,7 +70,7 @@ public interface PlanView extends PlanContext {
* {@link ReservationAllocation}
* @return {@link ReservationAllocation} identified by the specified id
*/
- public ReservationAllocation getReservationById(ReservationId reservationID);
+ ReservationAllocation getReservationById(ReservationId reservationID);
/**
* Return a set of {@link ReservationAllocation} that belongs to a certain
@@ -78,11 +78,10 @@ public interface PlanView extends PlanContext {
*
* @param user the user being considered
* @param t the instant in time being considered
- * @return set of active {@link ReservationAllocation}s for this
- * user at this time
+ * @return set of active {@link ReservationAllocation}s for this user at this
+ * time
*/
- public Set<ReservationAllocation> getReservationByUserAtTime(String user,
- long t);
+ Set<ReservationAllocation> getReservationByUserAtTime(String user, long t);
/**
* Gets all the active reservations at the specified point of time
@@ -91,14 +90,14 @@ public interface PlanView extends PlanContext {
* requested
* @return set of active reservations at the specified time
*/
- public Set<ReservationAllocation> getReservationsAtTime(long tick);
+ Set<ReservationAllocation> getReservationsAtTime(long tick);
/**
* Gets all the reservations in the plan
*
* @return set of all reservations handled by this Plan
*/
- public Set<ReservationAllocation> getAllReservations();
+ Set<ReservationAllocation> getAllReservations();
/**
* Returns the total {@link Resource} reserved for all users at the specified
@@ -126,61 +125,68 @@ public interface PlanView extends PlanContext {
*
* @return the time (UTC in ms) at which the first reservation starts
*/
- public long getEarliestStartTime();
+ long getEarliestStartTime();
/**
* Returns the time (UTC in ms) at which the last reservation terminates
*
* @return the time (UTC in ms) at which the last reservation terminates
*/
- public long getLastEndTime();
+ long getLastEndTime();
/**
* This method returns the amount of resources available to a given user
* (optionally if removing a certain reservation) over the start-end time
- * range.
+ * range. If the request is periodic (period is non-zero) we return the
+ * minimum amount of resources available to periodic reservations (in all
+ * "period" windows within the system maxPeriod / LCM).
*
- * @param user
- * @param oldId
- * @param start
- * @param end
+ * @param user the user being considered
+ * @param oldId the identifier of the existing reservation
+ * @param start start of the time interval.
+ * @param end end of the time interval.
+ * @param period the ms periodicty for this request (loop and pick min till
+ * maxPeriodicity)
* @return a view of the plan as it is available to this user
- * @throws PlanningException
+ * @throws PlanningException if operation is unsuccessful
*/
- public RLESparseResourceAllocation getAvailableResourceOverTime(String user,
- ReservationId oldId, long start, long end) throws PlanningException;
+ RLESparseResourceAllocation getAvailableResourceOverTime(String user,
+ ReservationId oldId, long start, long end, long period)
+ throws PlanningException;
/**
* This method returns a RLE encoded view of the user reservation count
* utilization between start and end time.
*
- * @param user
- * @param start
- * @param end
+ * @param user the user being considered
+ * @param start start of the time interval.
+ * @param end end of the time interval.
* @return RLE encoded view of reservation used over time
*/
- public RLESparseResourceAllocation getReservationCountForUserOverTime(
- String user, long start, long end);
+ RLESparseResourceAllocation getReservationCountForUserOverTime(String user,
+ long start, long end);
/**
* This method returns a RLE encoded view of the user reservation utilization
* between start and end time.
*
- * @param user
- * @param start
- * @param end
+ * @param user the user being considered
+ * @param start start of the time interval.
+ * @param end end of the time interval.
* @return RLE encoded view of resources used over time
*/
- public RLESparseResourceAllocation getConsumptionForUserOverTime(String user,
+ RLESparseResourceAllocation getConsumptionForUserOverTime(String user,
long start, long end);
/**
* Get the cumulative load over a time interval.
*
- * @param start Start of the time interval.
- * @param end End of the time interval.
+ * @param start start of the time interval.
+ * @param end end of the time interval.
* @return RLE sparse allocation.
+ * @throws PlanningException if operation is unsuccessful
*/
- RLESparseResourceAllocation getCumulativeLoadOverTime(long start, long end);
+ RLESparseResourceAllocation getCumulativeLoadOverTime(long start, long end)
+ throws PlanningException;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/40] hadoop git commit: YARN-5328. Plan/ResourceAllocation data
structure enhancements required to support recurring reservations in
ReservationSystem.
Posted by in...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e7d136/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/RLESparseResourceAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/RLESparseResourceAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/RLESparseResourceAllocation.java
index 658387b..3062f3d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/RLESparseResourceAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/RLESparseResourceAllocation.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.reservation;
-import java.io.IOException;
-import java.io.StringWriter;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
@@ -33,8 +31,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.Plan
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
-import com.google.gson.stream.JsonWriter;
-
/**
* This is a run length encoded sparse data structure that maintains resource
* allocations over time.
@@ -44,12 +40,14 @@ public class RLESparseResourceAllocation {
private static final int THRESHOLD = 100;
private static final Resource ZERO_RESOURCE = Resources.none();
- private NavigableMap<Long, Resource> cumulativeCapacity =
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected NavigableMap<Long, Resource> cumulativeCapacity =
new TreeMap<Long, Resource>();
private final ReentrantReadWriteLock readWriteLock =
new ReentrantReadWriteLock();
- private final Lock readLock = readWriteLock.readLock();
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected final Lock readLock = readWriteLock.readLock();
private final Lock writeLock = readWriteLock.writeLock();
private final ResourceCalculator resourceCalculator;
@@ -236,34 +234,6 @@ public class RLESparseResourceAllocation {
}
/**
- * Returns the JSON string representation of the current resources allocated
- * over time.
- *
- * @return the JSON string representation of the current resources allocated
- * over time
- */
- public String toMemJSONString() {
- StringWriter json = new StringWriter();
- JsonWriter jsonWriter = new JsonWriter(json);
- readLock.lock();
- try {
- jsonWriter.beginObject();
- // jsonWriter.name("timestamp").value("resource");
- for (Map.Entry<Long, Resource> r : cumulativeCapacity.entrySet()) {
- jsonWriter.name(r.getKey().toString()).value(r.getValue().toString());
- }
- jsonWriter.endObject();
- jsonWriter.close();
- return json.toString();
- } catch (IOException e) {
- // This should not happen
- return "";
- } finally {
- readLock.unlock();
- }
- }
-
- /**
* Returns the representation of the current resources allocated over time as
* an interval map (in the defined non-null range).
*
@@ -437,8 +407,8 @@ public class RLESparseResourceAllocation {
Resource val = Resources.negate(e.getValue());
// test for negative value and throws
if (operator == RLEOperator.subtractTestNonNegative
- && (Resources.fitsIn(val, ZERO_RESOURCE) &&
- !Resources.equals(val, ZERO_RESOURCE))) {
+ && (Resources.fitsIn(val, ZERO_RESOURCE)
+ && !Resources.equals(val, ZERO_RESOURCE))) {
throw new PlanningException(
"RLESparseResourceAllocation: merge failed as the "
+ "resulting RLESparseResourceAllocation would be negative");
@@ -504,22 +474,29 @@ public class RLESparseResourceAllocation {
}
+ /**
+ * Get a {@link RLESparseResourceAllocation} view of the {@link Resource}
+ * allocations between the specified start and end times.
+ *
+ * @param start the time from which the {@link Resource} allocations are
+ * required
+ * @param end the time upto which the {@link Resource} allocations are
+ * required
+ * @return the overlapping allocations
+ */
public RLESparseResourceAllocation getRangeOverlapping(long start, long end) {
readLock.lock();
try {
NavigableMap<Long, Resource> a = this.getCumulative();
-
if (a != null && !a.isEmpty()) {
// include the portion of previous entry that overlaps start
if (start > a.firstKey()) {
long previous = a.floorKey(start);
a = a.tailMap(previous, true);
}
-
if (end < a.lastKey()) {
a = a.headMap(end, true);
}
-
}
RLESparseResourceAllocation ret =
new RLESparseResourceAllocation(a, resourceCalculator);
@@ -527,7 +504,33 @@ public class RLESparseResourceAllocation {
} finally {
readLock.unlock();
}
+ }
+ /**
+ * This method shifts all the timestamp of the {@link Resource} entries by the
+ * specified "delta".
+ *
+ * @param delta the time by which to shift the {@link Resource} allocations
+ */
+ public void shift(long delta) {
+ writeLock.lock();
+ try {
+ TreeMap<Long, Resource> newCum = new TreeMap<>();
+ long start;
+ for (Map.Entry<Long, Resource> entry : cumulativeCapacity.entrySet()) {
+ if (delta > 0) {
+ start = (entry.getKey() == Long.MAX_VALUE) ? Long.MAX_VALUE
+ : entry.getKey() + delta;
+ } else {
+ start = (entry.getKey() == Long.MIN_VALUE) ? Long.MIN_VALUE
+ : entry.getKey() + delta;
+ }
+ newCum.put(start, entry.getValue());
+ }
+ cumulativeCapacity = newCum;
+ } finally {
+ writeLock.unlock();
+ }
}
/**
@@ -541,8 +544,8 @@ public class RLESparseResourceAllocation {
/**
* Get the maximum capacity across specified time instances. The search-space
* is specified using the starting value, tick, and the periodic interval for
- * search. Maximum resource allocation across tick, tick + period,
- * tick + 2 * period,..., tick + n * period .. is returned.
+ * search. Maximum resource allocation across tick, tick + period, tick + 2 *
+ * period,..., tick + n * period .. is returned.
*
* @param tick the starting time instance
* @param period interval at which capacity is evaluated
@@ -550,14 +553,19 @@ public class RLESparseResourceAllocation {
*/
public Resource getMaximumPeriodicCapacity(long tick, long period) {
Resource maxCapacity = ZERO_RESOURCE;
- if (!cumulativeCapacity.isEmpty()) {
- Long lastKey = cumulativeCapacity.lastKey();
- for (long t = tick; t <= lastKey; t = t + period) {
- maxCapacity = Resources.componentwiseMax(maxCapacity,
- cumulativeCapacity.floorEntry(t).getValue());
+ readLock.lock();
+ try {
+ if (!cumulativeCapacity.isEmpty()) {
+ Long lastKey = cumulativeCapacity.lastKey();
+ for (long t = tick; t <= lastKey; t = t + period) {
+ maxCapacity = Resources.componentwiseMax(maxCapacity,
+ cumulativeCapacity.floorEntry(t).getValue());
+ }
}
+ return maxCapacity;
+ } finally {
+ readLock.unlock();
}
- return maxCapacity;
}
/**
@@ -567,17 +575,17 @@ public class RLESparseResourceAllocation {
* @return minimum resource allocation
*/
public Resource getMinimumCapacityInInterval(ReservationInterval interval) {
- Resource minCapacity = Resource.newInstance(
- Integer.MAX_VALUE, Integer.MAX_VALUE);
+ Resource minCapacity =
+ Resource.newInstance(Integer.MAX_VALUE, Integer.MAX_VALUE);
long start = interval.getStartTime();
long end = interval.getEndTime();
NavigableMap<Long, Resource> capacityRange =
- this.getRangeOverlapping(start, end).getCumulative();
+ getRangeOverlapping(start, end).getCumulative();
if (!capacityRange.isEmpty()) {
for (Map.Entry<Long, Resource> entry : capacityRange.entrySet()) {
if (entry.getValue() != null) {
- minCapacity = Resources.componentwiseMin(minCapacity,
- entry.getValue());
+ minCapacity =
+ Resources.componentwiseMin(minCapacity, entry.getValue());
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e7d136/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationAllocation.java
index 0da95ac..bb4a7fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationAllocation.java
@@ -24,14 +24,16 @@ import org.apache.hadoop.yarn.api.records.ReservationDefinition;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* A ReservationAllocation represents a concrete allocation of resources over
* time that satisfy a certain {@link ReservationDefinition}. This is used
* internally by a {@link Plan} to store information about how each of the
* accepted {@link ReservationDefinition} have been allocated.
*/
-public interface ReservationAllocation extends
- Comparable<ReservationAllocation> {
+public interface ReservationAllocation
+ extends Comparable<ReservationAllocation> {
/**
* Returns the unique identifier {@link ReservationId} that represents the
@@ -40,28 +42,28 @@ public interface ReservationAllocation extends
* @return reservationId the unique identifier {@link ReservationId} that
* represents the reservation
*/
- public ReservationId getReservationId();
+ ReservationId getReservationId();
/**
* Returns the original {@link ReservationDefinition} submitted by the client
*
* @return the {@link ReservationDefinition} submitted by the client
*/
- public ReservationDefinition getReservationDefinition();
+ ReservationDefinition getReservationDefinition();
/**
* Returns the time at which the reservation is activated.
*
* @return the time at which the reservation is activated
*/
- public long getStartTime();
+ long getStartTime();
/**
* Returns the time at which the reservation terminates.
*
* @return the time at which the reservation terminates
*/
- public long getEndTime();
+ long getEndTime();
/**
* Returns the map of resources requested against the time interval for which
@@ -70,28 +72,28 @@ public interface ReservationAllocation extends
* @return the allocationRequests the map of resources requested against the
* time interval for which they were
*/
- public Map<ReservationInterval, Resource> getAllocationRequests();
+ Map<ReservationInterval, Resource> getAllocationRequests();
/**
* Return a string identifying the plan to which the reservation belongs
*
* @return the plan to which the reservation belongs
*/
- public String getPlanName();
+ String getPlanName();
/**
* Returns the user who requested the reservation
*
* @return the user who requested the reservation
*/
- public String getUser();
+ String getUser();
/**
* Returns whether the reservation has gang semantics or not
*
* @return true if there is a gang request, false otherwise
*/
- public boolean containsGangs();
+ boolean containsGangs();
/**
* Sets the time at which the reservation was accepted by the system
@@ -99,14 +101,14 @@ public interface ReservationAllocation extends
* @param acceptedAt the time at which the reservation was accepted by the
* system
*/
- public void setAcceptanceTimestamp(long acceptedAt);
+ void setAcceptanceTimestamp(long acceptedAt);
/**
* Returns the time at which the reservation was accepted by the system
*
* @return the time at which the reservation was accepted by the system
*/
- public long getAcceptanceTime();
+ long getAcceptanceTime();
/**
* Returns the capacity represented by cumulative resources reserved by the
@@ -116,12 +118,42 @@ public interface ReservationAllocation extends
* requested
* @return the resources reserved at the specified time
*/
- public Resource getResourcesAtTime(long tick);
+ Resource getResourcesAtTime(long tick);
+
+ /**
+ * Return a RLE representation of used resources.
+ *
+ * @return a RLE encoding of resources allocated over time.
+ */
+ RLESparseResourceAllocation getResourcesOverTime();
+
/**
* Return a RLE representation of used resources.
+ *
+ * @param start start of the time interval.
+ * @param end end of the time interval.
* @return a RLE encoding of resources allocated over time.
*/
- public RLESparseResourceAllocation getResourcesOverTime();
+ RLESparseResourceAllocation getResourcesOverTime(long start, long end);
+
+ /**
+ * Get the periodicity of this reservation representing the time period of the
+ * periodic job. Period is represented in milliseconds for periodic jobs.
+ * Period is 0 for non-periodic jobs.
+ *
+ * @return periodicity of this reservation
+ */
+ long getPeriodicity();
+
+ /**
+ * Set the periodicity of this reservation representing the time period of the
+ * periodic job. Period is represented in milliseconds for periodic jobs.
+ * Period is 0 for non-periodic jobs.
+ *
+ * @param period periodicity of this reservation
+ */
+ @VisibleForTesting
+ void setPeriodicity(long period);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e7d136/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java
index 027d066..a66d222 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java
@@ -44,6 +44,8 @@ public class ReservationInputValidator {
/**
* Utility class to validate reservation requests.
+ *
+ * @param clock the {@link Clock} to use
*/
public ReservationInputValidator(Clock clock) {
this.clock = clock;
@@ -53,22 +55,21 @@ public class ReservationInputValidator {
ReservationId reservationId, String auditConstant) throws YarnException {
// check if the reservation id is valid
if (reservationId == null) {
- String message =
- "Missing reservation id."
- + " Please try again by specifying a reservation id.";
+ String message = "Missing reservation id."
+ + " Please try again by specifying a reservation id.";
RMAuditLogger.logFailure("UNKNOWN", auditConstant,
"validate reservation input", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
}
String queue = reservationSystem.getQueueForReservation(reservationId);
String nullQueueErrorMessage =
- "The specified reservation with ID: " + reservationId
- + " is unknown. Please try again with a valid reservation.";
+ "The specified reservation with ID: " + reservationId
+ + " is unknown. Please try again with a valid reservation.";
String nullPlanErrorMessage = "The specified reservation: " + reservationId
- + " is not associated with any valid plan."
- + " Please try again with a valid reservation.";
+ + " is not associated with any valid plan."
+ + " Please try again with a valid reservation.";
return getPlanFromQueue(reservationSystem, queue, auditConstant,
- nullQueueErrorMessage, nullPlanErrorMessage);
+ nullQueueErrorMessage, nullPlanErrorMessage);
}
private void validateReservationDefinition(ReservationId reservationId,
@@ -77,17 +78,15 @@ public class ReservationInputValidator {
String message = "";
// check if deadline is in the past
if (contract == null) {
- message =
- "Missing reservation definition."
- + " Please try again by specifying a reservation definition.";
+ message = "Missing reservation definition."
+ + " Please try again by specifying a reservation definition.";
RMAuditLogger.logFailure("UNKNOWN", auditConstant,
"validate reservation input definition", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
}
if (contract.getDeadline() <= clock.getTime()) {
- message =
- "The specified deadline: " + contract.getDeadline()
- + " is the past. Please try again with deadline in the future.";
+ message = "The specified deadline: " + contract.getDeadline()
+ + " is the past. Please try again with deadline in the future.";
RMAuditLogger.logFailure("UNKNOWN", auditConstant,
"validate reservation input definition", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
@@ -95,18 +94,16 @@ public class ReservationInputValidator {
// Check if at least one RR has been specified
ReservationRequests resReqs = contract.getReservationRequests();
if (resReqs == null) {
- message =
- "No resources have been specified to reserve."
- + "Please try again by specifying the resources to reserve.";
+ message = "No resources have been specified to reserve."
+ + "Please try again by specifying the resources to reserve.";
RMAuditLogger.logFailure("UNKNOWN", auditConstant,
"validate reservation input definition", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
}
List<ReservationRequest> resReq = resReqs.getReservationResources();
if (resReq == null || resReq.isEmpty()) {
- message =
- "No resources have been specified to reserve."
- + " Please try again by specifying the resources to reserve.";
+ message = "No resources have been specified to reserve."
+ + " Please try again by specifying the resources to reserve.";
RMAuditLogger.logFailure("UNKNOWN", auditConstant,
"validate reservation input definition", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
@@ -123,22 +120,18 @@ public class ReservationInputValidator {
} else {
minDuration += rr.getDuration();
}
- maxGangSize =
- Resources.max(plan.getResourceCalculator(), plan.getTotalCapacity(),
- maxGangSize,
- Resources.multiply(rr.getCapability(), rr.getConcurrency()));
+ maxGangSize = Resources.max(plan.getResourceCalculator(),
+ plan.getTotalCapacity(), maxGangSize,
+ Resources.multiply(rr.getCapability(), rr.getConcurrency()));
}
// verify the allocation is possible (skip for ANY)
long duration = contract.getDeadline() - contract.getArrival();
- if (duration < minDuration
- && type != ReservationRequestInterpreter.R_ANY) {
- message =
- "The time difference ("
- + (duration)
- + ") between arrival (" + contract.getArrival() + ") "
- + "and deadline (" + contract.getDeadline() + ") must "
- + " be greater or equal to the minimum resource duration ("
- + minDuration + ")";
+ if (duration < minDuration && type != ReservationRequestInterpreter.R_ANY) {
+ message = "The time difference (" + (duration) + ") between arrival ("
+ + contract.getArrival() + ") " + "and deadline ("
+ + contract.getDeadline() + ") must "
+ + " be greater or equal to the minimum resource duration ("
+ + minDuration + ")";
RMAuditLogger.logFailure("UNKNOWN", auditConstant,
"validate reservation input definition", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
@@ -148,10 +141,9 @@ public class ReservationInputValidator {
if (Resources.greaterThan(plan.getResourceCalculator(),
plan.getTotalCapacity(), maxGangSize, plan.getTotalCapacity())
&& type != ReservationRequestInterpreter.R_ANY) {
- message =
- "The size of the largest gang in the reservation definition ("
- + maxGangSize + ") exceed the capacity available ("
- + plan.getTotalCapacity() + " )";
+ message = "The size of the largest gang in the reservation definition ("
+ + maxGangSize + ") exceed the capacity available ("
+ + plan.getTotalCapacity() + " )";
RMAuditLogger.logFailure("UNKNOWN", auditConstant,
"validate reservation input definition", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
@@ -179,32 +171,32 @@ public class ReservationInputValidator {
}
}
- private Plan getPlanFromQueue(ReservationSystem reservationSystem, String
- queue, String auditConstant) throws YarnException {
+ private Plan getPlanFromQueue(ReservationSystem reservationSystem,
+ String queue, String auditConstant) throws YarnException {
String nullQueueErrorMessage = "The queue is not specified."
- + " Please try again with a valid reservable queue.";
+ + " Please try again with a valid reservable queue.";
String nullPlanErrorMessage = "The specified queue: " + queue
- + " is not managed by reservation system."
- + " Please try again with a valid reservable queue.";
+ + " is not managed by reservation system."
+ + " Please try again with a valid reservable queue.";
return getPlanFromQueue(reservationSystem, queue, auditConstant,
- nullQueueErrorMessage, nullPlanErrorMessage);
+ nullQueueErrorMessage, nullPlanErrorMessage);
}
- private Plan getPlanFromQueue(ReservationSystem reservationSystem, String
- queue, String auditConstant, String nullQueueErrorMessage,
- String nullPlanErrorMessage) throws YarnException {
+ private Plan getPlanFromQueue(ReservationSystem reservationSystem,
+ String queue, String auditConstant, String nullQueueErrorMessage,
+ String nullPlanErrorMessage) throws YarnException {
if (queue == null || queue.isEmpty()) {
RMAuditLogger.logFailure("UNKNOWN", auditConstant,
- "validate reservation input", "ClientRMService",
- nullQueueErrorMessage);
+ "validate reservation input", "ClientRMService",
+ nullQueueErrorMessage);
throw RPCUtil.getRemoteException(nullQueueErrorMessage);
}
// check if the associated plan is valid
Plan plan = reservationSystem.getPlan(queue);
if (plan == null) {
RMAuditLogger.logFailure("UNKNOWN", auditConstant,
- "validate reservation input", "ClientRMService",
- nullPlanErrorMessage);
+ "validate reservation input", "ClientRMService",
+ nullPlanErrorMessage);
throw RPCUtil.getRemoteException(nullPlanErrorMessage);
}
return plan;
@@ -222,22 +214,21 @@ public class ReservationInputValidator {
* @param reservationId the {@link ReservationId} associated with the current
* request
* @return the {@link Plan} to submit the request to
- * @throws YarnException
+ * @throws YarnException if validation fails
*/
public Plan validateReservationSubmissionRequest(
- ReservationSystem reservationSystem,
- ReservationSubmissionRequest request, ReservationId reservationId)
- throws YarnException {
+ ReservationSystem reservationSystem, ReservationSubmissionRequest request,
+ ReservationId reservationId) throws YarnException {
String message;
if (reservationId == null) {
- message = "Reservation id cannot be null. Please try again " +
- "specifying a valid reservation id by creating a new reservation id.";
+ message = "Reservation id cannot be null. Please try again specifying "
+ + " a valid reservation id by creating a new reservation id.";
throw RPCUtil.getRemoteException(message);
}
// Check if it is a managed queue
String queue = request.getQueue();
Plan plan = getPlanFromQueue(reservationSystem, queue,
- AuditConstants.SUBMIT_RESERVATION_REQUEST);
+ AuditConstants.SUBMIT_RESERVATION_REQUEST);
validateReservationDefinition(reservationId,
request.getReservationDefinition(), plan,
@@ -255,15 +246,14 @@ public class ReservationInputValidator {
* @param request the {@link ReservationUpdateRequest} defining the resources
* required over time for the request
* @return the {@link Plan} to submit the request to
- * @throws YarnException
+ * @throws YarnException if validation fails
*/
public Plan validateReservationUpdateRequest(
ReservationSystem reservationSystem, ReservationUpdateRequest request)
throws YarnException {
ReservationId reservationId = request.getReservationId();
- Plan plan =
- validateReservation(reservationSystem, reservationId,
- AuditConstants.UPDATE_RESERVATION_REQUEST);
+ Plan plan = validateReservation(reservationSystem, reservationId,
+ AuditConstants.UPDATE_RESERVATION_REQUEST);
validateReservationDefinition(reservationId,
request.getReservationDefinition(), plan,
AuditConstants.UPDATE_RESERVATION_REQUEST);
@@ -278,28 +268,26 @@ public class ReservationInputValidator {
*
* @param reservationSystem the {@link ReservationSystem} to validate against
* @param request the {@link ReservationListRequest} defining search
- * parameters for reservations in the {@link ReservationSystem}
- * that is being validated against.
+ * parameters for reservations in the {@link ReservationSystem} that
+ * is being validated against.
* @return the {@link Plan} to list reservations of.
- * @throws YarnException
+ * @throws YarnException if validation fails
*/
public Plan validateReservationListRequest(
- ReservationSystem reservationSystem,
- ReservationListRequest request)
+ ReservationSystem reservationSystem, ReservationListRequest request)
throws YarnException {
String queue = request.getQueue();
if (request.getEndTime() < request.getStartTime()) {
- String errorMessage = "The specified end time must be greater than " +
- "the specified start time.";
+ String errorMessage = "The specified end time must be greater than "
+ + "the specified start time.";
RMAuditLogger.logFailure("UNKNOWN",
- AuditConstants.LIST_RESERVATION_REQUEST,
- "validate list reservation input", "ClientRMService",
- errorMessage);
+ AuditConstants.LIST_RESERVATION_REQUEST,
+ "validate list reservation input", "ClientRMService", errorMessage);
throw RPCUtil.getRemoteException(errorMessage);
}
// Check if it is a managed queue
return getPlanFromQueue(reservationSystem, queue,
- AuditConstants.LIST_RESERVATION_REQUEST);
+ AuditConstants.LIST_RESERVATION_REQUEST);
}
/**
@@ -312,7 +300,7 @@ public class ReservationInputValidator {
* @param request the {@link ReservationDeleteRequest} defining the resources
* required over time for the request
* @return the {@link Plan} to submit the request to
- * @throws YarnException
+ * @throws YarnException if validation fails
*/
public Plan validateReservationDeleteRequest(
ReservationSystem reservationSystem, ReservationDeleteRequest request)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e7d136/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystem.java
index 8b62972..a6c8fcf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystem.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+import java.util.Map;
+
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
@@ -29,8 +31,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.ReservationsACLsManager;
-import java.util.Map;
-
/**
* This interface is the one implemented by any system that wants to support
* Reservations i.e. make {@code Resource} allocations in future. Implementors
@@ -57,7 +57,7 @@ public interface ReservationSystem extends Recoverable {
*
* @param conf configuration
* @param rmContext current context of the {@code ResourceManager}
- * @throws YarnException
+ * @throws YarnException if initialization of the configured plan fails
*/
void reinitialize(Configuration conf, RMContext rmContext)
throws YarnException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e7d136/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/SharingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/SharingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/SharingPolicy.java
index e458055..cbf0f38 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/SharingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/SharingPolicy.java
@@ -38,7 +38,7 @@ public interface SharingPolicy {
* @param planQueuePath the name of the queue for this plan
* @param conf the system configuration
*/
- public void init(String planQueuePath, ReservationSchedulerConfiguration conf);
+ void init(String planQueuePath, ReservationSchedulerConfiguration conf);
/**
* This method runs the policy validation logic, and return true/false on
@@ -51,7 +51,7 @@ public interface SharingPolicy {
* @throws PlanningException if the policy is respected if we add this
* {@link ReservationAllocation} to the {@link Plan}
*/
- public void validate(Plan plan, ReservationAllocation newAllocation)
+ void validate(Plan plan, ReservationAllocation newAllocation)
throws PlanningException;
/**
@@ -68,9 +68,13 @@ public interface SharingPolicy {
* @param start the start time for the range we are querying
* @param end the end time for the range we are querying
* @param oldId (optional) the id of a reservation being updated
+ *
+ * @return the available resources expressed as a
+ * {@link RLESparseResourceAllocation}
+ *
* @throws PlanningException throws if the request is not valid
*/
- public RLESparseResourceAllocation availableResources(
+ RLESparseResourceAllocation availableResources(
RLESparseResourceAllocation available, Plan plan, String user,
ReservationId oldId, long start, long end) throws PlanningException;
@@ -82,7 +86,6 @@ public interface SharingPolicy {
*
* @return validWindow the window of validity considered by the policy.
*/
- public long getValidWindow();
-
+ long getValidWindow();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e7d136/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/Planner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/Planner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/Planner.java
index abac6ac..af0e712 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/Planner.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/Planner.java
@@ -34,7 +34,7 @@ public interface Planner {
*
* @param plan the {@link Plan} to replan
* @param contracts the list of reservation requests
- * @throws PlanningException
+ * @throws PlanningException if operation is unsuccessful
*/
public void plan(Plan plan, List<ReservationDefinition> contracts)
throws PlanningException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e7d136/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
index 199bfa5..bbbf0d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
@@ -50,7 +50,7 @@ public abstract class PlanningAlgorithm implements ReservationAgent {
* @return whether the allocateUser function was successful or not
*
* @throws PlanningException if the session cannot be fitted into the plan
- * @throws ContractValidationException
+ * @throws ContractValidationException if validation fails
*/
protected boolean allocateUser(ReservationId reservationId, String user,
Plan plan, ReservationDefinition contract,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e7d136/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocator.java
index ec6d9c0..8934b0f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocator.java
@@ -50,7 +50,7 @@ public interface StageAllocator {
*
* @return The computed allocation (or null if the stage could not be
* allocated)
- * @throws PlanningException
+ * @throws PlanningException if operation is unsuccessful
*/
Map<ReservationInterval, Resource> computeStageAllocation(Plan plan,
RLESparseResourceAllocation planLoads,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e7d136/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedy.java
index da04336..d107487 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedy.java
@@ -69,7 +69,7 @@ public class StageAllocatorGreedy implements StageAllocator {
RLESparseResourceAllocation netAvailable =
plan.getAvailableResourceOverTime(user, oldId, stageEarliestStart,
- stageDeadline);
+ stageDeadline, 0);
netAvailable =
RLESparseResourceAllocation.merge(plan.getResourceCalculator(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e7d136/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedyRLE.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedyRLE.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedyRLE.java
index ec83e02..ae7d91a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedyRLE.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedyRLE.java
@@ -83,9 +83,8 @@ public class StageAllocatorGreedyRLE implements StageAllocator {
int gangsToPlace = rr.getNumContainers() / rr.getConcurrency();
// get available resources from plan
- RLESparseResourceAllocation netRLERes =
- plan.getAvailableResourceOverTime(user, oldId, stageEarliestStart,
- stageDeadline);
+ RLESparseResourceAllocation netRLERes = plan.getAvailableResourceOverTime(
+ user, oldId, stageEarliestStart, stageDeadline, 0);
// remove plan modifications
netRLERes =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e7d136/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorLowCostAligned.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorLowCostAligned.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorLowCostAligned.java
index e45f58c..c014549 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorLowCostAligned.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorLowCostAligned.java
@@ -77,8 +77,8 @@ public class StageAllocatorLowCostAligned implements StageAllocator {
ResourceCalculator resCalc = plan.getResourceCalculator();
Resource capacity = plan.getTotalCapacity();
- RLESparseResourceAllocation netRLERes = plan
- .getAvailableResourceOverTime(user, oldId, stageArrival, stageDeadline);
+ RLESparseResourceAllocation netRLERes = plan.getAvailableResourceOverTime(
+ user, oldId, stageArrival, stageDeadline, 0);
long step = plan.getStep();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e7d136/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java
index e99842e..5337e06 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java
@@ -19,7 +19,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.reservation;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anySetOf;
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
import java.io.FileWriter;
import java.io.IOException;
@@ -76,7 +79,8 @@ public class ReservationSystemTestUtil {
String reservationQ, long timeWindow, float instConstraint,
float avgConstraint) {
- ReservationSchedulerConfiguration realConf = new CapacitySchedulerConfiguration();
+ ReservationSchedulerConfiguration realConf =
+ new CapacitySchedulerConfiguration();
ReservationSchedulerConfiguration conf = spy(realConf);
when(conf.getReservationWindow(reservationQ)).thenReturn(timeWindow);
when(conf.getInstantaneousMaxCapacity(reservationQ))
@@ -168,7 +172,6 @@ public class ReservationSystemTestUtil {
scheduler.start();
scheduler.reinitialize(conf, rmContext);
-
Resource resource =
ReservationSystemTestUtil.calculateClusterResource(numContainers);
RMNode node1 = MockNodes.newNodeInfo(1, resource, 1, "127.0.0.1");
@@ -184,10 +187,16 @@ public class ReservationSystemTestUtil {
public static ReservationDefinition createSimpleReservationDefinition(
long arrival, long deadline, long duration, int parallelism) {
+ return createSimpleReservationDefinition(arrival, deadline, duration,
+ parallelism, null);
+ }
+
+ public static ReservationDefinition createSimpleReservationDefinition(
+ long arrival, long deadline, long duration, int parallelism,
+ String recurrenceExpression) {
// create a request with a single atomic ask
- ReservationRequest r =
- ReservationRequest.newInstance(Resource.newInstance(1024, 1),
- parallelism, parallelism, duration);
+ ReservationRequest r = ReservationRequest.newInstance(
+ Resource.newInstance(1024, 1), parallelism, parallelism, duration);
ReservationDefinition rDef = new ReservationDefinitionPBImpl();
ReservationRequests reqs = new ReservationRequestsPBImpl();
reqs.setReservationResources(Collections.singletonList(r));
@@ -195,32 +204,31 @@ public class ReservationSystemTestUtil {
rDef.setReservationRequests(reqs);
rDef.setArrival(arrival);
rDef.setDeadline(deadline);
+ if (recurrenceExpression != null) {
+ rDef.setRecurrenceExpression(recurrenceExpression);
+ }
return rDef;
}
public static ReservationSubmissionRequest createSimpleReservationRequest(
ReservationId reservationId, int numContainers, long arrival,
long deadline, long duration) {
- return createSimpleReservationRequest(reservationId, numContainers,
- arrival, deadline, duration, Priority.UNDEFINED);
+ return createSimpleReservationRequest(reservationId, numContainers, arrival,
+ deadline, duration, Priority.UNDEFINED);
}
public static ReservationSubmissionRequest createSimpleReservationRequest(
ReservationId reservationId, int numContainers, long arrival,
long deadline, long duration, Priority priority) {
// create a request with a single atomic ask
- ReservationRequest r =
- ReservationRequest.newInstance(Resource.newInstance(1024, 1),
- numContainers, 1, duration);
- ReservationRequests reqs =
- ReservationRequests.newInstance(Collections.singletonList(r),
- ReservationRequestInterpreter.R_ALL);
- ReservationDefinition rDef =
- ReservationDefinition.newInstance(arrival, deadline, reqs,
- "testClientRMService#reservation", "0", priority);
- ReservationSubmissionRequest request =
- ReservationSubmissionRequest.newInstance(rDef,
- reservationQ, reservationId);
+ ReservationRequest r = ReservationRequest
+ .newInstance(Resource.newInstance(1024, 1), numContainers, 1, duration);
+ ReservationRequests reqs = ReservationRequests.newInstance(
+ Collections.singletonList(r), ReservationRequestInterpreter.R_ALL);
+ ReservationDefinition rDef = ReservationDefinition.newInstance(arrival,
+ deadline, reqs, "testClientRMService#reservation", "0", priority);
+ ReservationSubmissionRequest request = ReservationSubmissionRequest
+ .newInstance(rDef, reservationQ, reservationId);
return request;
}
@@ -252,9 +260,9 @@ public class ReservationSystemTestUtil {
return cs;
}
- @SuppressWarnings("rawtypes") public static void initializeRMContext(
- int numContainers, AbstractYarnScheduler scheduler,
- RMContext mockRMContext) {
+ @SuppressWarnings("rawtypes")
+ public static void initializeRMContext(int numContainers,
+ AbstractYarnScheduler scheduler, RMContext mockRMContext) {
when(mockRMContext.getScheduler()).thenReturn(scheduler);
Resource r = calculateClusterResource(numContainers);
@@ -262,26 +270,25 @@ public class ReservationSystemTestUtil {
}
public static RMContext createRMContext(Configuration conf) {
- RMContext mockRmContext = Mockito.spy(
- new RMContextImpl(null, null, null, null, null, null,
- new RMContainerTokenSecretManager(conf),
- new NMTokenSecretManagerInRM(conf),
- new ClientToAMTokenSecretManagerInRM(), null));
+ RMContext mockRmContext = Mockito.spy(new RMContextImpl(null, null, null,
+ null, null, null, new RMContainerTokenSecretManager(conf),
+ new NMTokenSecretManagerInRM(conf),
+ new ClientToAMTokenSecretManagerInRM(), null));
RMNodeLabelsManager nlm = mock(RMNodeLabelsManager.class);
when(nlm.getQueueResource(any(String.class), anySetOf(String.class),
- any(Resource.class))).thenAnswer(new Answer<Resource>() {
- @Override public Resource answer(InvocationOnMock invocation)
- throws Throwable {
- Object[] args = invocation.getArguments();
- return (Resource) args[2];
- }
- });
+ any(Resource.class))).thenAnswer(new Answer<Resource>() {
+ @Override
+ public Resource answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ return (Resource) args[2];
+ }
+ });
when(nlm.getResourceByLabel(any(String.class), any(Resource.class)))
.thenAnswer(new Answer<Resource>() {
- @Override public Resource answer(InvocationOnMock invocation)
- throws Throwable {
+ @Override
+ public Resource answer(InvocationOnMock invocation) throws Throwable {
Object[] args = invocation.getArguments();
return (Resource) args[1];
}
@@ -304,9 +311,8 @@ public class ReservationSystemTestUtil {
final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setCapacity(A, 10);
- final String dedicated =
- CapacitySchedulerConfiguration.ROOT + CapacitySchedulerConfiguration.DOT
- + reservationQ;
+ final String dedicated = CapacitySchedulerConfiguration.ROOT
+ + CapacitySchedulerConfiguration.DOT + reservationQ;
conf.setCapacity(dedicated, 80);
// Set as reservation queue
conf.setReservable(dedicated, true);
@@ -405,26 +411,55 @@ public class ReservationSystemTestUtil {
public static Map<ReservationInterval, Resource> generateAllocation(
long startTime, long step, int[] alloc) {
+ return generateAllocation(startTime, step, alloc, null);
+ }
+
+ public static Map<ReservationInterval, Resource> generateAllocation(
+ long startTime, long step, int[] alloc, String recurrenceExpression) {
Map<ReservationInterval, Resource> req = new TreeMap<>();
- for (int i = 0; i < alloc.length; i++) {
- req.put(new ReservationInterval(startTime + i * step,
- startTime + (i + 1) * step), ReservationSystemUtil.toResource(
- ReservationRequest
- .newInstance(Resource.newInstance(1024, 1), alloc[i])));
+
+ long period = 0;
+ if (recurrenceExpression != null) {
+ period = Long.parseLong(recurrenceExpression);
+ }
+
+ long rStart;
+ long rEnd;
+ for (int j = 0; j < 86400000; j += period) {
+ for (int i = 0; i < alloc.length; i++) {
+ rStart = (startTime + i * step) + j * period;
+ rEnd = (startTime + (i + 1) * step) + j * period;
+ if (period > 0) {
+ rStart = rStart % period + j * period;
+ rEnd = rEnd % period + j * period;
+ if (rStart > rEnd) {
+ // skip wrap-around entry
+ continue;
+ }
+ }
+
+ req.put(new ReservationInterval(rStart, rEnd),
+ ReservationSystemUtil.toResource(ReservationRequest
+ .newInstance(Resource.newInstance(1024, 1), alloc[i])));
+
+ }
+ // execute only once if non-periodic
+ if (period == 0) {
+ break;
+ }
}
return req;
}
- public static RLESparseResourceAllocation
- generateRLESparseResourceAllocation(int[] alloc, long[] timeSteps) {
+ public static RLESparseResourceAllocation generateRLESparseResourceAllocation(
+ int[] alloc, long[] timeSteps) {
TreeMap<Long, Resource> allocationsMap = new TreeMap<>();
for (int i = 0; i < alloc.length; i++) {
allocationsMap.put(timeSteps[i],
Resource.newInstance(alloc[i], alloc[i]));
}
- RLESparseResourceAllocation rleVector =
- new RLESparseResourceAllocation(allocationsMap,
- new DefaultResourceCalculator());
+ RLESparseResourceAllocation rleVector = new RLESparseResourceAllocation(
+ allocationsMap, new DefaultResourceCalculator());
return rleVector;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/40] hadoop git commit: YARN-6978. Add updateContainer API to
NMClient. (Kartheek Muthyala via asuresh)
Posted by in...@apache.org.
YARN-6978. Add updateContainer API to NMClient. (Kartheek Muthyala via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c41118a7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c41118a7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c41118a7
Branch: refs/heads/HDFS-10467
Commit: c41118a7f826dcbb269b4bd3d5877b35fbbee2b1
Parents: 13eda50
Author: Arun Suresh <as...@apache.org>
Authored: Thu Sep 7 10:23:12 2017 -0700
Committer: Arun Suresh <as...@apache.org>
Committed: Thu Sep 7 10:23:12 2017 -0700
----------------------------------------------------------------------
.../distributedshell/ApplicationMaster.java | 19 +++-
.../apache/hadoop/yarn/client/api/NMClient.java | 18 ++++
.../yarn/client/api/async/NMClientAsync.java | 36 ++++++++
.../api/async/impl/NMClientAsyncImpl.java | 92 +++++++++++++++-----
.../yarn/client/api/impl/NMClientImpl.java | 29 ++++++
.../api/async/impl/TestNMClientAsync.java | 60 ++++++++++++-
.../yarn/client/api/impl/TestAMRMClient.java | 1 +
.../yarn/client/api/impl/TestNMClient.java | 5 +-
8 files changed, 227 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c41118a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index a02af70..5ec9409 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -1027,10 +1027,6 @@ public class ApplicationMaster {
}
@Override
- public void onContainerResourceIncreased(
- ContainerId containerId, Resource resource) {}
-
- @Override
public void onStartContainerError(ContainerId containerId, Throwable t) {
LOG.error("Failed to start Container " + containerId, t);
containers.remove(containerId);
@@ -1050,10 +1046,25 @@ public class ApplicationMaster {
containers.remove(containerId);
}
+ @Deprecated
@Override
public void onIncreaseContainerResourceError(
ContainerId containerId, Throwable t) {}
+ @Deprecated
+ @Override
+ public void onContainerResourceIncreased(
+ ContainerId containerId, Resource resource) {}
+
+ @Override
+ public void onUpdateContainerResourceError(
+ ContainerId containerId, Throwable t) {
+ }
+
+ @Override
+ public void onContainerResourceUpdated(ContainerId containerId,
+ Resource resource) {
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c41118a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java
index c1447ba..17168f7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java
@@ -104,10 +104,28 @@ public abstract class NMClient extends AbstractService {
* @throws YarnException YarnException.
* @throws IOException IOException.
*/
+ @Deprecated
public abstract void increaseContainerResource(Container container)
throws YarnException, IOException;
/**
+ * <p>Update the resources of a container.</p>
+ *
+ * <p>The <code>ApplicationMaster</code> or other applications that use the
+ * client must provide the details of the container, including the Id and
+ * the target resource encapsulated in the updated container token via
+ * {@link Container}.
+ * </p>
+ *
+ * @param container the container with updated token.
+ *
+ * @throws YarnException YarnException.
+ * @throws IOException IOException.
+ */
+ public abstract void updateContainerResource(Container container)
+ throws YarnException, IOException;
+
+ /**
* <p>Stop an started container.</p>
*
* @param containerId the Id of the started container
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c41118a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/NMClientAsync.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/NMClientAsync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/NMClientAsync.java
index c94942a..62e2d99 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/NMClientAsync.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/NMClientAsync.java
@@ -177,9 +177,23 @@ public abstract class NMClientAsync extends AbstractService {
public abstract void startContainerAsync(
Container container, ContainerLaunchContext containerLaunchContext);
+ @Deprecated
public abstract void increaseContainerResourceAsync(Container container);
/**
+ * <p>Update the resources of a container.</p>
+ *
+ * <p>The <code>ApplicationMaster</code> or other applications that use the
+ * client must provide the details of the container, including the Id and
+ * the target resource encapsulated in the updated container token via
+ * {@link Container}.
+ * </p>
+ *
+ * @param container the container with updated token.
+ */
+ public abstract void updateContainerResourceAsync(Container container);
+
+ /**
* <p>Re-Initialize the Container.</p>
*
* @param containerId the Id of the container to Re-Initialize.
@@ -301,10 +315,21 @@ public abstract class NMClientAsync extends AbstractService {
* @param containerId the Id of the container
* @param resource the target resource of the container
*/
+ @Deprecated
public abstract void onContainerResourceIncreased(
ContainerId containerId, Resource resource);
/**
+ * The API is called when <code>NodeManager</code> responds to indicate
+ * the container resource has been successfully updated.
+ *
+ * @param containerId the Id of the container
+ * @param resource the target resource of the container
+ */
+ public abstract void onContainerResourceUpdated(
+ ContainerId containerId, Resource resource);
+
+ /**
* The API is called when an exception is raised in the process of
* querying the status of a container.
*
@@ -321,11 +346,22 @@ public abstract class NMClientAsync extends AbstractService {
* @param containerId the Id of the container
* @param t the raised exception
*/
+ @Deprecated
public abstract void onIncreaseContainerResourceError(
ContainerId containerId, Throwable t);
/**
* The API is called when an exception is raised in the process of
+ * updating container resource.
+ *
+ * @param containerId the Id of the container
+ * @param t the raised exception
+ */
+ public abstract void onUpdateContainerResourceError(
+ ContainerId containerId, Throwable t);
+
+ /**
+ * The API is called when an exception is raised in the process of
* stopping a container.
*
* @param containerId the Id of the container
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c41118a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/NMClientAsyncImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/NMClientAsyncImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/NMClientAsyncImpl.java
index 20be71e..4d14180 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/NMClientAsyncImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/NMClientAsyncImpl.java
@@ -259,6 +259,7 @@ public class NMClientAsyncImpl extends NMClientAsync {
}
}
+ @Deprecated
public void increaseContainerResourceAsync(Container container) {
if (!(callbackHandler instanceof AbstractCallbackHandler)) {
LOG.error("Callback handler does not implement container resource "
@@ -274,7 +275,7 @@ public class NMClientAsyncImpl extends NMClientAsync {
" is neither started nor scheduled to start"));
}
try {
- events.put(new IncreaseContainerResourceEvent(container));
+ events.put(new UpdateContainerResourceEvent(container, true));
} catch (InterruptedException e) {
LOG.warn("Exception when scheduling the event of increasing resource of "
+ "Container " + container.getId());
@@ -283,6 +284,30 @@ public class NMClientAsyncImpl extends NMClientAsync {
}
@Override
+ public void updateContainerResourceAsync(Container container) {
+ if (!(callbackHandler instanceof AbstractCallbackHandler)) {
+ LOG.error("Callback handler does not implement container resource "
+ + "increase callback methods");
+ return;
+ }
+ AbstractCallbackHandler handler = (AbstractCallbackHandler) callbackHandler;
+ if (containers.get(container.getId()) == null) {
+ handler.onUpdateContainerResourceError(
+ container.getId(),
+ RPCUtil.getRemoteException(
+ "Container " + container.getId() +
+ " is neither started nor scheduled to start"));
+ }
+ try {
+ events.put(new UpdateContainerResourceEvent(container, false));
+ } catch (InterruptedException e) {
+ LOG.warn("Exception when scheduling the event of increasing resource of "
+ + "Container " + container.getId());
+ handler.onUpdateContainerResourceError(container.getId(), e);
+ }
+ }
+
+ @Override
public void reInitializeContainerAsync(ContainerId containerId,
ContainerLaunchContext containerLaunchContex, boolean autoCommit){
if (!(callbackHandler instanceof AbstractCallbackHandler)) {
@@ -427,7 +452,7 @@ public class NMClientAsyncImpl extends NMClientAsync {
START_CONTAINER,
STOP_CONTAINER,
QUERY_CONTAINER,
- INCREASE_CONTAINER_RESOURCE,
+ UPDATE_CONTAINER_RESOURCE,
REINITIALIZE_CONTAINER,
RESTART_CONTAINER,
ROLLBACK_LAST_REINIT,
@@ -503,14 +528,20 @@ public class NMClientAsyncImpl extends NMClientAsync {
}
}
- protected static class IncreaseContainerResourceEvent extends ContainerEvent {
+ protected static class UpdateContainerResourceEvent extends ContainerEvent {
private Container container;
+ private boolean isIncreaseEvent;
- public IncreaseContainerResourceEvent(Container container) {
+ // UpdateContainerResourceEvent constructor takes in a
+ // flag to support callback API's calling through the deprecated
+ // increaseContainerResource
+ public UpdateContainerResourceEvent(Container container,
+ boolean isIncreaseEvent) {
super(container.getId(), container.getNodeId(),
container.getContainerToken(),
- ContainerEventType.INCREASE_CONTAINER_RESOURCE);
+ ContainerEventType.UPDATE_CONTAINER_RESOURCE);
this.container = container;
+ this.isIncreaseEvent = isIncreaseEvent;
}
public Container getContainer() {
@@ -536,8 +567,8 @@ public class NMClientAsyncImpl extends NMClientAsync {
// Transitions from RUNNING state
.addTransition(ContainerState.RUNNING, ContainerState.RUNNING,
- ContainerEventType.INCREASE_CONTAINER_RESOURCE,
- new IncreaseContainerResourceTransition())
+ ContainerEventType.UPDATE_CONTAINER_RESOURCE,
+ new UpdateContainerResourceTransition())
// Transitions for Container Upgrade
.addTransition(ContainerState.RUNNING,
@@ -566,7 +597,7 @@ public class NMClientAsyncImpl extends NMClientAsync {
.addTransition(ContainerState.DONE, ContainerState.DONE,
EnumSet.of(ContainerEventType.START_CONTAINER,
ContainerEventType.STOP_CONTAINER,
- ContainerEventType.INCREASE_CONTAINER_RESOURCE))
+ ContainerEventType.UPDATE_CONTAINER_RESOURCE))
// Transition from FAILED state
.addTransition(ContainerState.FAILED, ContainerState.FAILED,
@@ -576,7 +607,7 @@ public class NMClientAsyncImpl extends NMClientAsync {
ContainerEventType.RESTART_CONTAINER,
ContainerEventType.COMMIT_LAST_REINT,
ContainerEventType.ROLLBACK_LAST_REINIT,
- ContainerEventType.INCREASE_CONTAINER_RESOURCE));
+ ContainerEventType.UPDATE_CONTAINER_RESOURCE));
protected static class StartContainerTransition implements
MultipleArcTransition<StatefulContainer, ContainerEvent,
@@ -628,46 +659,61 @@ public class NMClientAsyncImpl extends NMClientAsync {
}
}
- protected static class IncreaseContainerResourceTransition implements
+ protected static class UpdateContainerResourceTransition implements
SingleArcTransition<StatefulContainer, ContainerEvent> {
+
+ @SuppressWarnings("deprecation")
@Override
public void transition(
StatefulContainer container, ContainerEvent event) {
+ boolean isIncreaseEvent = false;
if (!(container.nmClientAsync.getCallbackHandler()
instanceof AbstractCallbackHandler)) {
LOG.error("Callback handler does not implement container resource "
- + "increase callback methods");
+ + "update callback methods");
return;
}
AbstractCallbackHandler handler =
(AbstractCallbackHandler) container.nmClientAsync
.getCallbackHandler();
try {
- if (!(event instanceof IncreaseContainerResourceEvent)) {
+ if (!(event instanceof UpdateContainerResourceEvent)) {
throw new AssertionError("Unexpected event type. Expecting:"
- + "IncreaseContainerResourceEvent. Got:" + event);
+ + "UpdateContainerResourceEvent. Got:" + event);
}
- IncreaseContainerResourceEvent increaseEvent =
- (IncreaseContainerResourceEvent) event;
- container.nmClientAsync.getClient().increaseContainerResource(
- increaseEvent.getContainer());
+ UpdateContainerResourceEvent updateEvent =
+ (UpdateContainerResourceEvent) event;
+ container.nmClientAsync.getClient().updateContainerResource(
+ updateEvent.getContainer());
+ isIncreaseEvent = updateEvent.isIncreaseEvent;
try {
- handler.onContainerResourceIncreased(
- increaseEvent.getContainerId(), increaseEvent.getContainer()
- .getResource());
+ //If isIncreaseEvent is set, set the appropriate callbacks
+ //for backward compatibility
+ if (isIncreaseEvent) {
+ handler.onContainerResourceIncreased(updateEvent.getContainerId(),
+ updateEvent.getContainer().getResource());
+ } else {
+ handler.onContainerResourceUpdated(updateEvent.getContainerId(),
+ updateEvent.getContainer().getResource());
+ }
} catch (Throwable thr) {
// Don't process user created unchecked exception
LOG.info("Unchecked exception is thrown from "
- + "onContainerResourceIncreased for Container "
+ + "onContainerResourceUpdated for Container "
+ event.getContainerId(), thr);
}
} catch (Exception e) {
try {
- handler.onIncreaseContainerResourceError(event.getContainerId(), e);
+ if (isIncreaseEvent) {
+ handler
+ .onIncreaseContainerResourceError(event.getContainerId(), e);
+ } else {
+ handler.onUpdateContainerResourceError(event.getContainerId(), e);
+ }
} catch (Throwable thr) {
// Don't process user created unchecked exception
LOG.info("Unchecked exception is thrown from "
- + "onIncreaseContainerResourceError for Container "
+ + "onUpdateContainerResourceError for Container "
+ event.getContainerId(), thr);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c41118a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java
index 8171de2..ca44e2e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java
@@ -230,6 +230,7 @@ public class NMClientImpl extends NMClient {
}
}
+ @Deprecated
@Override
public void increaseContainerResource(Container container)
throws YarnException, IOException {
@@ -259,6 +260,34 @@ public class NMClientImpl extends NMClient {
}
@Override
+ public void updateContainerResource(Container container)
+ throws YarnException, IOException {
+ ContainerManagementProtocolProxyData proxy = null;
+ try {
+ proxy =
+ cmProxy.getProxy(container.getNodeId().toString(), container.getId());
+ List<Token> updateTokens = new ArrayList<>();
+ updateTokens.add(container.getContainerToken());
+
+ ContainerUpdateRequest request =
+ ContainerUpdateRequest.newInstance(updateTokens);
+ ContainerUpdateResponse response =
+ proxy.getContainerManagementProtocol().updateContainer(request);
+
+ if (response.getFailedRequests() != null && response.getFailedRequests()
+ .containsKey(container.getId())) {
+ Throwable t =
+ response.getFailedRequests().get(container.getId()).deSerialize();
+ parseAndThrowException(t);
+ }
+ } finally {
+ if (proxy != null) {
+ cmProxy.mayBeCloseProxy(proxy);
+ }
+ }
+ }
+
+ @Override
public void stopContainer(ContainerId containerId, NodeId nodeId)
throws YarnException, IOException {
StartedContainer startedContainer = startedContainers.get(containerId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c41118a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestNMClientAsync.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestNMClientAsync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestNMClientAsync.java
index dda3eec..6c72704 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestNMClientAsync.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestNMClientAsync.java
@@ -253,7 +253,7 @@ public class TestNMClientAsync {
int t = containerId.getId() % 5;
switch (t) {
case 0:
- asyncClient.increaseContainerResourceAsync(container);
+ asyncClient.updateContainerResourceAsync(container);
break;
case 1:
asyncClient.reInitializeContainerAsync(containerId,
@@ -295,7 +295,7 @@ public class TestNMClientAsync {
// containerId
Container container = Container.newInstance(
containerId, nodeId, null, null, null, containerToken);
- asyncClient.increaseContainerResourceAsync(container);
+ asyncClient.updateContainerResourceAsync(container);
// Shouldn't crash the test thread
throw new RuntimeException("Ignorable Exception");
@@ -322,6 +322,25 @@ public class TestNMClientAsync {
@SuppressWarnings("deprecation")
@Override
+ public void onContainerResourceUpdated(ContainerId containerId,
+ Resource resource) {
+ if (containerId.getId() >= expectedSuccess) {
+ errorMsgs.add("Container " + containerId +
+ " should throw the exception onContainerResourceUpdated");
+ return;
+ }
+ TestData td = testMap.get(OpsToTest.INCR);
+ td.success.addAndGet(1);
+ td.successArray.set(containerId.getId(), 1);
+ // move on to the following success tests
+ asyncClient.reInitializeContainerAsync(containerId,
+ Records.newRecord(ContainerLaunchContext.class), true);
+ // throw a fake user exception, and shouldn't crash the test
+ throw new RuntimeException("Ignorable Exception");
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
public void onContainerReInitialize(ContainerId containerId) {
if (containerId.getId() >= expectedSuccess) {
errorMsgs.add("Container " + containerId +
@@ -452,6 +471,27 @@ public class TestNMClientAsync {
@SuppressWarnings("deprecation")
@Override
+ public void onUpdateContainerResourceError(ContainerId containerId,
+ Throwable t) {
+ if (containerId.getId() < expectedSuccess + expectedFailure) {
+ errorMsgs.add("Container " + containerId +
+ " shouldn't throw the exception onUpdatedContainerResourceError");
+ return;
+ }
+ TestData td = testMap.get(OpsToTest.INCR);
+ td.failure.addAndGet(1);
+ td.failureArray.set(
+ containerId.getId() - expectedSuccess - expectedFailure, 1);
+ // increase container resource error should NOT change the
+ // the container status to FAILED
+ // move on to the following failure tests
+ asyncClient.stopContainerAsync(containerId, nodeId);
+ // Shouldn't crash the test thread
+ throw new RuntimeException("Ignorable Exception");
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
public void onContainerReInitializeError(ContainerId containerId,
Throwable t) {
if (containerId.getId() < expectedSuccess + expectedFailure) {
@@ -673,7 +713,7 @@ public class TestNMClientAsync {
when(client.getContainerStatus(any(ContainerId.class),
any(NodeId.class))).thenReturn(
recordFactory.newRecordInstance(ContainerStatus.class));
- doNothing().when(client).increaseContainerResource(
+ doNothing().when(client).updateContainerResource(
any(Container.class));
doNothing().when(client).reInitializeContainer(
any(ContainerId.class), any(ContainerLaunchContext.class),
@@ -703,7 +743,7 @@ public class TestNMClientAsync {
any(NodeId.class))).thenReturn(
recordFactory.newRecordInstance(ContainerStatus.class));
doThrow(RPCUtil.getRemoteException("Increase Resource Exception"))
- .when(client).increaseContainerResource(any(Container.class));
+ .when(client).updateContainerResource(any(Container.class));
doThrow(RPCUtil.getRemoteException("ReInitialize Exception"))
.when(client).reInitializeContainer(
any(ContainerId.class), any(ContainerLaunchContext.class),
@@ -818,11 +858,17 @@ public class TestNMClientAsync {
ContainerStatus containerStatus) {
}
+ @Deprecated
@Override
public void onContainerResourceIncreased(
ContainerId containerId, Resource resource) {}
@Override
+ public void onContainerResourceUpdated(ContainerId containerId,
+ Resource resource) {
+ }
+
+ @Override
public void onContainerStopped(ContainerId containerId) {
}
@@ -847,11 +893,17 @@ public class TestNMClientAsync {
Throwable t) {
}
+ @Deprecated
@Override
public void onIncreaseContainerResourceError(
ContainerId containerId, Throwable t) {}
@Override
+ public void onUpdateContainerResourceError(ContainerId containerId,
+ Throwable t) {
+ }
+
+ @Override
public void onStopContainerError(ContainerId containerId, Throwable t) {
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c41118a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
index 09b12f2..fa2e7a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
@@ -1428,6 +1428,7 @@ public class TestAMRMClient {
amClient.ask.clear();
}
+ @SuppressWarnings("deprecation")
private void updateContainerExecType(AllocateResponse allocResponse,
ExecutionType expectedExecType, NMClientImpl nmClient)
throws IOException, YarnException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c41118a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
index 9b79e2d..b23a923 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
@@ -301,10 +301,10 @@ public class TestNMClient {
assertTrue("The thrown exception is not expected",
e.getMessage().contains("is not handled by this NodeManager"));
}
- // increaseContainerResource shouldn't be called before startContainer,
+ // upadateContainerResource shouldn't be called before startContainer,
// otherwise, NodeManager cannot find the container
try {
- nmClient.increaseContainerResource(container);
+ nmClient.updateContainerResource(container);
fail("Exception is expected");
} catch (YarnException e) {
assertTrue("The thrown exception is not expected",
@@ -469,6 +469,7 @@ public class TestNMClient {
}
}
+ @SuppressWarnings("deprecation")
private void testIncreaseContainerResource(Container container)
throws YarnException, IOException {
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/40] hadoop git commit: HDFS-10646. Federation admin tool.
Contributed by Inigo Goiri.
Posted by in...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba43eea/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
new file mode 100644
index 0000000..170247f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
@@ -0,0 +1,261 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.synchronizeRecords;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
+import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.impl.MountTableStoreImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.util.Time;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * The administrator interface of the {@link Router} implemented by
+ * {@link RouterAdminServer}.
+ */
+public class TestRouterAdmin {
+
+ private static StateStoreDFSCluster cluster;
+ private static RouterContext routerContext;
+ public static final String RPC_BEAN =
+ "Hadoop:service=Router,name=FederationRPC";
+ private static List<MountTable> mockMountTable;
+ private static StateStoreService stateStore;
+
+ @BeforeClass
+ public static void globalSetUp() throws Exception {
+ cluster = new StateStoreDFSCluster(false, 1);
+ // Build and start a router with State Store + admin + RPC
+ Configuration conf = new RouterConfigBuilder()
+ .stateStore()
+ .admin()
+ .rpc()
+ .build();
+ cluster.addRouterOverrides(conf);
+ cluster.startRouters();
+ routerContext = cluster.getRandomRouter();
+ mockMountTable = cluster.generateMockMountTable();
+ Router router = routerContext.getRouter();
+ stateStore = router.getStateStore();
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ cluster.stopRouter(routerContext);
+ }
+
+ @Before
+ public void testSetup() throws Exception {
+ assertTrue(
+ synchronizeRecords(stateStore, mockMountTable, MountTable.class));
+ }
+
+ @Test
+ public void testAddMountTable() throws IOException {
+ MountTable newEntry = MountTable.newInstance(
+ "/testpath", Collections.singletonMap("ns0", "/testdir"),
+ Time.now(), Time.now());
+
+ RouterClient client = routerContext.getAdminClient();
+ MountTableManager mountTable = client.getMountTableManager();
+
+ // Existing mount table size
+ List<MountTable> records = getMountTableEntries(mountTable);
+ assertEquals(records.size(), mockMountTable.size());
+
+ // Add
+ AddMountTableEntryRequest addRequest =
+ AddMountTableEntryRequest.newInstance(newEntry);
+ AddMountTableEntryResponse addResponse =
+ mountTable.addMountTableEntry(addRequest);
+ assertTrue(addResponse.getStatus());
+
+ // New mount table size
+ List<MountTable> records2 = getMountTableEntries(mountTable);
+ assertEquals(records2.size(), mockMountTable.size() + 1);
+ }
+
+ @Test
+ public void testAddDuplicateMountTable() throws IOException {
+ MountTable newEntry = MountTable.newInstance("/testpath",
+ Collections.singletonMap("ns0", "/testdir"), Time.now(), Time.now());
+
+ RouterClient client = routerContext.getAdminClient();
+ MountTableManager mountTable = client.getMountTableManager();
+
+ // Existing mount table size
+ List<MountTable> entries1 = getMountTableEntries(mountTable);
+ assertEquals(entries1.size(), mockMountTable.size());
+
+ // Add
+ AddMountTableEntryRequest addRequest =
+ AddMountTableEntryRequest.newInstance(newEntry);
+ AddMountTableEntryResponse addResponse =
+ mountTable.addMountTableEntry(addRequest);
+ assertTrue(addResponse.getStatus());
+
+ // New mount table size
+ List<MountTable> entries2 = getMountTableEntries(mountTable);
+ assertEquals(entries2.size(), mockMountTable.size() + 1);
+
+ // Add again, should fail
+ AddMountTableEntryResponse addResponse2 =
+ mountTable.addMountTableEntry(addRequest);
+ assertFalse(addResponse2.getStatus());
+ }
+
+ @Test
+ public void testRemoveMountTable() throws IOException {
+
+ RouterClient client = routerContext.getAdminClient();
+ MountTableManager mountTable = client.getMountTableManager();
+
+ // Existing mount table size
+ List<MountTable> entries1 = getMountTableEntries(mountTable);
+ assertEquals(entries1.size(), mockMountTable.size());
+
+ // Remove an entry
+ RemoveMountTableEntryRequest removeRequest =
+ RemoveMountTableEntryRequest.newInstance("/");
+ mountTable.removeMountTableEntry(removeRequest);
+
+ // New mount table size
+ List<MountTable> entries2 = getMountTableEntries(mountTable);
+ assertEquals(entries2.size(), mockMountTable.size() - 1);
+ }
+
+ @Test
+ public void testEditMountTable() throws IOException {
+
+ RouterClient client = routerContext.getAdminClient();
+ MountTableManager mountTable = client.getMountTableManager();
+
+ // Verify starting condition
+ MountTable entry = getMountTableEntry("/");
+ assertEquals(
+ Collections.singletonList(new RemoteLocation("ns0", "/")),
+ entry.getDestinations());
+
+ // Edit the entry for /
+ MountTable updatedEntry = MountTable.newInstance(
+ "/", Collections.singletonMap("ns1", "/"), Time.now(), Time.now());
+ UpdateMountTableEntryRequest updateRequest =
+ UpdateMountTableEntryRequest.newInstance(updatedEntry);
+ mountTable.updateMountTableEntry(updateRequest);
+
+ // Verify edited condition
+ entry = getMountTableEntry("/");
+ assertEquals(
+ Collections.singletonList(new RemoteLocation("ns1", "/")),
+ entry.getDestinations());
+ }
+
+ @Test
+ public void testGetMountTable() throws IOException {
+
+ RouterClient client = routerContext.getAdminClient();
+ MountTableManager mountTable = client.getMountTableManager();
+
+ // Verify size of table
+ List<MountTable> entries = getMountTableEntries(mountTable);
+ assertEquals(mockMountTable.size(), entries.size());
+
+ // Verify all entries are present
+ int matches = 0;
+ for (MountTable e : entries) {
+ for (MountTable entry : mockMountTable) {
+ assertEquals(e.getDestinations().size(), 1);
+ assertNotNull(e.getDateCreated());
+ assertNotNull(e.getDateModified());
+ if (entry.getSourcePath().equals(e.getSourcePath())) {
+ matches++;
+ }
+ }
+ }
+ assertEquals(matches, mockMountTable.size());
+ }
+
+ @Test
+ public void testGetSingleMountTableEntry() throws IOException {
+ MountTable entry = getMountTableEntry("/ns0");
+ assertNotNull(entry);
+ assertEquals(entry.getSourcePath(), "/ns0");
+ }
+
+ /**
+ * Gets an existing mount table record in the state store.
+ *
+ * @param mount The mount point of the record to remove.
+ * @return The matching record if found, null if it is not found.
+ * @throws IOException If the state store could not be accessed.
+ */
+ private MountTable getMountTableEntry(final String mount) throws IOException {
+ // Refresh the cache
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+
+ GetMountTableEntriesRequest request =
+ GetMountTableEntriesRequest.newInstance(mount);
+ RouterClient client = routerContext.getAdminClient();
+ MountTableManager mountTable = client.getMountTableManager();
+ List<MountTable> results = getMountTableEntries(mountTable, request);
+ if (results.size() > 0) {
+ // First result is sorted to have the shortest mount string length
+ return results.get(0);
+ }
+ return null;
+ }
+
+ private List<MountTable> getMountTableEntries(MountTableManager mountTable)
+ throws IOException {
+ GetMountTableEntriesRequest request =
+ GetMountTableEntriesRequest.newInstance("/");
+ return getMountTableEntries(mountTable, request);
+ }
+
+ private List<MountTable> getMountTableEntries(MountTableManager mountTable,
+ GetMountTableEntriesRequest request) throws IOException {
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ GetMountTableEntriesResponse response =
+ mountTable.getMountTableEntries(request);
+ return response.getEntries();
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/40] hadoop git commit: HDFS-11546. Federation Router RPC server.
Contributed by Jason Kace and Inigo Goiri.
Posted by in...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e96a2c7f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
index 24792bb..4bae71e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
@@ -17,16 +17,109 @@
*/
package org.apache.hadoop.hdfs.server.federation.router;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HANDLER_COUNT_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HANDLER_COUNT_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HANDLER_QUEUE_SIZE_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HANDLER_QUEUE_SIZE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_READER_COUNT_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_READER_COUNT_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_READER_QUEUE_SIZE_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_READER_QUEUE_SIZE_KEY;
+
+import java.io.FileNotFoundException;
import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.CryptoProtocolVersion;
+import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries;
+import org.apache.hadoop.fs.CacheFlag;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FsServerDefaults;
+import org.apache.hadoop.fs.Options;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.QuotaUsage;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.fs.XAttr;
+import org.apache.hadoop.fs.XAttrSetFlag;
+import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.AddBlockFlag;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.inotify.EventBatchList;
+import org.apache.hadoop.hdfs.protocol.AddingECPolicyResponse;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
+import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
+import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
+import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.EncryptionZone;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ClientNamenodeProtocol;
+import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB;
+import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB;
+import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException;
+import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
+import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException;
+import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
+import org.apache.hadoop.io.EnumSetWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RPC.Server;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.ipc.StandbyException;
+import org.apache.hadoop.net.NodeBase;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.service.AbstractService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
+import com.google.protobuf.BlockingService;
/**
* This class is responsible for handling all of the RPC calls to the It is
@@ -36,10 +129,42 @@ import com.google.common.annotations.VisibleForTesting;
* the requests to the active
* {@link org.apache.hadoop.hdfs.server.namenode.NameNode NameNode}.
*/
-public class RouterRpcServer extends AbstractService {
+public class RouterRpcServer extends AbstractService implements ClientProtocol {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RouterRpcServer.class);
+
+
+ /** Configuration for the RPC server. */
+ private Configuration conf;
+
+ /** Identifier for the super user. */
+ private final String superUser;
+ /** Identifier for the super group. */
+ private final String superGroup;
+
+ /** Router using this RPC server. */
+ private final Router router;
/** The RPC server that listens to requests from clients. */
private final Server rpcServer;
+ /** The address for this RPC server. */
+ private final InetSocketAddress rpcAddress;
+
+ /** RPC clients to connect to the Namenodes. */
+ private final RouterRpcClient rpcClient;
+
+
+ /** Interface to identify the active NN for a nameservice or blockpool ID. */
+ private final ActiveNamenodeResolver namenodeResolver;
+
+ /** Interface to map global name space to HDFS subcluster name spaces. */
+ private final FileSubclusterResolver subclusterResolver;
+
+
+ /** Category of the operation that a thread is executing. */
+ private final ThreadLocal<OperationCategory> opCategory = new ThreadLocal<>();
+
/**
* Construct a router RPC server.
@@ -54,31 +179,94 @@ public class RouterRpcServer extends AbstractService {
throws IOException {
super(RouterRpcServer.class.getName());
- this.rpcServer = null;
- }
+ this.conf = configuration;
+ this.router = router;
+ this.namenodeResolver = nnResolver;
+ this.subclusterResolver = fileResolver;
- /**
- * Allow access to the client RPC server for testing.
- *
- * @return The RPC server.
- */
- @VisibleForTesting
- public Server getServer() {
- return this.rpcServer;
+ // User and group for reporting
+ this.superUser = System.getProperty("user.name");
+ this.superGroup = this.conf.get(
+ DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_KEY,
+ DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT);
+
+ // RPC server settings
+ int handlerCount = this.conf.getInt(DFS_ROUTER_HANDLER_COUNT_KEY,
+ DFS_ROUTER_HANDLER_COUNT_DEFAULT);
+
+ int readerCount = this.conf.getInt(DFS_ROUTER_READER_COUNT_KEY,
+ DFS_ROUTER_READER_COUNT_DEFAULT);
+
+ int handlerQueueSize = this.conf.getInt(DFS_ROUTER_HANDLER_QUEUE_SIZE_KEY,
+ DFS_ROUTER_HANDLER_QUEUE_SIZE_DEFAULT);
+
+ // Override Hadoop Common IPC setting
+ int readerQueueSize = this.conf.getInt(DFS_ROUTER_READER_QUEUE_SIZE_KEY,
+ DFS_ROUTER_READER_QUEUE_SIZE_DEFAULT);
+ this.conf.setInt(
+ CommonConfigurationKeys.IPC_SERVER_RPC_READ_CONNECTION_QUEUE_SIZE_KEY,
+ readerQueueSize);
+
+ RPC.setProtocolEngine(this.conf, ClientNamenodeProtocolPB.class,
+ ProtobufRpcEngine.class);
+
+ ClientNamenodeProtocolServerSideTranslatorPB
+ clientProtocolServerTranslator =
+ new ClientNamenodeProtocolServerSideTranslatorPB(this);
+ BlockingService clientNNPbService = ClientNamenodeProtocol
+ .newReflectiveBlockingService(clientProtocolServerTranslator);
+
+ InetSocketAddress confRpcAddress = conf.getSocketAddr(
+ DFSConfigKeys.DFS_ROUTER_RPC_BIND_HOST_KEY,
+ DFSConfigKeys.DFS_ROUTER_RPC_ADDRESS_KEY,
+ DFSConfigKeys.DFS_ROUTER_RPC_ADDRESS_DEFAULT,
+ DFSConfigKeys.DFS_ROUTER_RPC_PORT_DEFAULT);
+ LOG.info("RPC server binding to {} with {} handlers for Router {}",
+ confRpcAddress, handlerCount, this.router.getRouterId());
+
+ this.rpcServer = new RPC.Builder(this.conf)
+ .setProtocol(ClientNamenodeProtocolPB.class)
+ .setInstance(clientNNPbService)
+ .setBindAddress(confRpcAddress.getHostName())
+ .setPort(confRpcAddress.getPort())
+ .setNumHandlers(handlerCount)
+ .setnumReaders(readerCount)
+ .setQueueSizePerHandler(handlerQueueSize)
+ .setVerbose(false)
+ .build();
+ // We don't want the server to log the full stack trace for some exceptions
+ this.rpcServer.addTerseExceptions(
+ RemoteException.class,
+ StandbyException.class,
+ SafeModeException.class,
+ FileNotFoundException.class,
+ FileAlreadyExistsException.class,
+ AccessControlException.class,
+ LeaseExpiredException.class,
+ NotReplicatedYetException.class,
+ IOException.class);
+
+ // The RPC-server port can be ephemeral... ensure we have the correct info
+ InetSocketAddress listenAddress = this.rpcServer.getListenerAddress();
+ this.rpcAddress = new InetSocketAddress(
+ confRpcAddress.getHostName(), listenAddress.getPort());
+
+ // Create the client
+ this.rpcClient = new RouterRpcClient(this.conf, this.router.getRouterId(),
+ this.namenodeResolver);
}
@Override
protected void serviceInit(Configuration configuration) throws Exception {
+ this.conf = configuration;
super.serviceInit(configuration);
}
- /**
- * Start client and service RPC servers.
- */
@Override
protected void serviceStart() throws Exception {
if (this.rpcServer != null) {
this.rpcServer.start();
+ LOG.info("Router RPC up at: {}", this.getRpcAddress());
}
super.serviceStart();
}
@@ -92,11 +280,1652 @@ public class RouterRpcServer extends AbstractService {
}
/**
- * Wait until the RPC servers have shutdown.
+ * Get the RPC client to the Namenode.
+ *
+ * @return RPC clients to the Namenodes.
*/
- void join() throws InterruptedException {
- if (this.rpcServer != null) {
- this.rpcServer.join();
+ public RouterRpcClient getRPCClient() {
+ return rpcClient;
+ }
+
+ /**
+ * Allow access to the client RPC server for testing.
+ *
+ * @return The RPC server.
+ */
+ @VisibleForTesting
+ public Server getServer() {
+ return rpcServer;
+ }
+
+ /**
+ * Get the RPC address of the service.
+ *
+ * @return RPC service address.
+ */
+ public InetSocketAddress getRpcAddress() {
+ return rpcAddress;
+ }
+
+ /**
+ * Check if the Router is in safe mode. We should only see READ, WRITE, and
+ * UNCHECKED. It includes a default handler when we haven't implemented an
+ * operation. If not supported, it always throws an exception reporting the
+ * operation.
+ *
+ * @param op Category of the operation to check.
+ * @param supported If the operation is supported or not. If not, it will
+ * throw an UnsupportedOperationException.
+ * @throws StandbyException If the Router is in safe mode and cannot serve
+ * client requests.
+ * @throws UnsupportedOperationException If the operation is not supported.
+ */
+ private void checkOperation(OperationCategory op, boolean supported)
+ throws StandbyException, UnsupportedOperationException {
+ checkOperation(op);
+
+ if (!supported) {
+ String methodName = getMethodName();
+ throw new UnsupportedOperationException(
+ "Operation \"" + methodName + "\" is not supported");
+ }
+ }
+
+ /**
+ * Check if the Router is in safe mode. We should only see READ, WRITE, and
+ * UNCHECKED. This function should be called by all ClientProtocol functions.
+ *
+ * @param op Category of the operation to check.
+ * @throws StandbyException If the Router is in safe mode and cannot serve
+ * client requests.
+ */
+ private void checkOperation(OperationCategory op) throws StandbyException {
+ // Log the function we are currently calling.
+ if (LOG.isDebugEnabled()) {
+ String methodName = getMethodName();
+ LOG.debug("Proxying operation: {}", methodName);
+ }
+
+ // Store the category of the operation category for this thread
+ opCategory.set(op);
+
+ // We allow unchecked and read operations
+ if (op == OperationCategory.UNCHECKED || op == OperationCategory.READ) {
+ return;
+ }
+
+ // TODO check Router safe mode and return Standby exception
+ }
+
+ @Override // ClientProtocol
+ public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ return null;
+ }
+
+ /**
+ * The the delegation token from each name service.
+ * @param renewer
+ * @return Name service -> Token.
+ * @throws IOException
+ */
+ public Map<FederationNamespaceInfo, Token<DelegationTokenIdentifier>>
+ getDelegationTokens(Text renewer) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ return null;
+ }
+
+ @Override // ClientProtocol
+ public long renewDelegationToken(Token<DelegationTokenIdentifier> token)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ return 0;
+ }
+
+ @Override // ClientProtocol
+ public void cancelDelegationToken(Token<DelegationTokenIdentifier> token)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // ClientProtocol
+ public LocatedBlocks getBlockLocations(String src, final long offset,
+ final long length) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod remoteMethod = new RemoteMethod("getBlockLocations",
+ new Class<?>[] {String.class, long.class, long.class},
+ new RemoteParam(), offset, length);
+ return (LocatedBlocks) rpcClient.invokeSequential(locations, remoteMethod,
+ LocatedBlocks.class, null);
+ }
+
+ @Override // ClientProtocol
+ public FsServerDefaults getServerDefaults() throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ RemoteMethod method = new RemoteMethod("getServerDefaults");
+ String ns = subclusterResolver.getDefaultNamespace();
+ return (FsServerDefaults) rpcClient.invokeSingle(ns, method);
+ }
+
+ @Override // ClientProtocol
+ public HdfsFileStatus create(String src, FsPermission masked,
+ String clientName, EnumSetWritable<CreateFlag> flag,
+ boolean createParent, short replication, long blockSize,
+ CryptoProtocolVersion[] supportedVersions, String ecPolicyName)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ RemoteLocation createLocation = getCreateLocation(src);
+ RemoteMethod method = new RemoteMethod("create",
+ new Class<?>[] {String.class, FsPermission.class, String.class,
+ EnumSetWritable.class, boolean.class, short.class,
+ long.class, CryptoProtocolVersion[].class,
+ String.class},
+ createLocation.getDest(), masked, clientName, flag, createParent,
+ replication, blockSize, supportedVersions, ecPolicyName);
+ return (HdfsFileStatus) rpcClient.invokeSingle(createLocation, method);
+ }
+
+ /**
+ * Get the location to create a file. It checks if the file already existed
+ * in one of the locations.
+ *
+ * @param src Path of the file to check.
+ * @return The remote location for this file.
+ * @throws IOException If the file has no creation location.
+ */
+ private RemoteLocation getCreateLocation(final String src)
+ throws IOException {
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ if (locations == null || locations.isEmpty()) {
+ throw new IOException("Cannot get locations to create " + src);
+ }
+
+ RemoteLocation createLocation = locations.get(0);
+ if (locations.size() > 1) {
+ try {
+ // Check if this file already exists in other subclusters
+ LocatedBlocks existingLocation = getBlockLocations(src, 0, 1);
+ if (existingLocation != null) {
+ // Forward to the existing location and let the NN handle the error
+ LocatedBlock existingLocationLastLocatedBlock =
+ existingLocation.getLastLocatedBlock();
+ if (existingLocationLastLocatedBlock == null) {
+ // The block has no blocks yet, check for the meta data
+ for (RemoteLocation location : locations) {
+ RemoteMethod method = new RemoteMethod("getFileInfo",
+ new Class<?>[] {String.class}, new RemoteParam());
+ if (rpcClient.invokeSingle(location, method) != null) {
+ createLocation = location;
+ break;
+ }
+ }
+ } else {
+ ExtendedBlock existingLocationLastBlock =
+ existingLocationLastLocatedBlock.getBlock();
+ String blockPoolId = existingLocationLastBlock.getBlockPoolId();
+ createLocation = getLocationForPath(src, true, blockPoolId);
+ }
+ }
+ } catch (FileNotFoundException fne) {
+ // Ignore if the file is not found
+ }
+ }
+ return createLocation;
+ }
+
+ // Medium
+ @Override // ClientProtocol
+ public LastBlockWithStatus append(String src, final String clientName,
+ final EnumSetWritable<CreateFlag> flag) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("append",
+ new Class<?>[] {String.class, String.class, EnumSetWritable.class},
+ new RemoteParam(), clientName, flag);
+ return (LastBlockWithStatus) rpcClient.invokeSequential(
+ locations, method, LastBlockWithStatus.class, null);
+ }
+
+ // Low
+ @Override // ClientProtocol
+ public boolean recoverLease(String src, String clientName)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("recoverLease",
+ new Class<?>[] {String.class, String.class}, new RemoteParam(),
+ clientName);
+ Object result = rpcClient.invokeSequential(
+ locations, method, Boolean.class, Boolean.TRUE);
+ return (boolean) result;
+ }
+
+ @Override // ClientProtocol
+ public boolean setReplication(String src, short replication)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("setReplication",
+ new Class<?>[] {String.class, short.class}, new RemoteParam(),
+ replication);
+ Object result = rpcClient.invokeSequential(
+ locations, method, Boolean.class, Boolean.TRUE);
+ return (boolean) result;
+ }
+
+ @Override
+ public void setStoragePolicy(String src, String policyName)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("setStoragePolicy",
+ new Class<?>[] {String.class, String.class},
+ new RemoteParam(), policyName);
+ rpcClient.invokeSequential(locations, method, null, null);
+ }
+
+ @Override
+ public BlockStoragePolicy[] getStoragePolicies() throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ RemoteMethod method = new RemoteMethod("getStoragePolicies");
+ String ns = subclusterResolver.getDefaultNamespace();
+ return (BlockStoragePolicy[]) rpcClient.invokeSingle(ns, method);
+ }
+
+ @Override // ClientProtocol
+ public void setPermission(String src, FsPermission permissions)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("setPermission",
+ new Class<?>[] {String.class, FsPermission.class},
+ new RemoteParam(), permissions);
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @Override // ClientProtocol
+ public void setOwner(String src, String username, String groupname)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("setOwner",
+ new Class<?>[] {String.class, String.class, String.class},
+ new RemoteParam(), username, groupname);
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ /**
+ * Excluded and favored nodes are not verified and will be ignored by
+ * placement policy if they are not in the same nameservice as the file.
+ */
+ @Override // ClientProtocol
+ public LocatedBlock addBlock(String src, String clientName,
+ ExtendedBlock previous, DatanodeInfo[] excludedNodes, long fileId,
+ String[] favoredNodes, EnumSet<AddBlockFlag> addBlockFlags)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("addBlock",
+ new Class<?>[] {String.class, String.class, ExtendedBlock.class,
+ DatanodeInfo[].class, long.class, String[].class,
+ EnumSet.class},
+ new RemoteParam(), clientName, previous, excludedNodes, fileId,
+ favoredNodes, addBlockFlags);
+ // TODO verify the excludedNodes and favoredNodes are acceptable to this NN
+ return (LocatedBlock) rpcClient.invokeSequential(
+ locations, method, LocatedBlock.class, null);
+ }
+
+ /**
+ * Excluded nodes are not verified and will be ignored by placement if they
+ * are not in the same nameservice as the file.
+ */
+ @Override // ClientProtocol
+ public LocatedBlock getAdditionalDatanode(final String src, final long fileId,
+ final ExtendedBlock blk, final DatanodeInfo[] existings,
+ final String[] existingStorageIDs, final DatanodeInfo[] excludes,
+ final int numAdditionalNodes, final String clientName)
+ throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod method = new RemoteMethod("getAdditionalDatanode",
+ new Class<?>[] {String.class, long.class, ExtendedBlock.class,
+ DatanodeInfo[].class, String[].class,
+ DatanodeInfo[].class, int.class, String.class},
+ new RemoteParam(), fileId, blk, existings, existingStorageIDs, excludes,
+ numAdditionalNodes, clientName);
+ return (LocatedBlock) rpcClient.invokeSequential(
+ locations, method, LocatedBlock.class, null);
+ }
+
+ @Override // ClientProtocol
+ public void abandonBlock(ExtendedBlock b, long fileId, String src,
+ String holder) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ RemoteMethod method = new RemoteMethod("abandonBlock",
+ new Class<?>[] {ExtendedBlock.class, long.class, String.class,
+ String.class},
+ b, fileId, new RemoteParam(), holder);
+ rpcClient.invokeSingle(b, method);
+ }
+
+ @Override // ClientProtocol
+ public boolean complete(String src, String clientName, ExtendedBlock last,
+ long fileId) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("complete",
+ new Class<?>[] {String.class, String.class, ExtendedBlock.class,
+ long.class},
+ new RemoteParam(), clientName, last, fileId);
+ // Complete can return true/false, so don't expect a result
+ return ((Boolean) rpcClient.invokeSequential(
+ locations, method, Boolean.class, null)).booleanValue();
+ }
+
+ @Override // ClientProtocol
+ public LocatedBlock updateBlockForPipeline(
+ ExtendedBlock block, String clientName) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ RemoteMethod method = new RemoteMethod("updateBlockForPipeline",
+ new Class<?>[] {ExtendedBlock.class, String.class},
+ block, clientName);
+ return (LocatedBlock) rpcClient.invokeSingle(block, method);
+ }
+
+ /**
+ * Datanode are not verified to be in the same nameservice as the old block.
+ * TODO This may require validation.
+ */
+ @Override // ClientProtocol
+ public void updatePipeline(String clientName, ExtendedBlock oldBlock,
+ ExtendedBlock newBlock, DatanodeID[] newNodes, String[] newStorageIDs)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ RemoteMethod method = new RemoteMethod("updatePipeline",
+ new Class<?>[] {String.class, ExtendedBlock.class, ExtendedBlock.class,
+ DatanodeID[].class, String[].class},
+ clientName, oldBlock, newBlock, newNodes, newStorageIDs);
+ rpcClient.invokeSingle(oldBlock, method);
+ }
+
+ @Override // ClientProtocol
+ public long getPreferredBlockSize(String src) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("getPreferredBlockSize",
+ new Class<?>[] {String.class}, new RemoteParam());
+ return ((Long) rpcClient.invokeSequential(
+ locations, method, Long.class, null)).longValue();
+ }
+
+ /**
+ * Determines combinations of eligible src/dst locations for a rename. A
+ * rename cannot change the namespace. Renames are only allowed if there is an
+ * eligible dst location in the same namespace as the source.
+ *
+ * @param srcLocations List of all potential source destinations where the
+ * path may be located. On return this list is trimmed to include
+ * only the paths that have corresponding destinations in the same
+ * namespace.
+ * @param dst The destination path
+ * @return A map of all eligible source namespaces and their corresponding
+ * replacement value.
+ * @throws IOException If the dst paths could not be determined.
+ */
+ private RemoteParam getRenameDestinations(
+ final List<RemoteLocation> srcLocations, final String dst)
+ throws IOException {
+
+ final List<RemoteLocation> dstLocations = getLocationsForPath(dst, true);
+ final Map<RemoteLocation, String> dstMap = new HashMap<>();
+
+ Iterator<RemoteLocation> iterator = srcLocations.iterator();
+ while (iterator.hasNext()) {
+ RemoteLocation srcLocation = iterator.next();
+ RemoteLocation eligibleDst =
+ getFirstMatchingLocation(srcLocation, dstLocations);
+ if (eligibleDst != null) {
+ // Use this dst for this source location
+ dstMap.put(srcLocation, eligibleDst.getDest());
+ } else {
+ // This src destination is not valid, remove from the source list
+ iterator.remove();
+ }
+ }
+ return new RemoteParam(dstMap);
+ }
+
+ /**
+ * Get first matching location.
+ *
+ * @param location Location we are looking for.
+ * @param locations List of locations.
+ * @return The first matchin location in the list.
+ */
+ private RemoteLocation getFirstMatchingLocation(RemoteLocation location,
+ List<RemoteLocation> locations) {
+ for (RemoteLocation loc : locations) {
+ if (loc.getNameserviceId().equals(location.getNameserviceId())) {
+ // Return first matching location
+ return loc;
+ }
+ }
+ return null;
+ }
+
+ @Deprecated
+ @Override // ClientProtocol
+ public boolean rename(final String src, final String dst)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> srcLocations = getLocationsForPath(src, true);
+ // srcLocations may be trimmed by getRenameDestinations()
+ final List<RemoteLocation> locs = new LinkedList<>(srcLocations);
+ RemoteParam dstParam = getRenameDestinations(locs, dst);
+ if (locs.isEmpty()) {
+ throw new IOException(
+ "Rename of " + src + " to " + dst + " is not allowed," +
+ " no eligible destination in the same namespace was found.");
+ }
+ RemoteMethod method = new RemoteMethod("rename",
+ new Class<?>[] {String.class, String.class},
+ new RemoteParam(), dstParam);
+ return ((Boolean) rpcClient.invokeSequential(
+ locs, method, Boolean.class, Boolean.TRUE)).booleanValue();
+ }
+
+ @Override // ClientProtocol
+ public void rename2(final String src, final String dst,
+ final Options.Rename... options) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> srcLocations = getLocationsForPath(src, true);
+ // srcLocations may be trimmed by getRenameDestinations()
+ final List<RemoteLocation> locs = new LinkedList<>(srcLocations);
+ RemoteParam dstParam = getRenameDestinations(locs, dst);
+ if (locs.isEmpty()) {
+ throw new IOException(
+ "Rename of " + src + " to " + dst + " is not allowed," +
+ " no eligible destination in the same namespace was found.");
+ }
+ RemoteMethod method = new RemoteMethod("rename2",
+ new Class<?>[] {String.class, String.class, options.getClass()},
+ new RemoteParam(), dstParam, options);
+ rpcClient.invokeSequential(locs, method, null, null);
+ }
+
+ @Override // ClientProtocol
+ public void concat(String trg, String[] src) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // See if the src and target files are all in the same namespace
+ LocatedBlocks targetBlocks = getBlockLocations(trg, 0, 1);
+ if (targetBlocks == null) {
+ throw new IOException("Cannot locate blocks for target file - " + trg);
+ }
+ LocatedBlock lastLocatedBlock = targetBlocks.getLastLocatedBlock();
+ String targetBlockPoolId = lastLocatedBlock.getBlock().getBlockPoolId();
+ for (String source : src) {
+ LocatedBlocks sourceBlocks = getBlockLocations(source, 0, 1);
+ if (sourceBlocks == null) {
+ throw new IOException(
+ "Cannot located blocks for source file " + source);
+ }
+ String sourceBlockPoolId =
+ sourceBlocks.getLastLocatedBlock().getBlock().getBlockPoolId();
+ if (!sourceBlockPoolId.equals(targetBlockPoolId)) {
+ throw new IOException("Cannot concatenate source file " + source
+ + " because it is located in a different namespace"
+ + " with block pool id " + sourceBlockPoolId
+ + " from the target file with block pool id "
+ + targetBlockPoolId);
+ }
+ }
+
+ // Find locations in the matching namespace.
+ final RemoteLocation targetDestination =
+ getLocationForPath(trg, true, targetBlockPoolId);
+ String[] sourceDestinations = new String[src.length];
+ for (int i = 0; i < src.length; i++) {
+ String sourceFile = src[i];
+ RemoteLocation location =
+ getLocationForPath(sourceFile, true, targetBlockPoolId);
+ sourceDestinations[i] = location.getDest();
+ }
+ // Invoke
+ RemoteMethod method = new RemoteMethod("concat",
+ new Class<?>[] {String.class, String[].class},
+ targetDestination.getDest(), sourceDestinations);
+ rpcClient.invokeSingle(targetDestination, method);
+ }
+
+ @Override // ClientProtocol
+ public boolean truncate(String src, long newLength, String clientName)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("truncate",
+ new Class<?>[] {String.class, long.class, String.class},
+ new RemoteParam(), newLength, clientName);
+ return ((Boolean) rpcClient.invokeSequential(locations, method,
+ Boolean.class, Boolean.TRUE)).booleanValue();
+ }
+
+ @Override // ClientProtocol
+ public boolean delete(String src, boolean recursive) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("delete",
+ new Class<?>[] {String.class, boolean.class}, new RemoteParam(),
+ recursive);
+ return ((Boolean) rpcClient.invokeSequential(locations, method,
+ Boolean.class, Boolean.TRUE)).booleanValue();
+ }
+
+ @Override // ClientProtocol
+ public boolean mkdirs(String src, FsPermission masked, boolean createParent)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ if (locations.size() > 1) {
+ // Check if this directory already exists
+ try {
+ HdfsFileStatus fileStatus = getFileInfo(src);
+ if (fileStatus != null) {
+ // When existing, the NN doesn't return an exception; return true
+ return true;
+ }
+ } catch (IOException ioe) {
+ // Can't query if this file exists or not.
+ LOG.error("Error requesting file info for path {} while proxing mkdirs",
+ src, ioe);
+ }
+ }
+
+ RemoteLocation firstLocation = locations.get(0);
+ RemoteMethod method = new RemoteMethod("mkdirs",
+ new Class<?>[] {String.class, FsPermission.class, boolean.class},
+ new RemoteParam(), masked, createParent);
+ return ((Boolean) rpcClient.invokeSingle(firstLocation, method))
+ .booleanValue();
+ }
+
+ @Override // ClientProtocol
+ public void renewLease(String clientName) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ RemoteMethod method = new RemoteMethod("renewLease",
+ new Class<?>[] {String.class}, clientName);
+ Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ rpcClient.invokeConcurrent(nss, method, false, false);
+ }
+
+ @Override // ClientProtocol
+ public DirectoryListing getListing(String src, byte[] startAfter,
+ boolean needLocation) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ // Locate the dir and fetch the listing
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("getListing",
+ new Class<?>[] {String.class, startAfter.getClass(), boolean.class},
+ new RemoteParam(), startAfter, needLocation);
+ Map<RemoteLocation, Object> listings =
+ rpcClient.invokeConcurrent(locations, method, false, false);
+
+ Map<String, HdfsFileStatus> nnListing = new TreeMap<>();
+ int totalRemainingEntries = 0;
+ int remainingEntries = 0;
+ boolean namenodeListingExists = false;
+ if (listings != null) {
+ // Check the subcluster listing with the smallest name
+ String lastName = null;
+ for (Entry<RemoteLocation, Object> entry : listings.entrySet()) {
+ RemoteLocation location = entry.getKey();
+ DirectoryListing listing = (DirectoryListing) entry.getValue();
+ if (listing == null) {
+ LOG.debug("Cannot get listing from {}", location);
+ } else {
+ totalRemainingEntries += listing.getRemainingEntries();
+ HdfsFileStatus[] partialListing = listing.getPartialListing();
+ int length = partialListing.length;
+ if (length > 0) {
+ HdfsFileStatus lastLocalEntry = partialListing[length-1];
+ String lastLocalName = lastLocalEntry.getLocalName();
+ if (lastName == null || lastName.compareTo(lastLocalName) > 0) {
+ lastName = lastLocalName;
+ }
+ }
+ }
+ }
+
+ // Add existing entries
+ for (Object value : listings.values()) {
+ DirectoryListing listing = (DirectoryListing) value;
+ if (listing != null) {
+ namenodeListingExists = true;
+ for (HdfsFileStatus file : listing.getPartialListing()) {
+ String filename = file.getLocalName();
+ if (totalRemainingEntries > 0 && filename.compareTo(lastName) > 0) {
+ // Discarding entries further than the lastName
+ remainingEntries++;
+ } else {
+ nnListing.put(filename, file);
+ }
+ }
+ remainingEntries += listing.getRemainingEntries();
+ }
+ }
+ }
+
+ // Add mount points at this level in the tree
+ final List<String> children = subclusterResolver.getMountPoints(src);
+ if (children != null) {
+ // Get the dates for each mount point
+ Map<String, Long> dates = getMountPointDates(src);
+
+ // Create virtual folder with the mount name
+ for (String child : children) {
+ long date = 0;
+ if (dates != null && dates.containsKey(child)) {
+ date = dates.get(child);
+ }
+ // TODO add number of children
+ HdfsFileStatus dirStatus = getMountPointStatus(child, 0, date);
+
+ // This may overwrite existing listing entries with the mount point
+ // TODO don't add if already there?
+ nnListing.put(child, dirStatus);
+ }
+ }
+
+ if (!namenodeListingExists && nnListing.size() == 0) {
+ // NN returns a null object if the directory cannot be found and has no
+ // listing. If we didn't retrieve any NN listing data, and there are no
+ // mount points here, return null.
+ return null;
+ }
+
+ // Generate combined listing
+ HdfsFileStatus[] combinedData = new HdfsFileStatus[nnListing.size()];
+ combinedData = nnListing.values().toArray(combinedData);
+ return new DirectoryListing(combinedData, remainingEntries);
+ }
+
+ @Override // ClientProtocol
+ public HdfsFileStatus getFileInfo(String src) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod method = new RemoteMethod("getFileInfo",
+ new Class<?>[] {String.class}, new RemoteParam());
+ HdfsFileStatus ret = (HdfsFileStatus) rpcClient.invokeSequential(
+ locations, method, HdfsFileStatus.class, null);
+
+ // If there is no real path, check mount points
+ if (ret == null) {
+ List<String> children = subclusterResolver.getMountPoints(src);
+ if (children != null && !children.isEmpty()) {
+ Map<String, Long> dates = getMountPointDates(src);
+ long date = 0;
+ if (dates != null && dates.containsKey(src)) {
+ date = dates.get(src);
+ }
+ ret = getMountPointStatus(src, children.size(), date);
+ }
+ }
+
+ return ret;
+ }
+
+ @Override // ClientProtocol
+ public boolean isFileClosed(String src) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod method = new RemoteMethod("isFileClosed",
+ new Class<?>[] {String.class}, new RemoteParam());
+ return ((Boolean) rpcClient.invokeSequential(
+ locations, method, Boolean.class, Boolean.TRUE)).booleanValue();
+ }
+
+ @Override // ClientProtocol
+ public HdfsFileStatus getFileLinkInfo(String src) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod method = new RemoteMethod("getFileLinkInfo",
+ new Class<?>[] {String.class}, new RemoteParam());
+ return (HdfsFileStatus) rpcClient.invokeSequential(
+ locations, method, HdfsFileStatus.class, null);
+ }
+
+ @Override // ClientProtocol
+ public long[] getStats() throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+
+ RemoteMethod method = new RemoteMethod("getStats");
+ Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, Object> results =
+ rpcClient.invokeConcurrent(nss, method, true, false);
+ long[] combinedData = new long[STATS_ARRAY_LENGTH];
+ for (Object o : results.values()) {
+ long[] data = (long[]) o;
+ for (int i = 0; i < combinedData.length && i < data.length; i++) {
+ if (data[i] >= 0) {
+ combinedData[i] += data[i];
+ }
+ }
+ }
+ return combinedData;
+ }
+
+ @Override // ClientProtocol
+ public DatanodeInfo[] getDatanodeReport(DatanodeReportType type)
+ throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+
+ Map<String, DatanodeInfo> datanodesMap = new LinkedHashMap<>();
+ RemoteMethod method = new RemoteMethod("getDatanodeReport",
+ new Class<?>[] {DatanodeReportType.class}, type);
+
+ Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, Object> results =
+ rpcClient.invokeConcurrent(nss, method, true, false);
+ for (Entry<FederationNamespaceInfo, Object> entry : results.entrySet()) {
+ FederationNamespaceInfo ns = entry.getKey();
+ DatanodeInfo[] result = (DatanodeInfo[]) entry.getValue();
+ for (DatanodeInfo node : result) {
+ String nodeId = node.getXferAddr();
+ if (!datanodesMap.containsKey(nodeId)) {
+ // Add the subcluster as a suffix to the network location
+ node.setNetworkLocation(
+ NodeBase.PATH_SEPARATOR_STR + ns.getNameserviceId() +
+ node.getNetworkLocation());
+ datanodesMap.put(nodeId, node);
+ } else {
+ LOG.debug("{} is in multiple subclusters", nodeId);
+ }
+ }
+ }
+ // Map -> Array
+ Collection<DatanodeInfo> datanodes = datanodesMap.values();
+ DatanodeInfo[] combinedData = new DatanodeInfo[datanodes.size()];
+ combinedData = datanodes.toArray(combinedData);
+ return combinedData;
+ }
+
+ @Override // ClientProtocol
+ public DatanodeStorageReport[] getDatanodeStorageReport(
+ DatanodeReportType type) throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+
+ Map<String, DatanodeStorageReport> datanodesMap = new HashMap<>();
+ RemoteMethod method = new RemoteMethod("getDatanodeStorageReport",
+ new Class<?>[] {DatanodeReportType.class}, type);
+ Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, Object> results =
+ rpcClient.invokeConcurrent(nss, method, true, false);
+ for (Object r : results.values()) {
+ DatanodeStorageReport[] result = (DatanodeStorageReport[]) r;
+ for (DatanodeStorageReport node : result) {
+ String nodeId = node.getDatanodeInfo().getXferAddr();
+ if (!datanodesMap.containsKey(nodeId)) {
+ datanodesMap.put(nodeId, node);
+ }
+ // TODO merge somehow, right now it just takes the first one
+ }
+ }
+
+ Collection<DatanodeStorageReport> datanodes = datanodesMap.values();
+ // TODO sort somehow
+ DatanodeStorageReport[] combinedData =
+ new DatanodeStorageReport[datanodes.size()];
+ combinedData = datanodes.toArray(combinedData);
+ return combinedData;
+ }
+
+ @Override // ClientProtocol
+ public boolean setSafeMode(SafeModeAction action, boolean isChecked)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // Set safe mode in all the name spaces
+ RemoteMethod method = new RemoteMethod("setSafeMode",
+ new Class<?>[] {SafeModeAction.class, boolean.class},
+ action, isChecked);
+ Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, Object> results =
+ rpcClient.invokeConcurrent(nss, method, true, true);
+
+ // We only report true if all the name space are in safe mode
+ int numSafemode = 0;
+ for (Object result : results.values()) {
+ if (result instanceof Boolean) {
+ boolean safemode = (boolean) result;
+ if (safemode) {
+ numSafemode++;
+ }
+ }
}
+ return numSafemode == results.size();
+ }
+
+ @Override // ClientProtocol
+ public boolean restoreFailedStorage(String arg) throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+
+ RemoteMethod method = new RemoteMethod("restoreFailedStorage",
+ new Class<?>[] {String.class}, arg);
+ final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, Object> ret =
+ rpcClient.invokeConcurrent(nss, method, true, false);
+
+ boolean success = true;
+ Object obj = ret;
+ @SuppressWarnings("unchecked")
+ Map<FederationNamespaceInfo, Boolean> results =
+ (Map<FederationNamespaceInfo, Boolean>)obj;
+ Collection<Boolean> sucesses = results.values();
+ for (boolean s : sucesses) {
+ if (!s) {
+ success = false;
+ }
+ }
+ return success;
+ }
+
+ @Override // ClientProtocol
+ public boolean saveNamespace(long timeWindow, long txGap) throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+
+ RemoteMethod method = new RemoteMethod("saveNamespace",
+ new Class<?>[] {Long.class, Long.class}, timeWindow, txGap);
+ final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, Object> ret =
+ rpcClient.invokeConcurrent(nss, method, true, false);
+
+ boolean success = true;
+ Object obj = ret;
+ @SuppressWarnings("unchecked")
+ Map<FederationNamespaceInfo, Boolean> results =
+ (Map<FederationNamespaceInfo, Boolean>)obj;
+ Collection<Boolean> sucesses = results.values();
+ for (boolean s : sucesses) {
+ if (!s) {
+ success = false;
+ }
+ }
+ return success;
+ }
+
+ @Override // ClientProtocol
+ public long rollEdits() throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ RemoteMethod method = new RemoteMethod("rollEdits", new Class<?>[] {});
+ final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, Object> ret =
+ rpcClient.invokeConcurrent(nss, method, true, false);
+
+ // Return the maximum txid
+ long txid = 0;
+ Object obj = ret;
+ @SuppressWarnings("unchecked")
+ Map<FederationNamespaceInfo, Long> results =
+ (Map<FederationNamespaceInfo, Long>)obj;
+ Collection<Long> txids = results.values();
+ for (long t : txids) {
+ if (t > txid) {
+ txid = t;
+ }
+ }
+ return txid;
+ }
+
+ @Override // ClientProtocol
+ public void refreshNodes() throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+
+ RemoteMethod method = new RemoteMethod("refreshNodes", new Class<?>[] {});
+ final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ rpcClient.invokeConcurrent(nss, method, true, true);
+ }
+
+ @Override // ClientProtocol
+ public void finalizeUpgrade() throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+
+ RemoteMethod method = new RemoteMethod("finalizeUpgrade",
+ new Class<?>[] {});
+ final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ rpcClient.invokeConcurrent(nss, method, true, false);
+ }
+
+ @Override // ClientProtocol
+ public RollingUpgradeInfo rollingUpgrade(RollingUpgradeAction action)
+ throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ RemoteMethod method = new RemoteMethod("rollingUpgrade",
+ new Class<?>[] {RollingUpgradeAction.class}, action);
+ final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, Object> ret =
+ rpcClient.invokeConcurrent(nss, method, true, false);
+
+ // Return the first rolling upgrade info
+ RollingUpgradeInfo info = null;
+ Object obj = ret;
+ @SuppressWarnings("unchecked")
+ Map<FederationNamespaceInfo, RollingUpgradeInfo> results =
+ (Map<FederationNamespaceInfo, RollingUpgradeInfo>)obj;
+ Collection<RollingUpgradeInfo> infos = results.values();
+ for (RollingUpgradeInfo infoNs : infos) {
+ if (info == null && infoNs != null) {
+ info = infoNs;
+ }
+ }
+ return info;
+ }
+
+ @Override // ClientProtocol
+ public void metaSave(String filename) throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+
+ RemoteMethod method = new RemoteMethod("metaSave",
+ new Class<?>[] {String.class}, filename);
+ final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ rpcClient.invokeConcurrent(nss, method, true, false);
+ }
+
+ @Override // ClientProtocol
+ public CorruptFileBlocks listCorruptFileBlocks(String path, String cookie)
+ throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ final List<RemoteLocation> locations = getLocationsForPath(path, false);
+ RemoteMethod method = new RemoteMethod("listCorruptFileBlocks",
+ new Class<?>[] {String.class, String.class},
+ new RemoteParam(), cookie);
+ return (CorruptFileBlocks) rpcClient.invokeSequential(
+ locations, method, CorruptFileBlocks.class, null);
+ }
+
+ @Override // ClientProtocol
+ public void setBalancerBandwidth(long bandwidth) throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+
+ RemoteMethod method = new RemoteMethod("setBalancerBandwidth",
+ new Class<?>[] {Long.class}, bandwidth);
+ final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ rpcClient.invokeConcurrent(nss, method, true, false);
+ }
+
+ @Override // ClientProtocol
+ public ContentSummary getContentSummary(String path) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ // Get the summaries from regular files
+ Collection<ContentSummary> summaries = new LinkedList<>();
+ FileNotFoundException notFoundException = null;
+ try {
+ final List<RemoteLocation> locations = getLocationsForPath(path, false);
+ RemoteMethod method = new RemoteMethod("getContentSummary",
+ new Class<?>[] {String.class}, new RemoteParam());
+ @SuppressWarnings("unchecked")
+ Map<String, ContentSummary> results =
+ (Map<String, ContentSummary>) ((Object)rpcClient.invokeConcurrent(
+ locations, method, false, false));
+ summaries.addAll(results.values());
+ } catch (FileNotFoundException e) {
+ notFoundException = e;
+ }
+
+ // Add mount points at this level in the tree
+ final List<String> children = subclusterResolver.getMountPoints(path);
+ if (children != null) {
+ for (String child : children) {
+ Path childPath = new Path(path, child);
+ try {
+ ContentSummary mountSummary = getContentSummary(childPath.toString());
+ if (mountSummary != null) {
+ summaries.add(mountSummary);
+ }
+ } catch (Exception e) {
+ LOG.error("Cannot get content summary for mount {}: {}",
+ childPath, e.getMessage());
+ }
+ }
+ }
+
+ // Throw original exception if no original nor mount points
+ if (summaries.isEmpty() && notFoundException != null) {
+ throw notFoundException;
+ }
+
+ return aggregateContentSummary(summaries);
+ }
+
+ /**
+ * Aggregate content summaries for each subcluster.
+ *
+ * @param results Collection of individual summaries.
+ * @return Aggregated content summary.
+ */
+ private ContentSummary aggregateContentSummary(
+ Collection<ContentSummary> summaries) {
+ if (summaries.size() == 1) {
+ return summaries.iterator().next();
+ }
+
+ long length = 0;
+ long fileCount = 0;
+ long directoryCount = 0;
+ long quota = 0;
+ long spaceConsumed = 0;
+ long spaceQuota = 0;
+
+ for (ContentSummary summary : summaries) {
+ length += summary.getLength();
+ fileCount += summary.getFileCount();
+ directoryCount += summary.getDirectoryCount();
+ quota += summary.getQuota();
+ spaceConsumed += summary.getSpaceConsumed();
+ spaceQuota += summary.getSpaceQuota();
+ }
+
+ ContentSummary ret = new ContentSummary.Builder()
+ .length(length)
+ .fileCount(fileCount)
+ .directoryCount(directoryCount)
+ .quota(quota)
+ .spaceConsumed(spaceConsumed)
+ .spaceQuota(spaceQuota)
+ .build();
+ return ret;
+ }
+
+ @Override // ClientProtocol
+ public void fsync(String src, long fileId, String clientName,
+ long lastBlockLength) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("fsync",
+ new Class<?>[] {String.class, long.class, String.class, long.class },
+ new RemoteParam(), fileId, clientName, lastBlockLength);
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @Override // ClientProtocol
+ public void setTimes(String src, long mtime, long atime) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("setTimes",
+ new Class<?>[] {String.class, long.class, long.class},
+ new RemoteParam(), mtime, atime);
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @Override // ClientProtocol
+ public void createSymlink(String target, String link, FsPermission dirPerms,
+ boolean createParent) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO Verify that the link location is in the same NS as the targets
+ final List<RemoteLocation> targetLocations =
+ getLocationsForPath(target, true);
+ final List<RemoteLocation> linkLocations =
+ getLocationsForPath(link, true);
+ RemoteLocation linkLocation = linkLocations.get(0);
+ RemoteMethod method = new RemoteMethod("createSymlink",
+ new Class<?>[] {String.class, String.class, FsPermission.class,
+ boolean.class},
+ new RemoteParam(), linkLocation.getDest(), dirPerms, createParent);
+ rpcClient.invokeSequential(targetLocations, method);
+ }
+
+ @Override // ClientProtocol
+ public String getLinkTarget(String path) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ final List<RemoteLocation> locations = getLocationsForPath(path, true);
+ RemoteMethod method = new RemoteMethod("getLinkTarget",
+ new Class<?>[] {String.class}, new RemoteParam());
+ return (String) rpcClient.invokeSequential(
+ locations, method, String.class, null);
+ }
+
+ @Override // Client Protocol
+ public void allowSnapshot(String snapshotRoot) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // Client Protocol
+ public void disallowSnapshot(String snapshot) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // ClientProtocol
+ public void renameSnapshot(String snapshotRoot, String snapshotOldName,
+ String snapshotNewName) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // Client Protocol
+ public SnapshottableDirectoryStatus[] getSnapshottableDirListing()
+ throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override // ClientProtocol
+ public SnapshotDiffReport getSnapshotDiffReport(String snapshotRoot,
+ String earlierSnapshotName, String laterSnapshotName) throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override // ClientProtocol
+ public long addCacheDirective(CacheDirectiveInfo path,
+ EnumSet<CacheFlag> flags) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ return 0;
+ }
+
+ @Override // ClientProtocol
+ public void modifyCacheDirective(CacheDirectiveInfo directive,
+ EnumSet<CacheFlag> flags) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // ClientProtocol
+ public void removeCacheDirective(long id) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // ClientProtocol
+ public BatchedEntries<CacheDirectiveEntry> listCacheDirectives(
+ long prevId, CacheDirectiveInfo filter) throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override // ClientProtocol
+ public void addCachePool(CachePoolInfo info) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // ClientProtocol
+ public void modifyCachePool(CachePoolInfo info) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // ClientProtocol
+ public void removeCachePool(String cachePoolName) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // ClientProtocol
+ public BatchedEntries<CachePoolEntry> listCachePools(String prevKey)
+ throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override // ClientProtocol
+ public void modifyAclEntries(String src, List<AclEntry> aclSpec)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("modifyAclEntries",
+ new Class<?>[] {String.class, List.class},
+ new RemoteParam(), aclSpec);
+ rpcClient.invokeSequential(locations, method, null, null);
+ }
+
+ @Override // ClienProtocol
+ public void removeAclEntries(String src, List<AclEntry> aclSpec)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("removeAclEntries",
+ new Class<?>[] {String.class, List.class},
+ new RemoteParam(), aclSpec);
+ rpcClient.invokeSequential(locations, method, null, null);
+ }
+
+ @Override // ClientProtocol
+ public void removeDefaultAcl(String src) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("removeDefaultAcl",
+ new Class<?>[] {String.class}, new RemoteParam());
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @Override // ClientProtocol
+ public void removeAcl(String src) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("removeAcl",
+ new Class<?>[] {String.class}, new RemoteParam());
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @Override // ClientProtocol
+ public void setAcl(String src, List<AclEntry> aclSpec) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod(
+ "setAcl", new Class<?>[] {String.class, List.class},
+ new RemoteParam(), aclSpec);
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @Override // ClientProtocol
+ public AclStatus getAclStatus(String src) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod method = new RemoteMethod("getAclStatus",
+ new Class<?>[] {String.class}, new RemoteParam());
+ return (AclStatus) rpcClient.invokeSequential(
+ locations, method, AclStatus.class, null);
+ }
+
+ @Override // ClientProtocol
+ public void createEncryptionZone(String src, String keyName)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("createEncryptionZone",
+ new Class<?>[] {String.class, String.class},
+ new RemoteParam(), keyName);
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @Override // ClientProtocol
+ public EncryptionZone getEZForPath(String src) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod method = new RemoteMethod("getEZForPath",
+ new Class<?>[] {String.class}, new RemoteParam());
+ return (EncryptionZone) rpcClient.invokeSequential(
+ locations, method, EncryptionZone.class, null);
+ }
+
+ @Override // ClientProtocol
+ public BatchedEntries<EncryptionZone> listEncryptionZones(long prevId)
+ throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override // ClientProtocol
+ public void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("setXAttr",
+ new Class<?>[] {String.class, XAttr.class, EnumSet.class},
+ new RemoteParam(), xAttr, flag);
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override // ClientProtocol
+ public List<XAttr> getXAttrs(String src, List<XAttr> xAttrs)
+ throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod method = new RemoteMethod("getXAttrs",
+ new Class<?>[] {String.class, List.class}, new RemoteParam(), xAttrs);
+ return (List<XAttr>) rpcClient.invokeSequential(
+ locations, method, List.class, null);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override // ClientProtocol
+ public List<XAttr> listXAttrs(String src) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod method = new RemoteMethod("listXAttrs",
+ new Class<?>[] {String.class}, new RemoteParam());
+ return (List<XAttr>) rpcClient.invokeSequential(
+ locations, method, List.class, null);
+ }
+
+ @Override // ClientProtocol
+ public void removeXAttr(String src, XAttr xAttr) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("removeXAttr",
+ new Class<?>[] {String.class, XAttr.class}, new RemoteParam(), xAttr);
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @Override // ClientProtocol
+ public void checkAccess(String path, FsAction mode) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(path, true);
+ RemoteMethod method = new RemoteMethod("checkAccess",
+ new Class<?>[] {String.class, FsAction.class},
+ new RemoteParam(), mode);
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @Override // ClientProtocol
+ public long getCurrentEditLogTxid() throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ RemoteMethod method = new RemoteMethod(
+ "getCurrentEditLogTxid", new Class<?>[] {});
+ final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, Object> ret =
+ rpcClient.invokeConcurrent(nss, method, true, false);
+
+ // Return the maximum txid
+ long txid = 0;
+ Object obj = ret;
+ @SuppressWarnings("unchecked")
+ Map<FederationNamespaceInfo, Long> results =
+ (Map<FederationNamespaceInfo, Long>)obj;
+ Collection<Long> txids = results.values();
+ for (long t : txids) {
+ if (t > txid) {
+ txid = t;
+ }
+ }
+ return txid;
+ }
+
+ @Override // ClientProtocol
+ public EventBatchList getEditsFromTxid(long txid) throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override
+ public DataEncryptionKey getDataEncryptionKey() throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override
+ public String createSnapshot(String snapshotRoot, String snapshotName)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+ return null;
+ }
+
+ @Override
+ public void deleteSnapshot(String snapshotRoot, String snapshotName)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override
+ public ErasureCodingPolicy[] getErasureCodingPolicies() throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override // ClientProtocol
+ public ErasureCodingPolicy getErasureCodingPolicy(String src)
+ throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override // ClientProtocol
+ public void setErasureCodingPolicy(String src, String ecPolicyName)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // ClientProtocol
+ public AddingECPolicyResponse[] addErasureCodingPolicies(
+ ErasureCodingPolicy[] policies) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ return null;
+ }
+
+ @Override // ClientProtocol
+ public void unsetErasureCodingPolicy(String src) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // ClientProtocol
+ public void setQuota(String path, long namespaceQuota, long storagespaceQuota,
+ StorageType type) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO assign global replicas instead of applying them to each folder
+ final List<RemoteLocation> locations = getLocationsForPath(path, true);
+ RemoteMethod method = new RemoteMethod("setQuota",
+ new Class<?>[] {String.class, Long.class, Long.class,
+ StorageType.class},
+ new RemoteParam(), namespaceQuota, storagespaceQuota, type);
+ rpcClient.invokeConcurrent(locations, method, false, false);
+ }
+
+ @Override // ClientProtocol
+ public QuotaUsage getQuotaUsage(String path) throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override
+ public void reportBadBlocks(LocatedBlock[] blocks) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // Block pool id -> blocks
+ Map<String, List<LocatedBlock>> blockLocations = new HashMap<>();
+ for (LocatedBlock block : blocks) {
+ String bpId = block.getBlock().getBlockPoolId();
+ List<LocatedBlock> bpBlocks = blockLocations.get(bpId);
+ if (bpBlocks == null) {
+ bpBlocks = new LinkedList<>();
+ blockLocations.put(bpId, bpBlocks);
+ }
+ bpBlocks.add(block);
+ }
+
+ // Invoke each block pool
+ for (Entry<String, List<LocatedBlock>> entry : blockLocations.entrySet()) {
+ String bpId = entry.getKey();
+ List<LocatedBlock> bpBlocks = entry.getValue();
+
+ LocatedBlock[] bpBlocksArray =
+ bpBlocks.toArray(new LocatedBlock[bpBlocks.size()]);
+ RemoteMethod method = new RemoteMethod("reportBadBlocks",
+ new Class<?>[] {LocatedBlock[].class},
+ new Object[] {bpBlocksArray});
+ rpcClient.invokeSingleBlockPool(bpId, method);
+ }
+ }
+
+ @Override
+ public void unsetStoragePolicy(String src) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override
+ public BlockStoragePolicy getStoragePolicy(String path) throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ /**
+ * Locate the location with the matching block pool id.
+ *
+ * @param path Path to check.
+ * @param failIfLocked Fail the request if locked (top mount point).
+ * @param blockPoolId The block pool ID of the namespace to search for.
+ * @return Prioritized list of locations in the federated cluster.
+ * @throws IOException if the location for this path cannot be determined.
+ */
+ private RemoteLocation getLocationForPath(
+ String path, boolean failIfLocked, String blockPoolId)
+ throws IOException {
+
+ final List<RemoteLocation> locations =
+ getLocationsForPath(path, failIfLocked);
+
+ String nameserviceId = null;
+ Set<FederationNamespaceInfo> namespaces =
+ this.namenodeResolver.getNamespaces();
+ for (FederationNamespaceInfo namespace : namespaces) {
+ if (namespace.getBlockPoolId().equals(blockPoolId)) {
+ nameserviceId = namespace.getNameserviceId();
+ break;
+ }
+ }
+ if (nameserviceId != null) {
+ for (RemoteLocation location : locations) {
+ if (location.getNameserviceId().equals(nameserviceId)) {
+ return location;
+ }
+ }
+ }
+ throw new IOException(
+ "Cannot locate a nameservice for block pool " + blockPoolId);
+ }
+
+ /**
+ * Get the possible locations of a path in the federated cluster.
+ *
+ * @param path Path to check.
+ * @param failIfLocked Fail the request if locked (top mount point).
+ * @return Prioritized list of locations in the federated cluster.
+ * @throws IOException If the location for this path cannot be determined.
+ */
+ private List<RemoteLocation> getLocationsForPath(
+ String path, boolean failIfLocked) throws IOException {
+ // Check the location for this path
+ final PathLocation location =
+ this.subclusterResolver.getDestinationForPath(path);
+ if (location == null) {
+ throw new IOException("Cannot find locations for " + path + " in " +
+ this.subclusterResolver);
+ }
+
+ // Log the access to a path
+ return location.getDestinations();
+ }
+
+ /**
+ * Get the modification dates for mount points.
+ *
+ * @param path Name of the path to start checking dates from.
+ * @return Map with the modification dates for all sub-entries.
+ */
+ private Map<String, Long> getMountPointDates(String path) {
+ Map<String, Long> ret = new TreeMap<>();
+ // TODO add when we have a Mount Table
+ return ret;
+ }
+
+ /**
+ * Create a new file status for a mount point.
+ *
+ * @param name Name of the mount point.
+ * @param childrenNum Number of children.
+ * @param dates Map with the dates.
+ * @return New HDFS file status representing a mount point.
+ */
+ private HdfsFileStatus getMountPointStatus(
+ String name, int childrenNum, long date) {
+ long modTime = date;
+ long accessTime = date;
+ FsPermission permission = FsPermission.getDirDefault();
+ String owner = this.superUser;
+ String group = this.superGroup;
+ try {
+ // TODO support users, it should be the user for the pointed folder
+ UserGroupInformation ugi = getRemoteUser();
+ owner = ugi.getUserName();
+ group = ugi.getPrimaryGroupName();
+ } catch (IOException e) {
+ LOG.error("Cannot get the remote user: {}", e.getMessage());
+ }
+ long inodeId = 0;
+ return new HdfsFileStatus(0, true, 0, 0, modTime, accessTime, permission,
+ owner, group, new byte[0], DFSUtil.string2Bytes(name), inodeId,
+ childrenNum, null, (byte) 0, null);
+ }
+
+ /**
+ * Get the name of the method that is calling this function.
+ *
+ * @return Name of the method calling this function.
+ */
+ private static String getMethodName() {
+ final StackTraceElement[] stack = Thread.currentThread().getStackTrace();
+ String methodName = stack[3].getMethodName();
+ return methodName;
+ }
+
+ /**
+ * Get the user that is invoking this operation.
+ *
+ * @return Remote user group information.
+ * @throws IOException If we cannot get the user information.
+ */
+ static UserGroupInformation getRemoteUser() throws IOException {
+ UserGroupInformation ugi = Server.getRemoteUser();
+ return (ugi != null) ? ugi : UserGroupInformation.getCurrentUser();
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e96a2c7f/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 3935023..f4d60c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4632,6 +4632,101 @@
</property>
<property>
+ <name>dfs.federation.router.default.nameserviceId</name>
+ <value></value>
+ <description>
+ Nameservice identifier of the default subcluster to monitor.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.rpc.enable</name>
+ <value>true</value>
+ <description>
+ If the RPC service to handle client requests in the router is enabled.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.rpc-address</name>
+ <value>0.0.0.0:8888</value>
+ <description>
+ RPC address that handles all clients requests.
+ The value of this property will take the form of router-host1:rpc-port.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.rpc-bind-host</name>
+ <value></value>
+ <description>
+ The actual address the RPC server will bind to. If this optional address is
+ set, it overrides only the hostname portion of
+ dfs.federation.router.rpc-address. This is useful for making the name node
+ listen on all interfaces by setting it to 0.0.0.0.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.handler.count</name>
+ <value>10</value>
+ <description>
+ The number of server threads for the router to handle RPC requests from
+ clients.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.handler.queue.size</name>
+ <value>100</value>
+ <description>
+ The size of the queue for the number of handlers to handle RPC client requests.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.reader.count</name>
+ <value>1</value>
+ <description>
+ The number of readers for the router to handle RPC client requests.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.reader.queue.size</name>
+ <value>100</value>
+ <description>
+ The size of the queue for the number of readers for the router to handle RPC client requests.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.connection.pool-size</name>
+ <value>1</value>
+ <description>
+ Size of the pool of connections from the router to namenodes.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.connection.clean.ms</name>
+ <value>10000</value>
+ <description>
+ Time interval, in milliseconds, to check if the connection pool should
+ remove unused connections.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.connection.pool.clean.ms</name>
+ <value>60000</value>
+ <description>
+ Time interval, in milliseconds, to check if the connection manager should
+ remove unused connection pools.
+ </description>
+ </property>
+
+ <property>
<name>dfs.federation.router.file.resolver.client.class</name>
<value>org.apache.hadoop.hdfs.server.federation.MockResolver</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e96a2c7f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
index 8674682..bbb548ca 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.hdfs.server.federation;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.BufferedReader;
import java.io.FileNotFoundException;
@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
@@ -51,17 +52,11 @@ import org.apache.hadoop.security.AccessControlException;
*/
public final class FederationTestUtils {
- public final static String NAMESERVICE1 = "ns0";
- public final static String NAMESERVICE2 = "ns1";
- public final static String NAMENODE1 = "nn0";
- public final static String NAMENODE2 = "nn1";
- public final static String NAMENODE3 = "nn2";
- public final static String NAMENODE4 = "nn3";
- public final static String ROUTER1 = "router0";
- public final static String ROUTER2 = "router1";
- public final static String ROUTER3 = "router2";
- public final static String ROUTER4 = "router3";
- public final static long BLOCK_SIZE_BYTES = 134217728;
+ public final static String[] NAMESERVICES = {"ns0", "ns1"};
+ public final static String[] NAMENODES = {"nn0", "nn1", "nn2", "nn3"};
+ public final static String[] ROUTERS =
+ {"router0", "router1", "router2", "router3"};
+
private FederationTestUtils() {
// Utility class
@@ -81,7 +76,7 @@ public final class FederationTestUtils {
triggeredException = e;
}
if (exceptionClass != null) {
- assertNotNull("No exception was triggered, expected exception - "
+ assertNotNull("No exception was triggered, expected exception"
+ exceptionClass.getName(), triggeredException);
assertEquals(exceptionClass, triggeredException.getClass());
} else {
@@ -101,48 +96,43 @@ public final class FederationTestUtils {
return report;
}
report.setHAServiceState(state);
- report.setNamespaceInfo(new NamespaceInfo(1, "tesclusterid", ns, 0,
- "testbuildvesion", "testsoftwareversion"));
+ NamespaceInfo nsInfo = new NamespaceInfo(
+ 1, "tesclusterid", ns, 0, "testbuildvesion", "testsoftwareversion");
+ report.setNamespaceInfo(nsInfo);
return report;
}
public static void waitNamenodeRegistered(ActiveNamenodeResolver resolver,
- String nameserviceId, String namenodeId,
- FederationNamenodeServiceState finalState)
+ String nsId, String nnId, FederationNamenodeServiceState finalState)
throws InterruptedException, IllegalStateException, IOException {
for (int loopCount = 0; loopCount < 20; loopCount++) {
-
if (loopCount > 0) {
Thread.sleep(1000);
}
- List<? extends FederationNamenodeContext> namenodes;
- namenodes =
- resolver.getNamenodesForNameserviceId(nameserviceId);
+ List<? extends FederationNamenodeContext> namenodes =
+ resolver.getNamenodesForNameserviceId(nsId);
for (FederationNamenodeContext namenode : namenodes) {
- if (namenodeId != null
- && !namenode.getNamenodeId().equals(namenodeId)) {
- // Keep looking
- continue;
+ // Check if this is the Namenode we are checking
+ if (namenode.getNamenodeId() == nnId ||
+ namenode.getNamenodeId().equals(nnId)) {
+ if (finalState != null && !namenode.getState().equals(finalState)) {
+ // Wrong state, wait a bit more
+ break;
+ } else {
+ // Found and verified
+ return;
+ }
}
- if (finalState != null && !namenode.getState().equals(finalState)) {
- // Wrong state, wait a bit more
- break;
- }
- // Found
- return;
}
}
- assertTrue("Failed to verify state store registration for state - "
- + finalState + " - " + " - " + nameserviceId + " - ", false);
+ fail("Failed to verify State Store registration of " + nsId + " " + nnId +
+ " for state " + finalState);
}
public static boolean verifyDate(Date d1, Date d2, long precision) {
- if (Math.abs(d1.getTime() - d2.getTime()) < precision) {
- return true;
- }
- return false;
+ return Math.abs(d1.getTime() - d2.getTime()) < precision;
}
public static boolean addDirectory(FileSystem context, String path)
@@ -165,15 +155,14 @@ public final class FederationTestUtils {
} catch (Exception e) {
return false;
}
-
return false;
}
- public static boolean checkForFileInDirectory(FileSystem context,
- String testPath, String targetFile) throws AccessControlException,
- FileNotFoundException,
- UnsupportedFileSystemException, IllegalArgumentException,
- IOException {
+ public static boolean checkForFileInDirectory(
+ FileSystem context, String testPath, String targetFile)
+ throws IOException, AccessControlException, FileNotFoundException,
+ UnsupportedFileSystemException, IllegalArgumentException {
+
FileStatus[] fileStatus = context.listStatus(new Path(testPath));
String file = null;
String verifyPath = testPath + "/" + targetFile;
@@ -194,7 +183,8 @@ public final class FederationTestUtils {
public static int countContents(FileSystem context, String testPath)
throws IOException {
- FileStatus[] fileStatus = context.listStatus(new Path(testPath));
+ Path path = new Path(testPath);
+ FileStatus[] fileStatus = context.listStatus(path);
return fileStatus.length;
}
@@ -202,7 +192,7 @@ public final class FederationTestUtils {
throws IOException {
FsPermission permissions = new FsPermission("700");
FSDataOutputStream writeStream = fs.create(new Path(path), permissions,
- true, 1000, (short) 1, BLOCK_SIZE_BYTES, null);
+ true, 1000, (short) 1, DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT, null);
for (int i = 0; i < length; i++) {
writeStream.write(i);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/40] hadoop git commit: Revert "Plan/ResourceAllocation data
structure enhancements required to support recurring reservations in
ReservationSystem."
Posted by in...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3345e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/RLESparseResourceAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/RLESparseResourceAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/RLESparseResourceAllocation.java
index 100d38c..658387b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/RLESparseResourceAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/RLESparseResourceAllocation.java
@@ -18,7 +18,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.reservation;
-import java.util.Collections;
+import java.io.IOException;
+import java.io.StringWriter;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
@@ -32,6 +33,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.Plan
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
+import com.google.gson.stream.JsonWriter;
+
/**
* This is a run length encoded sparse data structure that maintains resource
* allocations over time.
@@ -41,14 +44,12 @@ public class RLESparseResourceAllocation {
private static final int THRESHOLD = 100;
private static final Resource ZERO_RESOURCE = Resources.none();
- @SuppressWarnings("checkstyle:visibilitymodifier")
- protected NavigableMap<Long, Resource> cumulativeCapacity =
+ private NavigableMap<Long, Resource> cumulativeCapacity =
new TreeMap<Long, Resource>();
private final ReentrantReadWriteLock readWriteLock =
new ReentrantReadWriteLock();
- @SuppressWarnings("checkstyle:visibilitymodifier")
- protected final Lock readLock = readWriteLock.readLock();
+ private final Lock readLock = readWriteLock.readLock();
private final Lock writeLock = readWriteLock.writeLock();
private final ResourceCalculator resourceCalculator;
@@ -235,6 +236,34 @@ public class RLESparseResourceAllocation {
}
/**
+ * Returns the JSON string representation of the current resources allocated
+ * over time.
+ *
+ * @return the JSON string representation of the current resources allocated
+ * over time
+ */
+ public String toMemJSONString() {
+ StringWriter json = new StringWriter();
+ JsonWriter jsonWriter = new JsonWriter(json);
+ readLock.lock();
+ try {
+ jsonWriter.beginObject();
+ // jsonWriter.name("timestamp").value("resource");
+ for (Map.Entry<Long, Resource> r : cumulativeCapacity.entrySet()) {
+ jsonWriter.name(r.getKey().toString()).value(r.getValue().toString());
+ }
+ jsonWriter.endObject();
+ jsonWriter.close();
+ return json.toString();
+ } catch (IOException e) {
+ // This should not happen
+ return "";
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ /**
* Returns the representation of the current resources allocated over time as
* an interval map (in the defined non-null range).
*
@@ -275,7 +304,7 @@ public class RLESparseResourceAllocation {
public NavigableMap<Long, Resource> getCumulative() {
readLock.lock();
try {
- return Collections.unmodifiableNavigableMap(cumulativeCapacity);
+ return cumulativeCapacity;
} finally {
readLock.unlock();
}
@@ -408,8 +437,8 @@ public class RLESparseResourceAllocation {
Resource val = Resources.negate(e.getValue());
// test for negative value and throws
if (operator == RLEOperator.subtractTestNonNegative
- && (Resources.fitsIn(val, ZERO_RESOURCE)
- && !Resources.equals(val, ZERO_RESOURCE))) {
+ && (Resources.fitsIn(val, ZERO_RESOURCE) &&
+ !Resources.equals(val, ZERO_RESOURCE))) {
throw new PlanningException(
"RLESparseResourceAllocation: merge failed as the "
+ "resulting RLESparseResourceAllocation would be negative");
@@ -475,29 +504,22 @@ public class RLESparseResourceAllocation {
}
- /**
- * Get a {@link RLESparseResourceAllocation} view of the {@link Resource}
- * allocations between the specified start and end times.
- *
- * @param start the time from which the {@link Resource} allocations are
- * required
- * @param end the time upto which the {@link Resource} allocations are
- * required
- * @return the overlapping allocations
- */
public RLESparseResourceAllocation getRangeOverlapping(long start, long end) {
readLock.lock();
try {
NavigableMap<Long, Resource> a = this.getCumulative();
+
if (a != null && !a.isEmpty()) {
// include the portion of previous entry that overlaps start
if (start > a.firstKey()) {
long previous = a.floorKey(start);
a = a.tailMap(previous, true);
}
+
if (end < a.lastKey()) {
a = a.headMap(end, true);
}
+
}
RLESparseResourceAllocation ret =
new RLESparseResourceAllocation(a, resourceCalculator);
@@ -505,33 +527,7 @@ public class RLESparseResourceAllocation {
} finally {
readLock.unlock();
}
- }
- /**
- * This method shifts all the timestamp of the {@link Resource} entries by the
- * specified "delta".
- *
- * @param delta the time by which to shift the {@link Resource} allocations
- */
- public void shift(long delta) {
- writeLock.lock();
- try {
- TreeMap<Long, Resource> newCum = new TreeMap<>();
- long start;
- for (Map.Entry<Long, Resource> entry : cumulativeCapacity.entrySet()) {
- if (delta > 0) {
- start = (entry.getKey() == Long.MAX_VALUE) ? Long.MAX_VALUE
- : entry.getKey() + delta;
- } else {
- start = (entry.getKey() == Long.MIN_VALUE) ? Long.MIN_VALUE
- : entry.getKey() + delta;
- }
- newCum.put(start, entry.getValue());
- }
- cumulativeCapacity = newCum;
- } finally {
- writeLock.unlock();
- }
}
/**
@@ -545,8 +541,8 @@ public class RLESparseResourceAllocation {
/**
* Get the maximum capacity across specified time instances. The search-space
* is specified using the starting value, tick, and the periodic interval for
- * search. Maximum resource allocation across tick, tick + period, tick + 2 *
- * period,..., tick + n * period .. is returned.
+ * search. Maximum resource allocation across tick, tick + period,
+ * tick + 2 * period,..., tick + n * period .. is returned.
*
* @param tick the starting time instance
* @param period interval at which capacity is evaluated
@@ -554,19 +550,14 @@ public class RLESparseResourceAllocation {
*/
public Resource getMaximumPeriodicCapacity(long tick, long period) {
Resource maxCapacity = ZERO_RESOURCE;
- readLock.lock();
- try {
- if (!cumulativeCapacity.isEmpty()) {
- Long lastKey = cumulativeCapacity.lastKey();
- for (long t = tick; t <= lastKey; t = t + period) {
- maxCapacity = Resources.componentwiseMax(maxCapacity,
- cumulativeCapacity.floorEntry(t).getValue());
- }
+ if (!cumulativeCapacity.isEmpty()) {
+ Long lastKey = cumulativeCapacity.lastKey();
+ for (long t = tick; t <= lastKey; t = t + period) {
+ maxCapacity = Resources.componentwiseMax(maxCapacity,
+ cumulativeCapacity.floorEntry(t).getValue());
}
- return maxCapacity;
- } finally {
- readLock.unlock();
}
+ return maxCapacity;
}
/**
@@ -576,17 +567,17 @@ public class RLESparseResourceAllocation {
* @return minimum resource allocation
*/
public Resource getMinimumCapacityInInterval(ReservationInterval interval) {
- Resource minCapacity =
- Resource.newInstance(Integer.MAX_VALUE, Integer.MAX_VALUE);
+ Resource minCapacity = Resource.newInstance(
+ Integer.MAX_VALUE, Integer.MAX_VALUE);
long start = interval.getStartTime();
long end = interval.getEndTime();
NavigableMap<Long, Resource> capacityRange =
- getRangeOverlapping(start, end).getCumulative();
+ this.getRangeOverlapping(start, end).getCumulative();
if (!capacityRange.isEmpty()) {
for (Map.Entry<Long, Resource> entry : capacityRange.entrySet()) {
if (entry.getValue() != null) {
- minCapacity =
- Resources.componentwiseMin(minCapacity, entry.getValue());
+ minCapacity = Resources.componentwiseMin(minCapacity,
+ entry.getValue());
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3345e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationAllocation.java
index bb4a7fb..0da95ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationAllocation.java
@@ -24,16 +24,14 @@ import org.apache.hadoop.yarn.api.records.ReservationDefinition;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
-import com.google.common.annotations.VisibleForTesting;
-
/**
* A ReservationAllocation represents a concrete allocation of resources over
* time that satisfy a certain {@link ReservationDefinition}. This is used
* internally by a {@link Plan} to store information about how each of the
* accepted {@link ReservationDefinition} have been allocated.
*/
-public interface ReservationAllocation
- extends Comparable<ReservationAllocation> {
+public interface ReservationAllocation extends
+ Comparable<ReservationAllocation> {
/**
* Returns the unique identifier {@link ReservationId} that represents the
@@ -42,28 +40,28 @@ public interface ReservationAllocation
* @return reservationId the unique identifier {@link ReservationId} that
* represents the reservation
*/
- ReservationId getReservationId();
+ public ReservationId getReservationId();
/**
* Returns the original {@link ReservationDefinition} submitted by the client
*
* @return the {@link ReservationDefinition} submitted by the client
*/
- ReservationDefinition getReservationDefinition();
+ public ReservationDefinition getReservationDefinition();
/**
* Returns the time at which the reservation is activated.
*
* @return the time at which the reservation is activated
*/
- long getStartTime();
+ public long getStartTime();
/**
* Returns the time at which the reservation terminates.
*
* @return the time at which the reservation terminates
*/
- long getEndTime();
+ public long getEndTime();
/**
* Returns the map of resources requested against the time interval for which
@@ -72,28 +70,28 @@ public interface ReservationAllocation
* @return the allocationRequests the map of resources requested against the
* time interval for which they were
*/
- Map<ReservationInterval, Resource> getAllocationRequests();
+ public Map<ReservationInterval, Resource> getAllocationRequests();
/**
* Return a string identifying the plan to which the reservation belongs
*
* @return the plan to which the reservation belongs
*/
- String getPlanName();
+ public String getPlanName();
/**
* Returns the user who requested the reservation
*
* @return the user who requested the reservation
*/
- String getUser();
+ public String getUser();
/**
* Returns whether the reservation has gang semantics or not
*
* @return true if there is a gang request, false otherwise
*/
- boolean containsGangs();
+ public boolean containsGangs();
/**
* Sets the time at which the reservation was accepted by the system
@@ -101,14 +99,14 @@ public interface ReservationAllocation
* @param acceptedAt the time at which the reservation was accepted by the
* system
*/
- void setAcceptanceTimestamp(long acceptedAt);
+ public void setAcceptanceTimestamp(long acceptedAt);
/**
* Returns the time at which the reservation was accepted by the system
*
* @return the time at which the reservation was accepted by the system
*/
- long getAcceptanceTime();
+ public long getAcceptanceTime();
/**
* Returns the capacity represented by cumulative resources reserved by the
@@ -118,42 +116,12 @@ public interface ReservationAllocation
* requested
* @return the resources reserved at the specified time
*/
- Resource getResourcesAtTime(long tick);
-
- /**
- * Return a RLE representation of used resources.
- *
- * @return a RLE encoding of resources allocated over time.
- */
- RLESparseResourceAllocation getResourcesOverTime();
-
+ public Resource getResourcesAtTime(long tick);
/**
* Return a RLE representation of used resources.
- *
- * @param start start of the time interval.
- * @param end end of the time interval.
* @return a RLE encoding of resources allocated over time.
*/
- RLESparseResourceAllocation getResourcesOverTime(long start, long end);
-
- /**
- * Get the periodicity of this reservation representing the time period of the
- * periodic job. Period is represented in milliseconds for periodic jobs.
- * Period is 0 for non-periodic jobs.
- *
- * @return periodicity of this reservation
- */
- long getPeriodicity();
-
- /**
- * Set the periodicity of this reservation representing the time period of the
- * periodic job. Period is represented in milliseconds for periodic jobs.
- * Period is 0 for non-periodic jobs.
- *
- * @param period periodicity of this reservation
- */
- @VisibleForTesting
- void setPeriodicity(long period);
+ public RLESparseResourceAllocation getResourcesOverTime();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3345e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java
index a66d222..027d066 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java
@@ -44,8 +44,6 @@ public class ReservationInputValidator {
/**
* Utility class to validate reservation requests.
- *
- * @param clock the {@link Clock} to use
*/
public ReservationInputValidator(Clock clock) {
this.clock = clock;
@@ -55,21 +53,22 @@ public class ReservationInputValidator {
ReservationId reservationId, String auditConstant) throws YarnException {
// check if the reservation id is valid
if (reservationId == null) {
- String message = "Missing reservation id."
- + " Please try again by specifying a reservation id.";
+ String message =
+ "Missing reservation id."
+ + " Please try again by specifying a reservation id.";
RMAuditLogger.logFailure("UNKNOWN", auditConstant,
"validate reservation input", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
}
String queue = reservationSystem.getQueueForReservation(reservationId);
String nullQueueErrorMessage =
- "The specified reservation with ID: " + reservationId
- + " is unknown. Please try again with a valid reservation.";
+ "The specified reservation with ID: " + reservationId
+ + " is unknown. Please try again with a valid reservation.";
String nullPlanErrorMessage = "The specified reservation: " + reservationId
- + " is not associated with any valid plan."
- + " Please try again with a valid reservation.";
+ + " is not associated with any valid plan."
+ + " Please try again with a valid reservation.";
return getPlanFromQueue(reservationSystem, queue, auditConstant,
- nullQueueErrorMessage, nullPlanErrorMessage);
+ nullQueueErrorMessage, nullPlanErrorMessage);
}
private void validateReservationDefinition(ReservationId reservationId,
@@ -78,15 +77,17 @@ public class ReservationInputValidator {
String message = "";
// check if deadline is in the past
if (contract == null) {
- message = "Missing reservation definition."
- + " Please try again by specifying a reservation definition.";
+ message =
+ "Missing reservation definition."
+ + " Please try again by specifying a reservation definition.";
RMAuditLogger.logFailure("UNKNOWN", auditConstant,
"validate reservation input definition", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
}
if (contract.getDeadline() <= clock.getTime()) {
- message = "The specified deadline: " + contract.getDeadline()
- + " is the past. Please try again with deadline in the future.";
+ message =
+ "The specified deadline: " + contract.getDeadline()
+ + " is the past. Please try again with deadline in the future.";
RMAuditLogger.logFailure("UNKNOWN", auditConstant,
"validate reservation input definition", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
@@ -94,16 +95,18 @@ public class ReservationInputValidator {
// Check if at least one RR has been specified
ReservationRequests resReqs = contract.getReservationRequests();
if (resReqs == null) {
- message = "No resources have been specified to reserve."
- + "Please try again by specifying the resources to reserve.";
+ message =
+ "No resources have been specified to reserve."
+ + "Please try again by specifying the resources to reserve.";
RMAuditLogger.logFailure("UNKNOWN", auditConstant,
"validate reservation input definition", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
}
List<ReservationRequest> resReq = resReqs.getReservationResources();
if (resReq == null || resReq.isEmpty()) {
- message = "No resources have been specified to reserve."
- + " Please try again by specifying the resources to reserve.";
+ message =
+ "No resources have been specified to reserve."
+ + " Please try again by specifying the resources to reserve.";
RMAuditLogger.logFailure("UNKNOWN", auditConstant,
"validate reservation input definition", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
@@ -120,18 +123,22 @@ public class ReservationInputValidator {
} else {
minDuration += rr.getDuration();
}
- maxGangSize = Resources.max(plan.getResourceCalculator(),
- plan.getTotalCapacity(), maxGangSize,
- Resources.multiply(rr.getCapability(), rr.getConcurrency()));
+ maxGangSize =
+ Resources.max(plan.getResourceCalculator(), plan.getTotalCapacity(),
+ maxGangSize,
+ Resources.multiply(rr.getCapability(), rr.getConcurrency()));
}
// verify the allocation is possible (skip for ANY)
long duration = contract.getDeadline() - contract.getArrival();
- if (duration < minDuration && type != ReservationRequestInterpreter.R_ANY) {
- message = "The time difference (" + (duration) + ") between arrival ("
- + contract.getArrival() + ") " + "and deadline ("
- + contract.getDeadline() + ") must "
- + " be greater or equal to the minimum resource duration ("
- + minDuration + ")";
+ if (duration < minDuration
+ && type != ReservationRequestInterpreter.R_ANY) {
+ message =
+ "The time difference ("
+ + (duration)
+ + ") between arrival (" + contract.getArrival() + ") "
+ + "and deadline (" + contract.getDeadline() + ") must "
+ + " be greater or equal to the minimum resource duration ("
+ + minDuration + ")";
RMAuditLogger.logFailure("UNKNOWN", auditConstant,
"validate reservation input definition", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
@@ -141,9 +148,10 @@ public class ReservationInputValidator {
if (Resources.greaterThan(plan.getResourceCalculator(),
plan.getTotalCapacity(), maxGangSize, plan.getTotalCapacity())
&& type != ReservationRequestInterpreter.R_ANY) {
- message = "The size of the largest gang in the reservation definition ("
- + maxGangSize + ") exceed the capacity available ("
- + plan.getTotalCapacity() + " )";
+ message =
+ "The size of the largest gang in the reservation definition ("
+ + maxGangSize + ") exceed the capacity available ("
+ + plan.getTotalCapacity() + " )";
RMAuditLogger.logFailure("UNKNOWN", auditConstant,
"validate reservation input definition", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
@@ -171,32 +179,32 @@ public class ReservationInputValidator {
}
}
- private Plan getPlanFromQueue(ReservationSystem reservationSystem,
- String queue, String auditConstant) throws YarnException {
+ private Plan getPlanFromQueue(ReservationSystem reservationSystem, String
+ queue, String auditConstant) throws YarnException {
String nullQueueErrorMessage = "The queue is not specified."
- + " Please try again with a valid reservable queue.";
+ + " Please try again with a valid reservable queue.";
String nullPlanErrorMessage = "The specified queue: " + queue
- + " is not managed by reservation system."
- + " Please try again with a valid reservable queue.";
+ + " is not managed by reservation system."
+ + " Please try again with a valid reservable queue.";
return getPlanFromQueue(reservationSystem, queue, auditConstant,
- nullQueueErrorMessage, nullPlanErrorMessage);
+ nullQueueErrorMessage, nullPlanErrorMessage);
}
- private Plan getPlanFromQueue(ReservationSystem reservationSystem,
- String queue, String auditConstant, String nullQueueErrorMessage,
- String nullPlanErrorMessage) throws YarnException {
+ private Plan getPlanFromQueue(ReservationSystem reservationSystem, String
+ queue, String auditConstant, String nullQueueErrorMessage,
+ String nullPlanErrorMessage) throws YarnException {
if (queue == null || queue.isEmpty()) {
RMAuditLogger.logFailure("UNKNOWN", auditConstant,
- "validate reservation input", "ClientRMService",
- nullQueueErrorMessage);
+ "validate reservation input", "ClientRMService",
+ nullQueueErrorMessage);
throw RPCUtil.getRemoteException(nullQueueErrorMessage);
}
// check if the associated plan is valid
Plan plan = reservationSystem.getPlan(queue);
if (plan == null) {
RMAuditLogger.logFailure("UNKNOWN", auditConstant,
- "validate reservation input", "ClientRMService",
- nullPlanErrorMessage);
+ "validate reservation input", "ClientRMService",
+ nullPlanErrorMessage);
throw RPCUtil.getRemoteException(nullPlanErrorMessage);
}
return plan;
@@ -214,21 +222,22 @@ public class ReservationInputValidator {
* @param reservationId the {@link ReservationId} associated with the current
* request
* @return the {@link Plan} to submit the request to
- * @throws YarnException if validation fails
+ * @throws YarnException
*/
public Plan validateReservationSubmissionRequest(
- ReservationSystem reservationSystem, ReservationSubmissionRequest request,
- ReservationId reservationId) throws YarnException {
+ ReservationSystem reservationSystem,
+ ReservationSubmissionRequest request, ReservationId reservationId)
+ throws YarnException {
String message;
if (reservationId == null) {
- message = "Reservation id cannot be null. Please try again specifying "
- + " a valid reservation id by creating a new reservation id.";
+ message = "Reservation id cannot be null. Please try again " +
+ "specifying a valid reservation id by creating a new reservation id.";
throw RPCUtil.getRemoteException(message);
}
// Check if it is a managed queue
String queue = request.getQueue();
Plan plan = getPlanFromQueue(reservationSystem, queue,
- AuditConstants.SUBMIT_RESERVATION_REQUEST);
+ AuditConstants.SUBMIT_RESERVATION_REQUEST);
validateReservationDefinition(reservationId,
request.getReservationDefinition(), plan,
@@ -246,14 +255,15 @@ public class ReservationInputValidator {
* @param request the {@link ReservationUpdateRequest} defining the resources
* required over time for the request
* @return the {@link Plan} to submit the request to
- * @throws YarnException if validation fails
+ * @throws YarnException
*/
public Plan validateReservationUpdateRequest(
ReservationSystem reservationSystem, ReservationUpdateRequest request)
throws YarnException {
ReservationId reservationId = request.getReservationId();
- Plan plan = validateReservation(reservationSystem, reservationId,
- AuditConstants.UPDATE_RESERVATION_REQUEST);
+ Plan plan =
+ validateReservation(reservationSystem, reservationId,
+ AuditConstants.UPDATE_RESERVATION_REQUEST);
validateReservationDefinition(reservationId,
request.getReservationDefinition(), plan,
AuditConstants.UPDATE_RESERVATION_REQUEST);
@@ -268,26 +278,28 @@ public class ReservationInputValidator {
*
* @param reservationSystem the {@link ReservationSystem} to validate against
* @param request the {@link ReservationListRequest} defining search
- * parameters for reservations in the {@link ReservationSystem} that
- * is being validated against.
+ * parameters for reservations in the {@link ReservationSystem}
+ * that is being validated against.
* @return the {@link Plan} to list reservations of.
- * @throws YarnException if validation fails
+ * @throws YarnException
*/
public Plan validateReservationListRequest(
- ReservationSystem reservationSystem, ReservationListRequest request)
+ ReservationSystem reservationSystem,
+ ReservationListRequest request)
throws YarnException {
String queue = request.getQueue();
if (request.getEndTime() < request.getStartTime()) {
- String errorMessage = "The specified end time must be greater than "
- + "the specified start time.";
+ String errorMessage = "The specified end time must be greater than " +
+ "the specified start time.";
RMAuditLogger.logFailure("UNKNOWN",
- AuditConstants.LIST_RESERVATION_REQUEST,
- "validate list reservation input", "ClientRMService", errorMessage);
+ AuditConstants.LIST_RESERVATION_REQUEST,
+ "validate list reservation input", "ClientRMService",
+ errorMessage);
throw RPCUtil.getRemoteException(errorMessage);
}
// Check if it is a managed queue
return getPlanFromQueue(reservationSystem, queue,
- AuditConstants.LIST_RESERVATION_REQUEST);
+ AuditConstants.LIST_RESERVATION_REQUEST);
}
/**
@@ -300,7 +312,7 @@ public class ReservationInputValidator {
* @param request the {@link ReservationDeleteRequest} defining the resources
* required over time for the request
* @return the {@link Plan} to submit the request to
- * @throws YarnException if validation fails
+ * @throws YarnException
*/
public Plan validateReservationDeleteRequest(
ReservationSystem reservationSystem, ReservationDeleteRequest request)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3345e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystem.java
index a6c8fcf..8b62972 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystem.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.reservation;
-import java.util.Map;
-
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
@@ -31,6 +29,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.ReservationsACLsManager;
+import java.util.Map;
+
/**
* This interface is the one implemented by any system that wants to support
* Reservations i.e. make {@code Resource} allocations in future. Implementors
@@ -57,7 +57,7 @@ public interface ReservationSystem extends Recoverable {
*
* @param conf configuration
* @param rmContext current context of the {@code ResourceManager}
- * @throws YarnException if initialization of the configured plan fails
+ * @throws YarnException
*/
void reinitialize(Configuration conf, RMContext rmContext)
throws YarnException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3345e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/SharingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/SharingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/SharingPolicy.java
index cbf0f38..e458055 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/SharingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/SharingPolicy.java
@@ -38,7 +38,7 @@ public interface SharingPolicy {
* @param planQueuePath the name of the queue for this plan
* @param conf the system configuration
*/
- void init(String planQueuePath, ReservationSchedulerConfiguration conf);
+ public void init(String planQueuePath, ReservationSchedulerConfiguration conf);
/**
* This method runs the policy validation logic, and return true/false on
@@ -51,7 +51,7 @@ public interface SharingPolicy {
* @throws PlanningException if the policy is respected if we add this
* {@link ReservationAllocation} to the {@link Plan}
*/
- void validate(Plan plan, ReservationAllocation newAllocation)
+ public void validate(Plan plan, ReservationAllocation newAllocation)
throws PlanningException;
/**
@@ -68,13 +68,9 @@ public interface SharingPolicy {
* @param start the start time for the range we are querying
* @param end the end time for the range we are querying
* @param oldId (optional) the id of a reservation being updated
- *
- * @return the available resources expressed as a
- * {@link RLESparseResourceAllocation}
- *
* @throws PlanningException throws if the request is not valid
*/
- RLESparseResourceAllocation availableResources(
+ public RLESparseResourceAllocation availableResources(
RLESparseResourceAllocation available, Plan plan, String user,
ReservationId oldId, long start, long end) throws PlanningException;
@@ -86,6 +82,7 @@ public interface SharingPolicy {
*
* @return validWindow the window of validity considered by the policy.
*/
- long getValidWindow();
+ public long getValidWindow();
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3345e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/Planner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/Planner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/Planner.java
index af0e712..abac6ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/Planner.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/Planner.java
@@ -34,7 +34,7 @@ public interface Planner {
*
* @param plan the {@link Plan} to replan
* @param contracts the list of reservation requests
- * @throws PlanningException if operation is unsuccessful
+ * @throws PlanningException
*/
public void plan(Plan plan, List<ReservationDefinition> contracts)
throws PlanningException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3345e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
index bbbf0d6..199bfa5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
@@ -50,7 +50,7 @@ public abstract class PlanningAlgorithm implements ReservationAgent {
* @return whether the allocateUser function was successful or not
*
* @throws PlanningException if the session cannot be fitted into the plan
- * @throws ContractValidationException if validation fails
+ * @throws ContractValidationException
*/
protected boolean allocateUser(ReservationId reservationId, String user,
Plan plan, ReservationDefinition contract,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3345e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocator.java
index 8934b0f..ec6d9c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocator.java
@@ -50,7 +50,7 @@ public interface StageAllocator {
*
* @return The computed allocation (or null if the stage could not be
* allocated)
- * @throws PlanningException if operation is unsuccessful
+ * @throws PlanningException
*/
Map<ReservationInterval, Resource> computeStageAllocation(Plan plan,
RLESparseResourceAllocation planLoads,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3345e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedy.java
index d107487..da04336 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedy.java
@@ -69,7 +69,7 @@ public class StageAllocatorGreedy implements StageAllocator {
RLESparseResourceAllocation netAvailable =
plan.getAvailableResourceOverTime(user, oldId, stageEarliestStart,
- stageDeadline, 0);
+ stageDeadline);
netAvailable =
RLESparseResourceAllocation.merge(plan.getResourceCalculator(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3345e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedyRLE.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedyRLE.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedyRLE.java
index ae7d91a..ec83e02 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedyRLE.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedyRLE.java
@@ -83,8 +83,9 @@ public class StageAllocatorGreedyRLE implements StageAllocator {
int gangsToPlace = rr.getNumContainers() / rr.getConcurrency();
// get available resources from plan
- RLESparseResourceAllocation netRLERes = plan.getAvailableResourceOverTime(
- user, oldId, stageEarliestStart, stageDeadline, 0);
+ RLESparseResourceAllocation netRLERes =
+ plan.getAvailableResourceOverTime(user, oldId, stageEarliestStart,
+ stageDeadline);
// remove plan modifications
netRLERes =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3345e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorLowCostAligned.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorLowCostAligned.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorLowCostAligned.java
index c014549..e45f58c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorLowCostAligned.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorLowCostAligned.java
@@ -77,8 +77,8 @@ public class StageAllocatorLowCostAligned implements StageAllocator {
ResourceCalculator resCalc = plan.getResourceCalculator();
Resource capacity = plan.getTotalCapacity();
- RLESparseResourceAllocation netRLERes = plan.getAvailableResourceOverTime(
- user, oldId, stageArrival, stageDeadline, 0);
+ RLESparseResourceAllocation netRLERes = plan
+ .getAvailableResourceOverTime(user, oldId, stageArrival, stageDeadline);
long step = plan.getStep();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3345e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java
index 5337e06..e99842e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java
@@ -19,10 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.reservation;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anySetOf;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.*;
import java.io.FileWriter;
import java.io.IOException;
@@ -79,8 +76,7 @@ public class ReservationSystemTestUtil {
String reservationQ, long timeWindow, float instConstraint,
float avgConstraint) {
- ReservationSchedulerConfiguration realConf =
- new CapacitySchedulerConfiguration();
+ ReservationSchedulerConfiguration realConf = new CapacitySchedulerConfiguration();
ReservationSchedulerConfiguration conf = spy(realConf);
when(conf.getReservationWindow(reservationQ)).thenReturn(timeWindow);
when(conf.getInstantaneousMaxCapacity(reservationQ))
@@ -172,6 +168,7 @@ public class ReservationSystemTestUtil {
scheduler.start();
scheduler.reinitialize(conf, rmContext);
+
Resource resource =
ReservationSystemTestUtil.calculateClusterResource(numContainers);
RMNode node1 = MockNodes.newNodeInfo(1, resource, 1, "127.0.0.1");
@@ -187,16 +184,10 @@ public class ReservationSystemTestUtil {
public static ReservationDefinition createSimpleReservationDefinition(
long arrival, long deadline, long duration, int parallelism) {
- return createSimpleReservationDefinition(arrival, deadline, duration,
- parallelism, null);
- }
-
- public static ReservationDefinition createSimpleReservationDefinition(
- long arrival, long deadline, long duration, int parallelism,
- String recurrenceExpression) {
// create a request with a single atomic ask
- ReservationRequest r = ReservationRequest.newInstance(
- Resource.newInstance(1024, 1), parallelism, parallelism, duration);
+ ReservationRequest r =
+ ReservationRequest.newInstance(Resource.newInstance(1024, 1),
+ parallelism, parallelism, duration);
ReservationDefinition rDef = new ReservationDefinitionPBImpl();
ReservationRequests reqs = new ReservationRequestsPBImpl();
reqs.setReservationResources(Collections.singletonList(r));
@@ -204,31 +195,32 @@ public class ReservationSystemTestUtil {
rDef.setReservationRequests(reqs);
rDef.setArrival(arrival);
rDef.setDeadline(deadline);
- if (recurrenceExpression != null) {
- rDef.setRecurrenceExpression(recurrenceExpression);
- }
return rDef;
}
public static ReservationSubmissionRequest createSimpleReservationRequest(
ReservationId reservationId, int numContainers, long arrival,
long deadline, long duration) {
- return createSimpleReservationRequest(reservationId, numContainers, arrival,
- deadline, duration, Priority.UNDEFINED);
+ return createSimpleReservationRequest(reservationId, numContainers,
+ arrival, deadline, duration, Priority.UNDEFINED);
}
public static ReservationSubmissionRequest createSimpleReservationRequest(
ReservationId reservationId, int numContainers, long arrival,
long deadline, long duration, Priority priority) {
// create a request with a single atomic ask
- ReservationRequest r = ReservationRequest
- .newInstance(Resource.newInstance(1024, 1), numContainers, 1, duration);
- ReservationRequests reqs = ReservationRequests.newInstance(
- Collections.singletonList(r), ReservationRequestInterpreter.R_ALL);
- ReservationDefinition rDef = ReservationDefinition.newInstance(arrival,
- deadline, reqs, "testClientRMService#reservation", "0", priority);
- ReservationSubmissionRequest request = ReservationSubmissionRequest
- .newInstance(rDef, reservationQ, reservationId);
+ ReservationRequest r =
+ ReservationRequest.newInstance(Resource.newInstance(1024, 1),
+ numContainers, 1, duration);
+ ReservationRequests reqs =
+ ReservationRequests.newInstance(Collections.singletonList(r),
+ ReservationRequestInterpreter.R_ALL);
+ ReservationDefinition rDef =
+ ReservationDefinition.newInstance(arrival, deadline, reqs,
+ "testClientRMService#reservation", "0", priority);
+ ReservationSubmissionRequest request =
+ ReservationSubmissionRequest.newInstance(rDef,
+ reservationQ, reservationId);
return request;
}
@@ -260,9 +252,9 @@ public class ReservationSystemTestUtil {
return cs;
}
- @SuppressWarnings("rawtypes")
- public static void initializeRMContext(int numContainers,
- AbstractYarnScheduler scheduler, RMContext mockRMContext) {
+ @SuppressWarnings("rawtypes") public static void initializeRMContext(
+ int numContainers, AbstractYarnScheduler scheduler,
+ RMContext mockRMContext) {
when(mockRMContext.getScheduler()).thenReturn(scheduler);
Resource r = calculateClusterResource(numContainers);
@@ -270,25 +262,26 @@ public class ReservationSystemTestUtil {
}
public static RMContext createRMContext(Configuration conf) {
- RMContext mockRmContext = Mockito.spy(new RMContextImpl(null, null, null,
- null, null, null, new RMContainerTokenSecretManager(conf),
- new NMTokenSecretManagerInRM(conf),
- new ClientToAMTokenSecretManagerInRM(), null));
+ RMContext mockRmContext = Mockito.spy(
+ new RMContextImpl(null, null, null, null, null, null,
+ new RMContainerTokenSecretManager(conf),
+ new NMTokenSecretManagerInRM(conf),
+ new ClientToAMTokenSecretManagerInRM(), null));
RMNodeLabelsManager nlm = mock(RMNodeLabelsManager.class);
when(nlm.getQueueResource(any(String.class), anySetOf(String.class),
- any(Resource.class))).thenAnswer(new Answer<Resource>() {
- @Override
- public Resource answer(InvocationOnMock invocation) throws Throwable {
- Object[] args = invocation.getArguments();
- return (Resource) args[2];
- }
- });
+ any(Resource.class))).thenAnswer(new Answer<Resource>() {
+ @Override public Resource answer(InvocationOnMock invocation)
+ throws Throwable {
+ Object[] args = invocation.getArguments();
+ return (Resource) args[2];
+ }
+ });
when(nlm.getResourceByLabel(any(String.class), any(Resource.class)))
.thenAnswer(new Answer<Resource>() {
- @Override
- public Resource answer(InvocationOnMock invocation) throws Throwable {
+ @Override public Resource answer(InvocationOnMock invocation)
+ throws Throwable {
Object[] args = invocation.getArguments();
return (Resource) args[1];
}
@@ -311,8 +304,9 @@ public class ReservationSystemTestUtil {
final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setCapacity(A, 10);
- final String dedicated = CapacitySchedulerConfiguration.ROOT
- + CapacitySchedulerConfiguration.DOT + reservationQ;
+ final String dedicated =
+ CapacitySchedulerConfiguration.ROOT + CapacitySchedulerConfiguration.DOT
+ + reservationQ;
conf.setCapacity(dedicated, 80);
// Set as reservation queue
conf.setReservable(dedicated, true);
@@ -411,55 +405,26 @@ public class ReservationSystemTestUtil {
public static Map<ReservationInterval, Resource> generateAllocation(
long startTime, long step, int[] alloc) {
- return generateAllocation(startTime, step, alloc, null);
- }
-
- public static Map<ReservationInterval, Resource> generateAllocation(
- long startTime, long step, int[] alloc, String recurrenceExpression) {
Map<ReservationInterval, Resource> req = new TreeMap<>();
-
- long period = 0;
- if (recurrenceExpression != null) {
- period = Long.parseLong(recurrenceExpression);
- }
-
- long rStart;
- long rEnd;
- for (int j = 0; j < 86400000; j += period) {
- for (int i = 0; i < alloc.length; i++) {
- rStart = (startTime + i * step) + j * period;
- rEnd = (startTime + (i + 1) * step) + j * period;
- if (period > 0) {
- rStart = rStart % period + j * period;
- rEnd = rEnd % period + j * period;
- if (rStart > rEnd) {
- // skip wrap-around entry
- continue;
- }
- }
-
- req.put(new ReservationInterval(rStart, rEnd),
- ReservationSystemUtil.toResource(ReservationRequest
- .newInstance(Resource.newInstance(1024, 1), alloc[i])));
-
- }
- // execute only once if non-periodic
- if (period == 0) {
- break;
- }
+ for (int i = 0; i < alloc.length; i++) {
+ req.put(new ReservationInterval(startTime + i * step,
+ startTime + (i + 1) * step), ReservationSystemUtil.toResource(
+ ReservationRequest
+ .newInstance(Resource.newInstance(1024, 1), alloc[i])));
}
return req;
}
- public static RLESparseResourceAllocation generateRLESparseResourceAllocation(
- int[] alloc, long[] timeSteps) {
+ public static RLESparseResourceAllocation
+ generateRLESparseResourceAllocation(int[] alloc, long[] timeSteps) {
TreeMap<Long, Resource> allocationsMap = new TreeMap<>();
for (int i = 0; i < alloc.length; i++) {
allocationsMap.put(timeSteps[i],
Resource.newInstance(alloc[i], alloc[i]));
}
- RLESparseResourceAllocation rleVector = new RLESparseResourceAllocation(
- allocationsMap, new DefaultResourceCalculator());
+ RLESparseResourceAllocation rleVector =
+ new RLESparseResourceAllocation(allocationsMap,
+ new DefaultResourceCalculator());
return rleVector;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org