You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2018/03/20 05:18:27 UTC
[01/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Repository: hadoop
Updated Branches:
refs/heads/trunk e65ff1c8b -> 6e2b5fa49
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMountTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMountTable.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMountTable.java
deleted file mode 100644
index d5fb9ba..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMountTable.java
+++ /dev/null
@@ -1,259 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.records;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
-import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
-import org.apache.hadoop.hdfs.server.federation.router.RouterQuotaUsage;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.Test;
-
-/**
- * Test the Mount Table entry in the State Store.
- */
-public class TestMountTable {
-
- private static final String SRC = "/test";
- private static final String DST_NS_0 = "ns0";
- private static final String DST_NS_1 = "ns1";
- private static final String DST_PATH_0 = "/path1";
- private static final String DST_PATH_1 = "/path/path2";
- private static final List<RemoteLocation> DST = new LinkedList<>();
- static {
- DST.add(new RemoteLocation(DST_NS_0, DST_PATH_0));
- DST.add(new RemoteLocation(DST_NS_1, DST_PATH_1));
- }
- private static final Map<String, String> DST_MAP = new LinkedHashMap<>();
- static {
- DST_MAP.put(DST_NS_0, DST_PATH_0);
- DST_MAP.put(DST_NS_1, DST_PATH_1);
- }
-
- private static final long DATE_CREATED = 100;
- private static final long DATE_MOD = 200;
-
- private static final long NS_COUNT = 1;
- private static final long NS_QUOTA = 5;
- private static final long SS_COUNT = 10;
- private static final long SS_QUOTA = 100;
-
- private static final RouterQuotaUsage QUOTA = new RouterQuotaUsage.Builder()
- .fileAndDirectoryCount(NS_COUNT).quota(NS_QUOTA).spaceConsumed(SS_COUNT)
- .spaceQuota(SS_QUOTA).build();
-
- @Test
- public void testGetterSetter() throws IOException {
-
- MountTable record = MountTable.newInstance(SRC, DST_MAP);
-
- validateDestinations(record);
- assertEquals(SRC, record.getSourcePath());
- assertEquals(DST, record.getDestinations());
- assertTrue(DATE_CREATED > 0);
- assertTrue(DATE_MOD > 0);
-
- RouterQuotaUsage quota = record.getQuota();
- assertEquals(0, quota.getFileAndDirectoryCount());
- assertEquals(HdfsConstants.QUOTA_DONT_SET, quota.getQuota());
- assertEquals(0, quota.getSpaceConsumed());
- assertEquals(HdfsConstants.QUOTA_DONT_SET, quota.getSpaceQuota());
-
- MountTable record2 =
- MountTable.newInstance(SRC, DST_MAP, DATE_CREATED, DATE_MOD);
-
- validateDestinations(record2);
- assertEquals(SRC, record2.getSourcePath());
- assertEquals(DST, record2.getDestinations());
- assertEquals(DATE_CREATED, record2.getDateCreated());
- assertEquals(DATE_MOD, record2.getDateModified());
- assertFalse(record.isReadOnly());
- assertEquals(DestinationOrder.HASH, record.getDestOrder());
- }
-
- @Test
- public void testSerialization() throws IOException {
- testSerialization(DestinationOrder.RANDOM);
- testSerialization(DestinationOrder.HASH);
- testSerialization(DestinationOrder.LOCAL);
- }
-
- private void testSerialization(final DestinationOrder order)
- throws IOException {
-
- MountTable record = MountTable.newInstance(
- SRC, DST_MAP, DATE_CREATED, DATE_MOD);
- record.setReadOnly(true);
- record.setDestOrder(order);
- record.setQuota(QUOTA);
-
- StateStoreSerializer serializer = StateStoreSerializer.getSerializer();
- String serializedString = serializer.serializeString(record);
- MountTable record2 =
- serializer.deserialize(serializedString, MountTable.class);
-
- validateDestinations(record2);
- assertEquals(SRC, record2.getSourcePath());
- assertEquals(DST, record2.getDestinations());
- assertEquals(DATE_CREATED, record2.getDateCreated());
- assertEquals(DATE_MOD, record2.getDateModified());
- assertTrue(record2.isReadOnly());
- assertEquals(order, record2.getDestOrder());
-
- RouterQuotaUsage quotaGet = record2.getQuota();
- assertEquals(NS_COUNT, quotaGet.getFileAndDirectoryCount());
- assertEquals(NS_QUOTA, quotaGet.getQuota());
- assertEquals(SS_COUNT, quotaGet.getSpaceConsumed());
- assertEquals(SS_QUOTA, quotaGet.getSpaceQuota());
- }
-
- @Test
- public void testReadOnly() throws IOException {
-
- Map<String, String> dest = new LinkedHashMap<>();
- dest.put(DST_NS_0, DST_PATH_0);
- dest.put(DST_NS_1, DST_PATH_1);
- MountTable record1 = MountTable.newInstance(SRC, dest);
- record1.setReadOnly(true);
-
- validateDestinations(record1);
- assertEquals(SRC, record1.getSourcePath());
- assertEquals(DST, record1.getDestinations());
- assertTrue(DATE_CREATED > 0);
- assertTrue(DATE_MOD > 0);
- assertTrue(record1.isReadOnly());
-
- MountTable record2 = MountTable.newInstance(
- SRC, DST_MAP, DATE_CREATED, DATE_MOD);
- record2.setReadOnly(true);
-
- validateDestinations(record2);
- assertEquals(SRC, record2.getSourcePath());
- assertEquals(DST, record2.getDestinations());
- assertEquals(DATE_CREATED, record2.getDateCreated());
- assertEquals(DATE_MOD, record2.getDateModified());
- assertTrue(record2.isReadOnly());
- }
-
- @Test
- public void testOrder() throws IOException {
- testOrder(DestinationOrder.HASH);
- testOrder(DestinationOrder.LOCAL);
- testOrder(DestinationOrder.RANDOM);
- }
-
- private void testOrder(final DestinationOrder order)
- throws IOException {
-
- MountTable record = MountTable.newInstance(
- SRC, DST_MAP, DATE_CREATED, DATE_MOD);
- record.setDestOrder(order);
-
- validateDestinations(record);
- assertEquals(SRC, record.getSourcePath());
- assertEquals(DST, record.getDestinations());
- assertEquals(DATE_CREATED, record.getDateCreated());
- assertEquals(DATE_MOD, record.getDateModified());
- assertEquals(order, record.getDestOrder());
- }
-
- private void validateDestinations(MountTable record) {
-
- assertEquals(SRC, record.getSourcePath());
- assertEquals(2, record.getDestinations().size());
-
- RemoteLocation location1 = record.getDestinations().get(0);
- assertEquals(DST_NS_0, location1.getNameserviceId());
- assertEquals(DST_PATH_0, location1.getDest());
-
- RemoteLocation location2 = record.getDestinations().get(1);
- assertEquals(DST_NS_1, location2.getNameserviceId());
- assertEquals(DST_PATH_1, location2.getDest());
- }
-
- @Test
- public void testQuota() throws IOException {
- MountTable record = MountTable.newInstance(SRC, DST_MAP);
- record.setQuota(QUOTA);
-
- validateDestinations(record);
- assertEquals(SRC, record.getSourcePath());
- assertEquals(DST, record.getDestinations());
- assertTrue(DATE_CREATED > 0);
- assertTrue(DATE_MOD > 0);
-
- RouterQuotaUsage quotaGet = record.getQuota();
- assertEquals(NS_COUNT, quotaGet.getFileAndDirectoryCount());
- assertEquals(NS_QUOTA, quotaGet.getQuota());
- assertEquals(SS_COUNT, quotaGet.getSpaceConsumed());
- assertEquals(SS_QUOTA, quotaGet.getSpaceQuota());
- }
-
- @Test
- public void testValidation() throws IOException {
- Map<String, String> destinations = new HashMap<>();
- destinations.put("ns0", "/testValidate-dest");
- try {
- MountTable.newInstance("testValidate", destinations);
- fail("Mount table entry should be created failed.");
- } catch (Exception e) {
- GenericTestUtils.assertExceptionContains(
- MountTable.ERROR_MSG_MUST_START_WITH_BACK_SLASH, e);
- }
-
- destinations.clear();
- destinations.put("ns0", "testValidate-dest");
- try {
- MountTable.newInstance("/testValidate", destinations);
- fail("Mount table entry should be created failed.");
- } catch (Exception e) {
- GenericTestUtils.assertExceptionContains(
- MountTable.ERROR_MSG_ALL_DEST_MUST_START_WITH_BACK_SLASH, e);
- }
-
- destinations.clear();
- destinations.put("", "/testValidate-dest");
- try {
- MountTable.newInstance("/testValidate", destinations);
- fail("Mount table entry should be created failed.");
- } catch (Exception e) {
- GenericTestUtils.assertExceptionContains(
- MountTable.ERROR_MSG_INVAILD_DEST_NS, e);
- }
-
- destinations.clear();
- destinations.put("ns0", "/testValidate-dest");
- MountTable record = MountTable.newInstance("/testValidate", destinations);
- assertNotNull(record);
-
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestRouterState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestRouterState.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestRouterState.java
deleted file mode 100644
index dfe2bc9..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestRouterState.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.records;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.server.federation.router.RouterServiceState;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-import org.junit.Test;
-
-/**
- * Test the Router State records.
- */
-public class TestRouterState {
-
- private static final String ADDRESS = "address";
- private static final String VERSION = "version";
- private static final String COMPILE_INFO = "compileInfo";
- private static final long START_TIME = 100;
- private static final long DATE_MODIFIED = 200;
- private static final long DATE_CREATED = 300;
- private static final long FILE_RESOLVER_VERSION = 500;
- private static final RouterServiceState STATE = RouterServiceState.RUNNING;
-
-
- private RouterState generateRecord() throws IOException {
- RouterState record = RouterState.newInstance(ADDRESS, START_TIME, STATE);
- record.setVersion(VERSION);
- record.setCompileInfo(COMPILE_INFO);
- record.setDateCreated(DATE_CREATED);
- record.setDateModified(DATE_MODIFIED);
-
- StateStoreVersion version = StateStoreVersion.newInstance();
- version.setMountTableVersion(FILE_RESOLVER_VERSION);
- record.setStateStoreVersion(version);
- return record;
- }
-
- private void validateRecord(RouterState record) throws IOException {
- assertEquals(ADDRESS, record.getAddress());
- assertEquals(START_TIME, record.getDateStarted());
- assertEquals(STATE, record.getStatus());
- assertEquals(COMPILE_INFO, record.getCompileInfo());
- assertEquals(VERSION, record.getVersion());
-
- StateStoreVersion version = record.getStateStoreVersion();
- assertEquals(FILE_RESOLVER_VERSION, version.getMountTableVersion());
- }
-
- @Test
- public void testGetterSetter() throws IOException {
- RouterState record = generateRecord();
- validateRecord(record);
- }
-
- @Test
- public void testSerialization() throws IOException {
-
- RouterState record = generateRecord();
-
- StateStoreSerializer serializer = StateStoreSerializer.getSerializer();
- String serializedString = serializer.serializeString(record);
- RouterState newRecord =
- serializer.deserialize(serializedString, RouterState.class);
-
- validateRecord(newRecord);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/pom.xml b/hadoop-hdfs-project/pom.xml
index 33930fa..0aa1112 100644
--- a/hadoop-hdfs-project/pom.xml
+++ b/hadoop-hdfs-project/pom.xml
@@ -35,6 +35,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<module>hadoop-hdfs-native-client</module>
<module>hadoop-hdfs-httpfs</module>
<module>hadoop-hdfs-nfs</module>
+ <module>hadoop-hdfs-rbf</module>
</modules>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index ef0d1cd..1ab41d2 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -252,6 +252,11 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-rbf</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-project/src/site/site.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index 38d4138..b5ecd73 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -105,7 +105,7 @@
<item name="Disk Balancer" href="hadoop-project-dist/hadoop-hdfs/HDFSDiskbalancer.html"/>
<item name="Upgrade Domain" href="hadoop-project-dist/hadoop-hdfs/HdfsUpgradeDomain.html"/>
<item name="DataNode Admin" href="hadoop-project-dist/hadoop-hdfs/HdfsDataNodeAdminGuide.html"/>
- <item name="Router Federation" href="hadoop-project-dist/hadoop-hdfs/HDFSRouterFederation.html"/>
+ <item name="Router Federation" href="hadoop-project-dist/hadoop-hdfs-rbf/HDFSRouterFederation.html"/>
</menu>
<menu name="MapReduce" inherit="top">
@@ -203,6 +203,7 @@
<menu name="Configuration" inherit="top">
<item name="core-default.xml" href="hadoop-project-dist/hadoop-common/core-default.xml"/>
<item name="hdfs-default.xml" href="hadoop-project-dist/hadoop-hdfs/hdfs-default.xml"/>
+ <item name="hdfs-rbf-default.xml" href="hadoop-project-dist/hadoop-hdfs-rbf/hdfs-rbf-default.xml"/>
<item name="mapred-default.xml" href="hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml"/>
<item name="yarn-default.xml" href="hadoop-yarn/hadoop-yarn-common/yarn-default.xml"/>
<item name="Deprecated Properties" href="hadoop-project-dist/hadoop-common/DeprecatedProperties.html"/>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java
deleted file mode 100644
index 9667489..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java
+++ /dev/null
@@ -1,317 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.records.impl.pb;
-
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProto.Builder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProto.DestOrder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProtoOrBuilder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoteLocationProto;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.QuotaUsageProto;
-import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
-import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
-import org.apache.hadoop.hdfs.server.federation.router.RouterAdminServer;
-import org.apache.hadoop.hdfs.server.federation.router.RouterPermissionChecker;
-import org.apache.hadoop.hdfs.server.federation.router.RouterQuotaUsage;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.FederationProtocolPBTranslator;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the MountTable record.
- */
-public class MountTablePBImpl extends MountTable implements PBRecord {
-
- private FederationProtocolPBTranslator<MountTableRecordProto, Builder,
- MountTableRecordProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<MountTableRecordProto, Builder,
- MountTableRecordProtoOrBuilder>(MountTableRecordProto.class);
-
- public MountTablePBImpl() {
- }
-
- public MountTablePBImpl(MountTableRecordProto proto) {
- this.setProto(proto);
- }
-
- @Override
- public MountTableRecordProto getProto() {
- return this.translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public String getSourcePath() {
- MountTableRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
- if (!proto.hasSrcPath()) {
- return null;
- }
- return proto.getSrcPath();
- }
-
- @Override
- public void setSourcePath(String path) {
- Builder builder = this.translator.getBuilder();
- if (path == null) {
- builder.clearSrcPath();
- } else {
- builder.setSrcPath(path);
- }
- }
-
- @Override
- public List<RemoteLocation> getDestinations() {
- MountTableRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
- if (proto.getDestinationsCount() == 0) {
- return null;
- }
-
- final List<RemoteLocation> ret = new LinkedList<>();
- final List<RemoteLocationProto> destList = proto.getDestinationsList();
- for (RemoteLocationProto dest : destList) {
- String nsId = dest.getNameserviceId();
- String path = dest.getPath();
- RemoteLocation loc = new RemoteLocation(nsId, path);
- ret.add(loc);
- }
- return ret;
- }
-
- @Override
- public void setDestinations(final List<RemoteLocation> dests) {
- Builder builder = this.translator.getBuilder();
- builder.clearDestinations();
- for (RemoteLocation dest : dests) {
- RemoteLocationProto.Builder itemBuilder =
- RemoteLocationProto.newBuilder();
- String nsId = dest.getNameserviceId();
- String path = dest.getDest();
- itemBuilder.setNameserviceId(nsId);
- itemBuilder.setPath(path);
- RemoteLocationProto item = itemBuilder.build();
- builder.addDestinations(item);
- }
- }
-
- @Override
- public boolean addDestination(String nsId, String path) {
- // Check if the location is already there
- List<RemoteLocation> dests = getDestinations();
- for (RemoteLocation dest : dests) {
- if (dest.getNameserviceId().equals(nsId) && dest.getDest().equals(path)) {
- return false;
- }
- }
-
- // Add it to the existing list
- Builder builder = this.translator.getBuilder();
- RemoteLocationProto.Builder itemBuilder =
- RemoteLocationProto.newBuilder();
- itemBuilder.setNameserviceId(nsId);
- itemBuilder.setPath(path);
- RemoteLocationProto item = itemBuilder.build();
- builder.addDestinations(item);
- return true;
- }
-
- @Override
- public void setDateModified(long time) {
- this.translator.getBuilder().setDateModified(time);
- }
-
- @Override
- public long getDateModified() {
- return this.translator.getProtoOrBuilder().getDateModified();
- }
-
- @Override
- public void setDateCreated(long time) {
- this.translator.getBuilder().setDateCreated(time);
- }
-
- @Override
- public long getDateCreated() {
- return this.translator.getProtoOrBuilder().getDateCreated();
- }
-
- @Override
- public boolean isReadOnly() {
- MountTableRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
- if (!proto.hasReadOnly()) {
- return false;
- }
- return proto.getReadOnly();
- }
-
- @Override
- public void setReadOnly(boolean ro) {
- this.translator.getBuilder().setReadOnly(ro);
- }
-
- @Override
- public DestinationOrder getDestOrder() {
- MountTableRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
- return convert(proto.getDestOrder());
- }
-
- @Override
- public void setDestOrder(DestinationOrder order) {
- Builder builder = this.translator.getBuilder();
- if (order == null) {
- builder.clearDestOrder();
- } else {
- builder.setDestOrder(convert(order));
- }
- }
-
- @Override
- public String getOwnerName() {
- MountTableRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
- if (!proto.hasOwnerName()) {
- return RouterAdminServer.getSuperUser();
- }
- return proto.getOwnerName();
- }
-
- @Override
- public void setOwnerName(String owner) {
- Builder builder = this.translator.getBuilder();
- if (owner == null) {
- builder.clearOwnerName();
- } else {
- builder.setOwnerName(owner);
- }
- }
-
- @Override
- public String getGroupName() {
- MountTableRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
- if (!proto.hasGroupName()) {
- return RouterAdminServer.getSuperGroup();
- }
- return proto.getGroupName();
- }
-
- @Override
- public void setGroupName(String group) {
- Builder builder = this.translator.getBuilder();
- if (group == null) {
- builder.clearGroupName();
- } else {
- builder.setGroupName(group);
- }
- }
-
- @Override
- public FsPermission getMode() {
- MountTableRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
- short mode = RouterPermissionChecker.MOUNT_TABLE_PERMISSION_DEFAULT;
- if (proto.hasMode()) {
- mode = (short) proto.getMode();
- }
- return new FsPermission(mode);
- }
-
- @Override
- public void setMode(FsPermission mode) {
- Builder builder = this.translator.getBuilder();
- if (mode == null) {
- builder.clearMode();
- } else {
- builder.setMode(mode.toShort());
- }
- }
-
- @Override
- public RouterQuotaUsage getQuota() {
- MountTableRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
-
- long nsQuota = HdfsConstants.QUOTA_DONT_SET;
- long nsCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
- long ssQuota = HdfsConstants.QUOTA_DONT_SET;
- long ssCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
- if (proto.hasQuota()) {
- QuotaUsageProto quotaProto = proto.getQuota();
- nsQuota = quotaProto.getQuota();
- nsCount = quotaProto.getFileAndDirectoryCount();
- ssQuota = quotaProto.getSpaceQuota();
- ssCount = quotaProto.getSpaceConsumed();
- }
-
- RouterQuotaUsage.Builder builder = new RouterQuotaUsage.Builder()
- .quota(nsQuota).fileAndDirectoryCount(nsCount).spaceQuota(ssQuota)
- .spaceConsumed(ssCount);
- return builder.build();
- }
-
- @Override
- public void setQuota(RouterQuotaUsage quota) {
- Builder builder = this.translator.getBuilder();
- if (quota == null) {
- builder.clearQuota();
- } else {
- QuotaUsageProto quotaUsage = QuotaUsageProto.newBuilder()
- .setFileAndDirectoryCount(quota.getFileAndDirectoryCount())
- .setQuota(quota.getQuota()).setSpaceConsumed(quota.getSpaceConsumed())
- .setSpaceQuota(quota.getSpaceQuota()).build();
- builder.setQuota(quotaUsage);
- }
- }
-
- private DestinationOrder convert(DestOrder order) {
- switch (order) {
- case LOCAL:
- return DestinationOrder.LOCAL;
- case RANDOM:
- return DestinationOrder.RANDOM;
- case HASH_ALL:
- return DestinationOrder.HASH_ALL;
- default:
- return DestinationOrder.HASH;
- }
- }
-
- private DestOrder convert(DestinationOrder order) {
- switch (order) {
- case LOCAL:
- return DestOrder.LOCAL;
- case RANDOM:
- return DestOrder.RANDOM;
- case HASH_ALL:
- return DestOrder.HASH_ALL;
- default:
- return DestOrder.HASH;
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/PBRecord.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/PBRecord.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/PBRecord.java
deleted file mode 100644
index c369275..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/PBRecord.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.records.impl.pb;
-
-import java.io.IOException;
-
-import com.google.protobuf.Message;
-
-/**
- * A record implementation using Protobuf.
- */
-public interface PBRecord {
-
- /**
- * Get the protocol for the record.
- * @return The protocol for this record.
- */
- Message getProto();
-
- /**
- * Set the protocol for the record.
- * @param proto Protocol for this record.
- */
- void setProto(Message proto);
-
- /**
- * Populate this record with serialized data.
- * @param base64String Serialized data in base64.
- * @throws IOException If it cannot read the data.
- */
- void readInstance(String base64String) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/RouterStatePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/RouterStatePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/RouterStatePBImpl.java
deleted file mode 100644
index 23a61f9..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/RouterStatePBImpl.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.records.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterRecordProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterRecordProto.Builder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterRecordProtoOrBuilder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.StateStoreVersionRecordProto;
-import org.apache.hadoop.hdfs.server.federation.router.RouterServiceState;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.FederationProtocolPBTranslator;
-import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
-import org.apache.hadoop.hdfs.server.federation.store.records.StateStoreVersion;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the RouterState record.
- */
-public class RouterStatePBImpl extends RouterState implements PBRecord {
-
- private FederationProtocolPBTranslator<RouterRecordProto, Builder,
- RouterRecordProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<RouterRecordProto, Builder,
- RouterRecordProtoOrBuilder>(RouterRecordProto.class);
-
- public RouterStatePBImpl() {
- }
-
- public RouterStatePBImpl(RouterRecordProto proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public RouterRecordProto getProto() {
- return this.translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public void setAddress(String address) {
- RouterRecordProto.Builder builder = this.translator.getBuilder();
- if (address == null) {
- builder.clearAddress();
- } else {
- builder.setAddress(address);
- }
- }
-
- @Override
- public String getAddress() {
- RouterRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
- if (!proto.hasAddress()) {
- return null;
- }
- return proto.getAddress();
- }
-
- @Override
- public void setStateStoreVersion(StateStoreVersion version) {
- RouterRecordProto.Builder builder = this.translator.getBuilder();
- if (version instanceof StateStoreVersionPBImpl) {
- StateStoreVersionPBImpl versionPB = (StateStoreVersionPBImpl)version;
- StateStoreVersionRecordProto versionProto =
- (StateStoreVersionRecordProto)versionPB.getProto();
- builder.setStateStoreVersion(versionProto);
- } else {
- builder.clearStateStoreVersion();
- }
- }
-
- @Override
- public StateStoreVersion getStateStoreVersion() throws IOException {
- RouterRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
- if (!proto.hasStateStoreVersion()) {
- return null;
- }
- StateStoreVersionRecordProto versionProto = proto.getStateStoreVersion();
- StateStoreVersion version =
- StateStoreSerializer.newRecord(StateStoreVersion.class);
- if (version instanceof StateStoreVersionPBImpl) {
- StateStoreVersionPBImpl versionPB = (StateStoreVersionPBImpl)version;
- versionPB.setProto(versionProto);
- return versionPB;
- } else {
- throw new IOException("Cannot get State Store version");
- }
- }
-
- @Override
- public RouterServiceState getStatus() {
- RouterRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
- if (!proto.hasStatus()) {
- return null;
- }
- return RouterServiceState.valueOf(proto.getStatus());
- }
-
- @Override
- public void setStatus(RouterServiceState newStatus) {
- RouterRecordProto.Builder builder = this.translator.getBuilder();
- if (newStatus == null) {
- builder.clearStatus();
- } else {
- builder.setStatus(newStatus.toString());
- }
- }
-
- @Override
- public String getVersion() {
- RouterRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
- if (!proto.hasVersion()) {
- return null;
- }
- return proto.getVersion();
- }
-
- @Override
- public void setVersion(String version) {
- RouterRecordProto.Builder builder = this.translator.getBuilder();
- if (version == null) {
- builder.clearVersion();
- } else {
- builder.setVersion(version);
- }
- }
-
- @Override
- public String getCompileInfo() {
- RouterRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
- if (!proto.hasCompileInfo()) {
- return null;
- }
- return proto.getCompileInfo();
- }
-
- @Override
- public void setCompileInfo(String info) {
- RouterRecordProto.Builder builder = this.translator.getBuilder();
- if (info == null) {
- builder.clearCompileInfo();
- } else {
- builder.setCompileInfo(info);
- }
- }
-
- @Override
- public void setDateStarted(long dateStarted) {
- this.translator.getBuilder().setDateStarted(dateStarted);
- }
-
- @Override
- public long getDateStarted() {
- return this.translator.getProtoOrBuilder().getDateStarted();
- }
-
- @Override
- public void setDateModified(long time) {
- this.translator.getBuilder().setDateModified(time);
- }
-
- @Override
- public long getDateModified() {
- return this.translator.getProtoOrBuilder().getDateModified();
- }
-
- @Override
- public void setDateCreated(long time) {
- this.translator.getBuilder().setDateCreated(time);
- }
-
- @Override
- public long getDateCreated() {
- return this.translator.getProtoOrBuilder().getDateCreated();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/StateStoreVersionPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/StateStoreVersionPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/StateStoreVersionPBImpl.java
deleted file mode 100644
index 7696136..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/StateStoreVersionPBImpl.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.records.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.StateStoreVersionRecordProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.StateStoreVersionRecordProto.Builder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.StateStoreVersionRecordProtoOrBuilder;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.FederationProtocolPBTranslator;
-import org.apache.hadoop.hdfs.server.federation.store.records.StateStoreVersion;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the StateStoreVersion record.
- */
-public class StateStoreVersionPBImpl extends StateStoreVersion
- implements PBRecord {
-
- private FederationProtocolPBTranslator<StateStoreVersionRecordProto, Builder,
- StateStoreVersionRecordProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<StateStoreVersionRecordProto,
- Builder, StateStoreVersionRecordProtoOrBuilder>(
- StateStoreVersionRecordProto.class);
-
- public StateStoreVersionPBImpl() {
- }
-
- @Override
- public StateStoreVersionRecordProto getProto() {
- return this.translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public long getMembershipVersion() {
- return this.translator.getProtoOrBuilder().getMembershipVersion();
- }
-
- @Override
- public void setMembershipVersion(long version) {
- this.translator.getBuilder().setMembershipVersion(version);
- }
-
- @Override
- public long getMountTableVersion() {
- return this.translator.getProtoOrBuilder().getMountTableVersion();
- }
-
- @Override
- public void setMountTableVersion(long version) {
- this.translator.getBuilder().setMountTableVersion(version);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/package-info.java
deleted file mode 100644
index b329732..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/package-info.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * The protobuf implementations of state store data records defined in the
- * org.apache.hadoop.hdfs.server.federation.store.records package. Each
- * implementation wraps an associated protobuf proto definition.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-package org.apache.hadoop.hdfs.server.federation.store.records.impl.pb;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/package-info.java
deleted file mode 100644
index 63b13af..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/package-info.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Contains the abstract definitions of the state store data records. The state
- * store supports multiple multiple data records.
- * <p>
- * Data records inherit from a common class
- * {@link org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord
- * BaseRecord}. Data records are serialized when written to the data store using
- * a modular serialization implementation. The default is profobuf
- * serialization. Data is stored as rows of records of the same type with each
- * data member in a record representing a column.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-
-package org.apache.hadoop.hdfs.server.federation.store.records;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/utils/ConsistentHashRing.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/utils/ConsistentHashRing.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/utils/ConsistentHashRing.java
deleted file mode 100644
index 89273db..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/utils/ConsistentHashRing.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.utils;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import org.apache.hadoop.io.MD5Hash;
-
-/**
- * Consistent hash ring to distribute items across nodes (locations). If we add
- * or remove nodes, it minimizes the item migration.
- */
-public class ConsistentHashRing {
- private static final String SEPERATOR = "/";
- private static final String VIRTUAL_NODE_FORMAT = "%s" + SEPERATOR + "%d";
-
- /** Hash ring. */
- private SortedMap<String, String> ring = new TreeMap<String, String>();
- /** Entry -> num virtual nodes on ring. */
- private Map<String, Integer> entryToVirtualNodes =
- new HashMap<String, Integer>();
-
- /** Synchronization. */
- private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
- private final Lock readLock = readWriteLock.readLock();
- private final Lock writeLock = readWriteLock.writeLock();
-
- public ConsistentHashRing(Set<String> locations) {
- for (String location : locations) {
- addLocation(location);
- }
- }
-
- /**
- * Add entry to consistent hash ring.
- *
- * @param location Node to add to the ring.
- */
- public void addLocation(String location) {
- addLocation(location, 100);
- }
-
- /**
- * Add entry to consistent hash ring.
- *
- * @param location Node to add to the ring.
- * @param numVirtualNodes Number of virtual nodes to add.
- */
- public void addLocation(String location, int numVirtualNodes) {
- writeLock.lock();
- try {
- entryToVirtualNodes.put(location, numVirtualNodes);
- for (int i = 0; i < numVirtualNodes; i++) {
- String key = String.format(VIRTUAL_NODE_FORMAT, location, i);
- String hash = getHash(key);
- ring.put(hash, key);
- }
- } finally {
- writeLock.unlock();
- }
- }
-
- /**
- * Remove specified entry from hash ring.
- *
- * @param location Node to remove from the ring.
- */
- public void removeLocation(String location) {
- writeLock.lock();
- try {
- Integer numVirtualNodes = entryToVirtualNodes.remove(location);
- for (int i = 0; i < numVirtualNodes; i++) {
- String key = String.format(VIRTUAL_NODE_FORMAT, location, i);
- String hash = getHash(key);
- ring.remove(hash);
- }
- } finally {
- writeLock.unlock();
- }
- }
-
- /**
- * Return location (owner) of specified item. Owner is the next
- * entry on the hash ring (with a hash value > hash value of item).
- * @param item Item to look for.
- * @return The location of the item.
- */
- public String getLocation(String item) {
- readLock.lock();
- try {
- if (ring.isEmpty()) {
- return null;
- }
- String hash = getHash(item);
- if (!ring.containsKey(hash)) {
- SortedMap<String, String> tailMap = ring.tailMap(hash);
- hash = tailMap.isEmpty() ? ring.firstKey() : tailMap.firstKey();
- }
- String virtualNode = ring.get(hash);
- int index = virtualNode.lastIndexOf(SEPERATOR);
- if (index >= 0) {
- return virtualNode.substring(0, index);
- } else {
- return virtualNode;
- }
- } finally {
- readLock.unlock();
- }
- }
-
- public String getHash(String key) {
- return MD5Hash.digest(key).toString();
- }
-
- /**
- * Get the locations in the ring.
- * @return Set of locations in the ring.
- */
- public Set<String> getLocations() {
- return entryToVirtualNodes.keySet();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/utils/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/utils/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/utils/package-info.java
deleted file mode 100644
index 7149675..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/utils/package-info.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Assorted utility classes and helpers for HDFS Federation.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-
-package org.apache.hadoop.hdfs.server.federation.utils;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
deleted file mode 100644
index 9dfd1b7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
+++ /dev/null
@@ -1,627 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.tools.federation;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
-import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
-import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
-import org.apache.hadoop.hdfs.server.federation.router.RouterClient;
-import org.apache.hadoop.hdfs.server.federation.router.RouterQuotaUsage;
-import org.apache.hadoop.hdfs.server.federation.router.RouterStateManager;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class provides some Federation administrative access shell commands.
- */
-@Private
-public class RouterAdmin extends Configured implements Tool {
-
- private static final Logger LOG = LoggerFactory.getLogger(RouterAdmin.class);
-
- private RouterClient client;
-
- public static void main(String[] argv) throws Exception {
- Configuration conf = new HdfsConfiguration();
- RouterAdmin admin = new RouterAdmin(conf);
-
- int res = ToolRunner.run(admin, argv);
- System.exit(res);
- }
-
- public RouterAdmin(Configuration conf) {
- super(conf);
- }
-
- /**
- * Print the usage message.
- */
- public void printUsage() {
- String usage = "Federation Admin Tools:\n"
- + "\t[-add <source> <nameservice> <destination> "
- + "[-readonly] [-order HASH|LOCAL|RANDOM|HASH_ALL] "
- + "-owner <owner> -group <group> -mode <mode>]\n"
- + "\t[-rm <source>]\n"
- + "\t[-ls <path>]\n"
- + "\t[-setQuota <path> -nsQuota <nsQuota> -ssQuota "
- + "<quota in bytes or quota size string>]\n"
- + "\t[-clrQuota <path>]\n"
- + "\t[-safemode enter | leave | get]\n";
-
- System.out.println(usage);
- }
-
- @Override
- public int run(String[] argv) throws Exception {
- if (argv.length < 1) {
- System.err.println("Not enough parameters specificed");
- printUsage();
- return -1;
- }
-
- int exitCode = -1;
- int i = 0;
- String cmd = argv[i++];
-
- // Verify that we have enough command line parameters
- if ("-add".equals(cmd)) {
- if (argv.length < 4) {
- System.err.println("Not enough parameters specificed for cmd " + cmd);
- printUsage();
- return exitCode;
- }
- } else if ("-rm".equalsIgnoreCase(cmd)) {
- if (argv.length < 2) {
- System.err.println("Not enough parameters specificed for cmd " + cmd);
- printUsage();
- return exitCode;
- }
- } else if ("-setQuota".equalsIgnoreCase(cmd)) {
- if (argv.length < 4) {
- System.err.println("Not enough parameters specificed for cmd " + cmd);
- printUsage();
- return exitCode;
- }
- } else if ("-clrQuota".equalsIgnoreCase(cmd)) {
- if (argv.length < 2) {
- System.err.println("Not enough parameters specificed for cmd " + cmd);
- printUsage();
- return exitCode;
- }
- } else if ("-safemode".equalsIgnoreCase(cmd)) {
- if (argv.length < 2) {
- System.err.println("Not enough parameters specificed for cmd " + cmd);
- printUsage();
- return exitCode;
- }
- }
-
- // Initialize RouterClient
- try {
- String address = getConf().getTrimmed(
- DFSConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_KEY,
- DFSConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_DEFAULT);
- InetSocketAddress routerSocket = NetUtils.createSocketAddr(address);
- client = new RouterClient(routerSocket, getConf());
- } catch (RPC.VersionMismatch v) {
- System.err.println(
- "Version mismatch between client and server... command aborted");
- return exitCode;
- } catch (IOException e) {
- System.err.println("Bad connection to Router... command aborted");
- return exitCode;
- }
-
- Exception debugException = null;
- exitCode = 0;
- try {
- if ("-add".equals(cmd)) {
- if (addMount(argv, i)) {
- System.out.println("Successfuly added mount point " + argv[i]);
- }
- } else if ("-rm".equals(cmd)) {
- if (removeMount(argv[i])) {
- System.out.println("Successfully removed mount point " + argv[i]);
- }
- } else if ("-ls".equals(cmd)) {
- if (argv.length > 1) {
- listMounts(argv[i]);
- } else {
- listMounts("/");
- }
- } else if ("-setQuota".equals(cmd)) {
- if (setQuota(argv, i)) {
- System.out.println(
- "Successfully set quota for mount point " + argv[i]);
- }
- } else if ("-clrQuota".equals(cmd)) {
- if (clrQuota(argv[i])) {
- System.out.println(
- "Successfully clear quota for mount point " + argv[i]);
- }
- } else if ("-safemode".equals(cmd)) {
- manageSafeMode(argv[i]);
- } else {
- printUsage();
- return exitCode;
- }
- } catch (IllegalArgumentException arge) {
- debugException = arge;
- exitCode = -1;
- System.err.println(cmd.substring(1) + ": " + arge.getLocalizedMessage());
- printUsage();
- } catch (RemoteException e) {
- // This is a error returned by the server.
- // Print out the first line of the error message, ignore the stack trace.
- exitCode = -1;
- debugException = e;
- try {
- String[] content;
- content = e.getLocalizedMessage().split("\n");
- System.err.println(cmd.substring(1) + ": " + content[0]);
- e.printStackTrace();
- } catch (Exception ex) {
- System.err.println(cmd.substring(1) + ": " + ex.getLocalizedMessage());
- e.printStackTrace();
- debugException = ex;
- }
- } catch (Exception e) {
- exitCode = -1;
- debugException = e;
- System.err.println(cmd.substring(1) + ": " + e.getLocalizedMessage());
- e.printStackTrace();
- }
- if (debugException != null) {
- LOG.debug("Exception encountered", debugException);
- }
- return exitCode;
- }
-
- /**
- * Add a mount table entry or update if it exists.
- *
- * @param parameters Parameters for the mount point.
- * @param i Index in the parameters.
- */
- public boolean addMount(String[] parameters, int i) throws IOException {
- // Mandatory parameters
- String mount = parameters[i++];
- String[] nss = parameters[i++].split(",");
- String dest = parameters[i++];
-
- // Optional parameters
- boolean readOnly = false;
- String owner = null;
- String group = null;
- FsPermission mode = null;
- DestinationOrder order = DestinationOrder.HASH;
- while (i < parameters.length) {
- if (parameters[i].equals("-readonly")) {
- readOnly = true;
- } else if (parameters[i].equals("-order")) {
- i++;
- try {
- order = DestinationOrder.valueOf(parameters[i]);
- } catch(Exception e) {
- System.err.println("Cannot parse order: " + parameters[i]);
- }
- } else if (parameters[i].equals("-owner")) {
- i++;
- owner = parameters[i];
- } else if (parameters[i].equals("-group")) {
- i++;
- group = parameters[i];
- } else if (parameters[i].equals("-mode")) {
- i++;
- short modeValue = Short.parseShort(parameters[i], 8);
- mode = new FsPermission(modeValue);
- }
-
- i++;
- }
-
- return addMount(mount, nss, dest, readOnly, order,
- new ACLEntity(owner, group, mode));
- }
-
- /**
- * Add a mount table entry or update if it exists.
- *
- * @param mount Mount point.
- * @param nss Namespaces where this is mounted to.
- * @param dest Destination path.
- * @param readonly If the mount point is read only.
- * @param order Order of the destination locations.
- * @param aclInfo the ACL info for mount point.
- * @return If the mount point was added.
- * @throws IOException Error adding the mount point.
- */
- public boolean addMount(String mount, String[] nss, String dest,
- boolean readonly, DestinationOrder order, ACLEntity aclInfo)
- throws IOException {
- // Get the existing entry
- MountTableManager mountTable = client.getMountTableManager();
- GetMountTableEntriesRequest getRequest =
- GetMountTableEntriesRequest.newInstance(mount);
- GetMountTableEntriesResponse getResponse =
- mountTable.getMountTableEntries(getRequest);
- List<MountTable> results = getResponse.getEntries();
- MountTable existingEntry = null;
- for (MountTable result : results) {
- if (mount.equals(result.getSourcePath())) {
- existingEntry = result;
- }
- }
-
- if (existingEntry == null) {
- // Create and add the entry if it doesn't exist
- Map<String, String> destMap = new LinkedHashMap<>();
- for (String ns : nss) {
- destMap.put(ns, dest);
- }
- MountTable newEntry = MountTable.newInstance(mount, destMap);
- if (readonly) {
- newEntry.setReadOnly(true);
- }
- if (order != null) {
- newEntry.setDestOrder(order);
- }
-
- // Set ACL info for mount table entry
- if (aclInfo.getOwner() != null) {
- newEntry.setOwnerName(aclInfo.getOwner());
- }
-
- if (aclInfo.getGroup() != null) {
- newEntry.setGroupName(aclInfo.getGroup());
- }
-
- if (aclInfo.getMode() != null) {
- newEntry.setMode(aclInfo.getMode());
- }
-
- AddMountTableEntryRequest request =
- AddMountTableEntryRequest.newInstance(newEntry);
- AddMountTableEntryResponse addResponse =
- mountTable.addMountTableEntry(request);
- boolean added = addResponse.getStatus();
- if (!added) {
- System.err.println("Cannot add mount point " + mount);
- }
- return added;
- } else {
- // Update the existing entry if it exists
- for (String nsId : nss) {
- if (!existingEntry.addDestination(nsId, dest)) {
- System.err.println("Cannot add destination at " + nsId + " " + dest);
- }
- }
- if (readonly) {
- existingEntry.setReadOnly(true);
- }
- if (order != null) {
- existingEntry.setDestOrder(order);
- }
-
- // Update ACL info of mount table entry
- if (aclInfo.getOwner() != null) {
- existingEntry.setOwnerName(aclInfo.getOwner());
- }
-
- if (aclInfo.getGroup() != null) {
- existingEntry.setGroupName(aclInfo.getGroup());
- }
-
- if (aclInfo.getMode() != null) {
- existingEntry.setMode(aclInfo.getMode());
- }
-
- UpdateMountTableEntryRequest updateRequest =
- UpdateMountTableEntryRequest.newInstance(existingEntry);
- UpdateMountTableEntryResponse updateResponse =
- mountTable.updateMountTableEntry(updateRequest);
- boolean updated = updateResponse.getStatus();
- if (!updated) {
- System.err.println("Cannot update mount point " + mount);
- }
- return updated;
- }
- }
-
- /**
- * Remove mount point.
- *
- * @param path Path to remove.
- * @throws IOException If it cannot be removed.
- */
- public boolean removeMount(String path) throws IOException {
- MountTableManager mountTable = client.getMountTableManager();
- RemoveMountTableEntryRequest request =
- RemoveMountTableEntryRequest.newInstance(path);
- RemoveMountTableEntryResponse response =
- mountTable.removeMountTableEntry(request);
- boolean removed = response.getStatus();
- if (!removed) {
- System.out.println("Cannot remove mount point " + path);
- }
- return removed;
- }
-
- /**
- * List mount points.
- *
- * @param path Path to list.
- * @throws IOException If it cannot be listed.
- */
- public void listMounts(String path) throws IOException {
- MountTableManager mountTable = client.getMountTableManager();
- GetMountTableEntriesRequest request =
- GetMountTableEntriesRequest.newInstance(path);
- GetMountTableEntriesResponse response =
- mountTable.getMountTableEntries(request);
- List<MountTable> entries = response.getEntries();
- printMounts(entries);
- }
-
- private static void printMounts(List<MountTable> entries) {
- System.out.println("Mount Table Entries:");
- System.out.println(String.format(
- "%-25s %-25s %-25s %-25s %-25s %-25s",
- "Source", "Destinations", "Owner", "Group", "Mode", "Quota/Usage"));
- for (MountTable entry : entries) {
- StringBuilder destBuilder = new StringBuilder();
- for (RemoteLocation location : entry.getDestinations()) {
- if (destBuilder.length() > 0) {
- destBuilder.append(",");
- }
- destBuilder.append(String.format("%s->%s", location.getNameserviceId(),
- location.getDest()));
- }
- System.out.print(String.format("%-25s %-25s", entry.getSourcePath(),
- destBuilder.toString()));
-
- System.out.print(String.format(" %-25s %-25s %-25s",
- entry.getOwnerName(), entry.getGroupName(), entry.getMode()));
-
- System.out.println(String.format(" %-25s", entry.getQuota()));
- }
- }
-
- /**
- * Set quota for a mount table entry.
- *
- * @param parameters Parameters of the quota.
- * @param i Index in the parameters.
- */
- private boolean setQuota(String[] parameters, int i) throws IOException {
- long nsQuota = HdfsConstants.QUOTA_DONT_SET;
- long ssQuota = HdfsConstants.QUOTA_DONT_SET;
-
- String mount = parameters[i++];
- while (i < parameters.length) {
- if (parameters[i].equals("-nsQuota")) {
- i++;
- try {
- nsQuota = Long.parseLong(parameters[i]);
- } catch (Exception e) {
- System.err.println("Cannot parse nsQuota: " + parameters[i]);
- }
- } else if (parameters[i].equals("-ssQuota")) {
- i++;
- try {
- ssQuota = StringUtils.TraditionalBinaryPrefix
- .string2long(parameters[i]);
- } catch (Exception e) {
- System.err.println("Cannot parse ssQuota: " + parameters[i]);
- }
- }
-
- i++;
- }
-
- if (nsQuota <= 0 || ssQuota <= 0) {
- System.err.println("Input quota value should be a positive number.");
- return false;
- }
-
- return updateQuota(mount, nsQuota, ssQuota);
- }
-
- /**
- * Clear quota of the mount point.
- *
- * @param mount Mount table to clear
- * @return If the quota was cleared.
- * @throws IOException Error clearing the mount point.
- */
- private boolean clrQuota(String mount) throws IOException {
- return updateQuota(mount, HdfsConstants.QUOTA_DONT_SET,
- HdfsConstants.QUOTA_DONT_SET);
- }
-
- /**
- * Update quota of specified mount table.
- *
- * @param mount Specified mount table to update.
- * @param nsQuota Namespace quota.
- * @param ssQuota Storage space quota.
- * @return If the quota was updated.
- * @throws IOException Error updating quota.
- */
- private boolean updateQuota(String mount, long nsQuota, long ssQuota)
- throws IOException {
- // Get existing entry
- MountTableManager mountTable = client.getMountTableManager();
- GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest
- .newInstance(mount);
- GetMountTableEntriesResponse getResponse = mountTable
- .getMountTableEntries(getRequest);
- List<MountTable> results = getResponse.getEntries();
- MountTable existingEntry = null;
- for (MountTable result : results) {
- if (mount.equals(result.getSourcePath())) {
- existingEntry = result;
- break;
- }
- }
-
- if (existingEntry == null) {
- return false;
- } else {
- long nsCount = existingEntry.getQuota().getFileAndDirectoryCount();
- long ssCount = existingEntry.getQuota().getSpaceConsumed();
- // If nsQuota or ssQuota was unset, reset corresponding usage
- // value to zero.
- if (nsQuota == HdfsConstants.QUOTA_DONT_SET) {
- nsCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
- }
-
- if (nsQuota == HdfsConstants.QUOTA_DONT_SET) {
- ssCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
- }
-
- RouterQuotaUsage updatedQuota = new RouterQuotaUsage.Builder()
- .fileAndDirectoryCount(nsCount).quota(nsQuota)
- .spaceConsumed(ssCount).spaceQuota(ssQuota).build();
- existingEntry.setQuota(updatedQuota);
- }
-
- UpdateMountTableEntryRequest updateRequest =
- UpdateMountTableEntryRequest.newInstance(existingEntry);
- UpdateMountTableEntryResponse updateResponse = mountTable
- .updateMountTableEntry(updateRequest);
- return updateResponse.getStatus();
- }
-
- /**
- * Manager the safe mode state.
- * @param cmd Input command, enter or leave safe mode.
- * @throws IOException
- */
- private void manageSafeMode(String cmd) throws IOException {
- if (cmd.equals("enter")) {
- if (enterSafeMode()) {
- System.out.println("Successfully enter safe mode.");
- }
- } else if (cmd.equals("leave")) {
- if (leaveSafeMode()) {
- System.out.println("Successfully leave safe mode.");
- }
- } else if (cmd.equals("get")) {
- boolean result = getSafeMode();
- System.out.println("Safe Mode: " + result);
- }
- }
-
- /**
- * Request the Router entering safemode state.
- * @return Return true if entering safemode successfully.
- * @throws IOException
- */
- private boolean enterSafeMode() throws IOException {
- RouterStateManager stateManager = client.getRouterStateManager();
- EnterSafeModeResponse response = stateManager.enterSafeMode(
- EnterSafeModeRequest.newInstance());
- return response.getStatus();
- }
-
- /**
- * Request the Router leaving safemode state.
- * @return Return true if leaving safemode successfully.
- * @throws IOException
- */
- private boolean leaveSafeMode() throws IOException {
- RouterStateManager stateManager = client.getRouterStateManager();
- LeaveSafeModeResponse response = stateManager.leaveSafeMode(
- LeaveSafeModeRequest.newInstance());
- return response.getStatus();
- }
-
- /**
- * Verify if current Router state is safe mode state.
- * @return True if the Router is in safe mode.
- * @throws IOException
- */
- private boolean getSafeMode() throws IOException {
- RouterStateManager stateManager = client.getRouterStateManager();
- GetSafeModeResponse response = stateManager.getSafeMode(
- GetSafeModeRequest.newInstance());
- return response.isInSafeMode();
- }
-
- /**
- * Inner class that stores ACL info of mount table.
- */
- static class ACLEntity {
- private final String owner;
- private final String group;
- private final FsPermission mode;
-
- ACLEntity(String owner, String group, FsPermission mode) {
- this.owner = owner;
- this.group = group;
- this.mode = mode;
- }
-
- public String getOwner() {
- return owner;
- }
-
- public String getGroup() {
- return group;
- }
-
- public FsPermission getMode() {
- return mode;
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/package-info.java
deleted file mode 100644
index 466c3d3..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/package-info.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * It includes the tools to manage the Router-based federation. Includes the
- * utilities to add and remove mount table entries.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-package org.apache.hadoop.hdfs.tools.federation;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/FederationProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/FederationProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/FederationProtocol.proto
deleted file mode 100644
index b0d6982..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/FederationProtocol.proto
+++ /dev/null
@@ -1,244 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-option java_package = "org.apache.hadoop.hdfs.federation.protocol.proto";
-option java_outer_classname = "HdfsServerFederationProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-package hadoop.hdfs;
-
-import "hdfs.proto";
-
-/////////////////////////////////////////////////
-// Membership
-/////////////////////////////////////////////////
-
-message NamenodeMembershipStatsRecordProto {
- optional uint64 totalSpace = 1;
- optional uint64 availableSpace = 2;
- optional uint64 providedSpace = 3;
-
- optional uint64 numOfFiles = 10;
- optional uint64 numOfBlocks = 11;
- optional uint64 numOfBlocksMissing = 12;
- optional uint64 numOfBlocksPendingReplication = 13;
- optional uint64 numOfBlocksUnderReplicated = 14;
- optional uint64 numOfBlocksPendingDeletion = 15;
-
- optional uint32 numOfActiveDatanodes = 20;
- optional uint32 numOfDeadDatanodes = 21;
- optional uint32 numOfDecommissioningDatanodes = 22;
- optional uint32 numOfDecomActiveDatanodes = 23;
- optional uint32 numOfDecomDeadDatanodes = 24;
-}
-
-message NamenodeMembershipRecordProto {
- optional uint64 dateCreated = 1;
- optional uint64 dateModified = 2;
- optional uint64 lastContact = 3;
- optional string routerId = 4;
- optional string nameserviceId = 5;
- optional string namenodeId = 6;
- optional string clusterId = 7;
- optional string blockPoolId = 8;
- optional string webAddress = 9;
- optional string rpcAddress = 10;
- optional string serviceAddress = 11;
- optional string lifelineAddress = 12;
- optional string state = 13;
- optional bool isSafeMode = 14;
-
- optional NamenodeMembershipStatsRecordProto stats = 15;
-}
-
-message FederationNamespaceInfoProto {
- optional string blockPoolId = 1;
- optional string clusterId = 2;
- optional string nameserviceId = 3;
-}
-
-message GetNamenodeRegistrationsRequestProto {
- optional NamenodeMembershipRecordProto membership = 1;
-}
-
-message GetNamenodeRegistrationsResponseProto {
- repeated NamenodeMembershipRecordProto namenodeMemberships = 1;
-}
-
-message GetExpiredRegistrationsRequestProto {
-}
-
-message GetNamespaceInfoRequestProto {
-}
-
-message GetNamespaceInfoResponseProto {
- repeated FederationNamespaceInfoProto namespaceInfos = 1;
-}
-
-message UpdateNamenodeRegistrationRequestProto {
- optional string nameserviceId = 1;
- optional string namenodeId = 2;
- optional string state = 3;
-}
-
-message UpdateNamenodeRegistrationResponseProto {
- optional bool status = 1;
-}
-
-message NamenodeHeartbeatRequestProto {
- optional NamenodeMembershipRecordProto namenodeMembership = 1;
-}
-
-message NamenodeHeartbeatResponseProto {
- optional bool status = 1;
-}
-
-
-/////////////////////////////////////////////////
-// Mount table
-/////////////////////////////////////////////////
-
-message RemoteLocationProto {
- optional string nameserviceId = 1;
- optional string path = 2;
-}
-
-message MountTableRecordProto {
- optional string srcPath = 1;
- repeated RemoteLocationProto destinations = 2;
- optional uint64 dateCreated = 3;
- optional uint64 dateModified = 4;
- optional bool readOnly = 5 [default = false];
-
- enum DestOrder {
- HASH = 0;
- LOCAL = 1;
- RANDOM = 2;
- HASH_ALL = 3;
- }
- optional DestOrder destOrder = 6 [default = HASH];
-
- optional string ownerName = 10;
- optional string groupName = 11;
- optional int32 mode = 12;
-
- optional QuotaUsageProto quota = 13;
-}
-
-message AddMountTableEntryRequestProto {
- optional MountTableRecordProto entry = 1;
-}
-
-message AddMountTableEntryResponseProto {
- optional bool status = 1;
-}
-
-message UpdateMountTableEntryRequestProto {
- optional MountTableRecordProto entry = 1;
-}
-
-message UpdateMountTableEntryResponseProto {
- optional bool status = 1;
-}
-
-message RemoveMountTableEntryRequestProto {
- optional string srcPath = 1;
-}
-
-message RemoveMountTableEntryResponseProto{
- optional bool status = 1;
-}
-
-message GetMountTableEntriesRequestProto {
- optional string srcPath = 1;
-}
-
-message GetMountTableEntriesResponseProto {
- repeated MountTableRecordProto entries = 1;
- optional uint64 timestamp = 2;
-}
-
-
-/////////////////////////////////////////////////
-// Routers
-/////////////////////////////////////////////////
-
-message StateStoreVersionRecordProto {
- optional uint64 membershipVersion = 1;
- optional uint64 mountTableVersion = 2;
-}
-
-message RouterRecordProto {
- optional uint64 dateCreated = 1;
- optional uint64 dateModified = 2;
- optional string address = 3;
- optional string status = 4;
- optional StateStoreVersionRecordProto stateStoreVersion = 5;
- optional string version = 6;
- optional string compileInfo = 7;
- optional uint64 dateStarted = 8;
-}
-
-message GetRouterRegistrationRequestProto {
- optional string routerId = 1;
-}
-
-message GetRouterRegistrationResponseProto {
- optional RouterRecordProto router = 1;
-}
-
-message GetRouterRegistrationsRequestProto {
-}
-
-message GetRouterRegistrationsResponseProto {
- optional uint64 timestamp = 1;
- repeated RouterRecordProto routers = 2;
-}
-
-message RouterHeartbeatRequestProto {
- optional RouterRecordProto router = 1;
-}
-
-message RouterHeartbeatResponseProto {
- optional bool status = 1;
-}
-
-/////////////////////////////////////////////////
-// Route State
-/////////////////////////////////////////////////
-
-message EnterSafeModeRequestProto {
-}
-
-message EnterSafeModeResponseProto {
- optional bool status = 1;
-}
-
-message LeaveSafeModeRequestProto {
-}
-
-message LeaveSafeModeResponseProto {
- optional bool status = 1;
-}
-
-message GetSafeModeRequestProto {
-}
-
-message GetSafeModeResponseProto {
- optional bool isInSafeMode = 1;
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/RouterProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/RouterProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/RouterProtocol.proto
deleted file mode 100644
index a4e4d65..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/RouterProtocol.proto
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-option java_package = "org.apache.hadoop.hdfs.protocol.proto";
-option java_outer_classname = "RouterProtocolProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-package hadoop.hdfs.router;
-
-import "FederationProtocol.proto";
-
-service RouterAdminProtocolService {
- /**
- * Add a mount table entry.
- */
- rpc addMountTableEntry(AddMountTableEntryRequestProto) returns(AddMountTableEntryResponseProto);
-
- /**
- * Update an existing mount table entry without copying files.
- */
- rpc updateMountTableEntry(UpdateMountTableEntryRequestProto) returns(UpdateMountTableEntryResponseProto);
-
- /**
- * Remove a mount table entry.
- */
- rpc removeMountTableEntry(RemoveMountTableEntryRequestProto) returns(RemoveMountTableEntryResponseProto);
-
- /**
- * Get matching mount entries
- */
- rpc getMountTableEntries(GetMountTableEntriesRequestProto) returns(GetMountTableEntriesResponseProto);
-
- /**
- * Transform Router state to safe mode state.
- */
- rpc enterSafeMode(EnterSafeModeRequestProto) returns (EnterSafeModeResponseProto);
-
- /**
- * Transform Router state from safe mode to running state.
- */
- rpc leaveSafeMode(LeaveSafeModeRequestProto) returns (LeaveSafeModeResponseProto);
-
- /**
- * Verify if current Router state is safe mode state.
- */
- rpc getSafeMode(GetSafeModeRequestProto) returns (GetSafeModeResponseProto);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index f90daba..1e83325 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4871,412 +4871,4 @@
will throw NameNodeFormatException.
</description>
</property>
-
- <property>
- <name>dfs.federation.router.default.nameserviceId</name>
- <value></value>
- <description>
- Nameservice identifier of the default subcluster to monitor.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.rpc.enable</name>
- <value>true</value>
- <description>
- If true, the RPC service to handle client requests in the router is
- enabled.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.rpc-address</name>
- <value>0.0.0.0:8888</value>
- <description>
- RPC address that handles all clients requests.
- The value of this property will take the form of router-host1:rpc-port.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.rpc-bind-host</name>
- <value></value>
- <description>
- The actual address the RPC server will bind to. If this optional address is
- set, it overrides only the hostname portion of
- dfs.federation.router.rpc-address. This is useful for making the name node
- listen on all interfaces by setting it to 0.0.0.0.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.handler.count</name>
- <value>10</value>
- <description>
- The number of server threads for the router to handle RPC requests from
- clients.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.handler.queue.size</name>
- <value>100</value>
- <description>
- The size of the queue for the number of handlers to handle RPC client requests.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.reader.count</name>
- <value>1</value>
- <description>
- The number of readers for the router to handle RPC client requests.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.reader.queue.size</name>
- <value>100</value>
- <description>
- The size of the queue for the number of readers for the router to handle RPC client requests.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.connection.pool-size</name>
- <value>1</value>
- <description>
- Size of the pool of connections from the router to namenodes.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.connection.clean.ms</name>
- <value>10000</value>
- <description>
- Time interval, in milliseconds, to check if the connection pool should
- remove unused connections.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.connection.pool.clean.ms</name>
- <value>60000</value>
- <description>
- Time interval, in milliseconds, to check if the connection manager should
- remove unused connection pools.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.metrics.enable</name>
- <value>true</value>
- <description>
- If the metrics in the router are enabled.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.metrics.class</name>
- <value>org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCPerformanceMonitor</value>
- <description>
- Class to monitor the RPC system in the router. It must implement the
- RouterRpcMonitor interface.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.admin.enable</name>
- <value>true</value>
- <description>
- If true, the RPC admin service to handle client requests in the router is
- enabled.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.admin-address</name>
- <value>0.0.0.0:8111</value>
- <description>
- RPC address that handles the admin requests.
- The value of this property will take the form of router-host1:rpc-port.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.admin-bind-host</name>
- <value></value>
- <description>
- The actual address the RPC admin server will bind to. If this optional
- address is set, it overrides only the hostname portion of
- dfs.federation.router.admin-address. This is useful for making the name
- node listen on all interfaces by setting it to 0.0.0.0.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.admin.handler.count</name>
- <value>1</value>
- <description>
- The number of server threads for the router to handle RPC requests from
- admin.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.http-address</name>
- <value>0.0.0.0:50071</value>
- <description>
- HTTP address that handles the web requests to the Router.
- The value of this property will take the form of router-host1:http-port.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.http-bind-host</name>
- <value></value>
- <description>
- The actual address the HTTP server will bind to. If this optional
- address is set, it overrides only the hostname portion of
- dfs.federation.router.http-address. This is useful for making the name
- node listen on all interfaces by setting it to 0.0.0.0.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.https-address</name>
- <value>0.0.0.0:50072</value>
- <description>
- HTTPS address that handles the web requests to the Router.
- The value of this property will take the form of router-host1:https-port.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.https-bind-host</name>
- <value></value>
- <description>
- The actual address the HTTPS server will bind to. If this optional
- address is set, it overrides only the hostname portion of
- dfs.federation.router.https-address. This is useful for making the name
- node listen on all interfaces by setting it to 0.0.0.0.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.http.enable</name>
- <value>true</value>
- <description>
- If the HTTP service to handle client requests in the router is enabled.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.metrics.enable</name>
- <value>true</value>
- <description>
- If the metrics service in the router is enabled.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.file.resolver.client.class</name>
- <value>org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver</value>
- <description>
- Class to resolve files to subclusters.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.namenode.resolver.client.class</name>
- <value>org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver</value>
- <description>
- Class to resolve the namenode for a subcluster.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.store.enable</name>
- <value>true</value>
- <description>
- If true, the Router connects to the State Store.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.store.serializer</name>
- <value>org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreSerializerPBImpl</value>
- <description>
- Class to serialize State Store records.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.store.driver.class</name>
- <value>org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl</value>
- <description>
- Class to implement the State Store. There are three implementation classes currently
- being supported:
- org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl,
- org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileSystemImpl and
- org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl.
- These implementation classes use the local file, filesystem and ZooKeeper as a backend respectively.
- By default it uses the ZooKeeper as the default State Store.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.store.connection.test</name>
- <value>60000</value>
- <description>
- How often to check for the connection to the State Store in milliseconds.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.cache.ttl</name>
- <value>1m</value>
- <description>
- How often to refresh the State Store caches in milliseconds. This setting
- supports multiple time unit suffixes as described in
- dfs.heartbeat.interval. If no suffix is specified then milliseconds is
- assumed.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.store.membership.expiration</name>
- <value>300000</value>
- <description>
- Expiration time in milliseconds for a membership record.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.heartbeat.enable</name>
- <value>true</value>
- <description>
- If true, the Router heartbeats into the State Store.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.heartbeat.interval</name>
- <value>5000</value>
- <description>
- How often the Router should heartbeat into the State Store in milliseconds.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.heartbeat-state.interval</name>
- <value>5s</value>
- <description>
- How often the Router should heartbeat its state into the State Store in
- milliseconds. This setting supports multiple time unit suffixes as
- described in dfs.federation.router.quota-cache.update.interval.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.store.router.expiration</name>
- <value>5m</value>
- <description>
- Expiration time in milliseconds for a router state record. This setting
- supports multiple time unit suffixes as described in
- dfs.federation.router.quota-cache.update.interval.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.safemode.enable</name>
- <value>true</value>
- <description>
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.safemode.extension</name>
- <value>30s</value>
- <description>
- Time after startup that the Router is in safe mode. This setting
- supports multiple time unit suffixes as described in
- dfs.heartbeat.interval. If no suffix is specified then milliseconds is
- assumed.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.safemode.expiration</name>
- <value>3m</value>
- <description>
- Time without being able to reach the State Store to enter safe mode. This
- setting supports multiple time unit suffixes as described in
- dfs.heartbeat.interval. If no suffix is specified then milliseconds is
- assumed.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.monitor.namenode</name>
- <value></value>
- <description>
- The identifier of the namenodes to monitor and heartbeat.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.monitor.localnamenode.enable</name>
- <value>true</value>
- <description>
- If true, the Router should monitor the namenode in the local machine.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.mount-table.max-cache-size</name>
- <value>10000</value>
- <description>
- Maximum number of mount table cache entries to have.
- By default, remove cache entries if we have more than 10k.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.quota.enable</name>
- <value>false</value>
- <description>
- Set to true to enable quota system in Router.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.quota-cache.update.interval</name>
- <value>60s</value>
- <description>
- Interval time for updating quota usage cache in Router.
- This property is used only if the value of
- dfs.federation.router.quota.enable is true.
- This setting supports multiple time unit suffixes as described
- in dfs.heartbeat.interval. If no suffix is specified then milliseconds
- is assumed.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.client.thread-size</name>
- <value>32</value>
- <description>
- Max threads size for the RouterClient to execute concurrent
- requests.
- </description>
- </property>
-
- <property>
- <name>dfs.federation.router.client.retry.max.attempts</name>
- <value>3</value>
- <description>
- Max retry attempts for the RouterClient talking to the Router.
- </description>
- </property>
</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
HDFS-13215. RBF: Move Router to its own module. Contributed by Wei Yan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6e2b5fa4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6e2b5fa4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6e2b5fa4
Branch: refs/heads/trunk
Commit: 6e2b5fa493ff8e8c2bb28e6f6f4c19347bc9b99d
Parents: e65ff1c
Author: weiy <we...@apache.org>
Authored: Mon Mar 19 22:13:40 2018 -0700
Committer: weiy <we...@apache.org>
Committed: Mon Mar 19 22:13:40 2018 -0700
----------------------------------------------------------------------
dev-support/bin/dist-layout-stitching | 1 +
.../dev-support/findbugsExcludeFile.xml | 22 +
hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 236 ++
.../hdfs/protocolPB/RouterAdminProtocolPB.java | 44 +
...uterAdminProtocolServerSideTranslatorPB.java | 211 ++
.../RouterAdminProtocolTranslatorPB.java | 208 ++
.../hadoop/hdfs/protocolPB/package-info.java | 18 +
.../federation/metrics/FederationMBean.java | 223 ++
.../federation/metrics/FederationMetrics.java | 766 ++++++
.../federation/metrics/FederationRPCMBean.java | 92 +
.../metrics/FederationRPCMetrics.java | 250 ++
.../FederationRPCPerformanceMonitor.java | 221 ++
.../federation/metrics/NamenodeBeanMetrics.java | 634 +++++
.../federation/metrics/StateStoreMBean.java | 45 +
.../federation/metrics/StateStoreMetrics.java | 149 ++
.../server/federation/metrics/package-info.java | 27 +
.../resolver/ActiveNamenodeResolver.java | 118 +
.../resolver/FederationNamenodeContext.java | 87 +
.../FederationNamenodeServiceState.java | 46 +
.../resolver/FederationNamespaceInfo.java | 73 +
.../resolver/FileSubclusterResolver.java | 75 +
.../resolver/MembershipNamenodeResolver.java | 317 +++
.../federation/resolver/MountTableManager.java | 80 +
.../federation/resolver/MountTableResolver.java | 597 +++++
.../MultipleDestinationMountTableResolver.java | 116 +
.../resolver/NamenodePriorityComparator.java | 63 +
.../resolver/NamenodeStatusReport.java | 406 ++++
.../federation/resolver/PathLocation.java | 212 ++
.../federation/resolver/RemoteLocation.java | 76 +
.../resolver/order/DestinationOrder.java | 30 +
.../resolver/order/HashFirstResolver.java | 71 +
.../federation/resolver/order/HashResolver.java | 137 ++
.../resolver/order/LocalResolver.java | 297 +++
.../resolver/order/OrderedResolver.java | 37 +
.../resolver/order/RandomResolver.java | 62 +
.../federation/resolver/order/package-info.java | 29 +
.../federation/resolver/package-info.java | 41 +
.../federation/router/ConnectionContext.java | 104 +
.../federation/router/ConnectionManager.java | 435 ++++
.../federation/router/ConnectionPool.java | 337 +++
.../federation/router/ConnectionPoolId.java | 123 +
.../server/federation/router/DFSRouter.java | 76 +
.../server/federation/router/ErasureCoding.java | 198 ++
.../federation/router/FederationUtil.java | 208 ++
.../router/NamenodeHeartbeatService.java | 362 +++
.../federation/router/PeriodicService.java | 198 ++
.../hdfs/server/federation/router/Quota.java | 208 ++
.../server/federation/router/RBFConfigKeys.java | 229 ++
.../router/RemoteLocationContext.java | 68 +
.../server/federation/router/RemoteMethod.java | 164 ++
.../server/federation/router/RemoteParam.java | 71 +
.../hdfs/server/federation/router/Router.java | 655 ++++++
.../federation/router/RouterAdminServer.java | 298 +++
.../server/federation/router/RouterClient.java | 80 +
.../router/RouterHeartbeatService.java | 166 ++
.../federation/router/RouterHttpServer.java | 124 +
.../server/federation/router/RouterMetrics.java | 73 +
.../federation/router/RouterMetricsService.java | 108 +
.../router/RouterPermissionChecker.java | 82 +
.../federation/router/RouterQuotaManager.java | 172 ++
.../router/RouterQuotaUpdateService.java | 227 ++
.../federation/router/RouterQuotaUsage.java | 119 +
.../federation/router/RouterRpcClient.java | 1021 ++++++++
.../federation/router/RouterRpcMonitor.java | 108 +
.../federation/router/RouterRpcServer.java | 2214 ++++++++++++++++++
.../router/RouterSafeModeException.java | 53 +
.../router/RouterSafemodeService.java | 149 ++
.../federation/router/RouterServiceState.java | 31 +
.../federation/router/RouterStateManager.java | 50 +
.../server/federation/router/package-info.java | 31 +
.../federation/store/CachedRecordStore.java | 242 ++
.../federation/store/MembershipStore.java | 126 +
.../federation/store/MountTableStore.java | 49 +
.../server/federation/store/RecordStore.java | 100 +
.../server/federation/store/RouterStore.java | 78 +
.../federation/store/StateStoreCache.java | 36 +
.../store/StateStoreCacheUpdateService.java | 70 +
.../StateStoreConnectionMonitorService.java | 67 +
.../federation/store/StateStoreService.java | 450 ++++
.../store/StateStoreUnavailableException.java | 33 +
.../federation/store/StateStoreUtils.java | 109 +
.../store/driver/StateStoreDriver.java | 203 ++
.../driver/StateStoreRecordOperations.java | 148 ++
.../store/driver/StateStoreSerializer.java | 119 +
.../store/driver/impl/StateStoreBaseImpl.java | 86 +
.../driver/impl/StateStoreFileBaseImpl.java | 462 ++++
.../store/driver/impl/StateStoreFileImpl.java | 148 ++
.../driver/impl/StateStoreFileSystemImpl.java | 178 ++
.../driver/impl/StateStoreSerializableImpl.java | 98 +
.../driver/impl/StateStoreSerializerPBImpl.java | 115 +
.../driver/impl/StateStoreZooKeeperImpl.java | 328 +++
.../store/driver/impl/package-info.java | 39 +
.../federation/store/driver/package-info.java | 37 +
.../store/impl/MembershipStoreImpl.java | 310 +++
.../store/impl/MountTableStoreImpl.java | 154 ++
.../federation/store/impl/RouterStoreImpl.java | 93 +
.../federation/store/impl/package-info.java | 31 +
.../server/federation/store/package-info.java | 62 +
.../protocol/AddMountTableEntryRequest.java | 47 +
.../protocol/AddMountTableEntryResponse.java | 42 +
.../store/protocol/EnterSafeModeRequest.java | 32 +
.../store/protocol/EnterSafeModeResponse.java | 50 +
.../protocol/GetMountTableEntriesRequest.java | 49 +
.../protocol/GetMountTableEntriesResponse.java | 53 +
.../GetNamenodeRegistrationsRequest.java | 52 +
.../GetNamenodeRegistrationsResponse.java | 55 +
.../store/protocol/GetNamespaceInfoRequest.java | 30 +
.../protocol/GetNamespaceInfoResponse.java | 52 +
.../protocol/GetRouterRegistrationRequest.java | 47 +
.../protocol/GetRouterRegistrationResponse.java | 44 +
.../protocol/GetRouterRegistrationsRequest.java | 34 +
.../GetRouterRegistrationsResponse.java | 55 +
.../store/protocol/GetSafeModeRequest.java | 31 +
.../store/protocol/GetSafeModeResponse.java | 49 +
.../store/protocol/LeaveSafeModeRequest.java | 32 +
.../store/protocol/LeaveSafeModeResponse.java | 50 +
.../protocol/NamenodeHeartbeatRequest.java | 52 +
.../protocol/NamenodeHeartbeatResponse.java | 49 +
.../protocol/RemoveMountTableEntryRequest.java | 49 +
.../protocol/RemoveMountTableEntryResponse.java | 42 +
.../store/protocol/RouterHeartbeatRequest.java | 50 +
.../store/protocol/RouterHeartbeatResponse.java | 49 +
.../protocol/UpdateMountTableEntryRequest.java | 51 +
.../protocol/UpdateMountTableEntryResponse.java | 43 +
.../UpdateNamenodeRegistrationRequest.java | 72 +
.../UpdateNamenodeRegistrationResponse.java | 51 +
.../pb/AddMountTableEntryRequestPBImpl.java | 84 +
.../pb/AddMountTableEntryResponsePBImpl.java | 76 +
.../impl/pb/EnterSafeModeRequestPBImpl.java | 62 +
.../impl/pb/EnterSafeModeResponsePBImpl.java | 73 +
.../impl/pb/FederationProtocolPBTranslator.java | 145 ++
.../pb/GetMountTableEntriesRequestPBImpl.java | 76 +
.../pb/GetMountTableEntriesResponsePBImpl.java | 104 +
.../GetNamenodeRegistrationsRequestPBImpl.java | 87 +
.../GetNamenodeRegistrationsResponsePBImpl.java | 99 +
.../impl/pb/GetNamespaceInfoRequestPBImpl.java | 60 +
.../impl/pb/GetNamespaceInfoResponsePBImpl.java | 95 +
.../pb/GetRouterRegistrationRequestPBImpl.java | 70 +
.../pb/GetRouterRegistrationResponsePBImpl.java | 79 +
.../pb/GetRouterRegistrationsRequestPBImpl.java | 61 +
.../GetRouterRegistrationsResponsePBImpl.java | 102 +
.../impl/pb/GetSafeModeRequestPBImpl.java | 62 +
.../impl/pb/GetSafeModeResponsePBImpl.java | 73 +
.../impl/pb/LeaveSafeModeRequestPBImpl.java | 62 +
.../impl/pb/LeaveSafeModeResponsePBImpl.java | 73 +
.../impl/pb/NamenodeHeartbeatRequestPBImpl.java | 93 +
.../pb/NamenodeHeartbeatResponsePBImpl.java | 71 +
.../pb/RemoveMountTableEntryRequestPBImpl.java | 76 +
.../pb/RemoveMountTableEntryResponsePBImpl.java | 76 +
.../impl/pb/RouterHeartbeatRequestPBImpl.java | 78 +
.../impl/pb/RouterHeartbeatResponsePBImpl.java | 70 +
.../pb/UpdateMountTableEntryRequestPBImpl.java | 96 +
.../pb/UpdateMountTableEntryResponsePBImpl.java | 76 +
...UpdateNamenodeRegistrationRequestPBImpl.java | 95 +
...pdateNamenodeRegistrationResponsePBImpl.java | 73 +
.../store/protocol/impl/pb/package-info.java | 29 +
.../federation/store/protocol/package-info.java | 31 +
.../federation/store/records/BaseRecord.java | 215 ++
.../store/records/MembershipState.java | 336 +++
.../store/records/MembershipStats.java | 130 +
.../federation/store/records/MountTable.java | 430 ++++
.../server/federation/store/records/Query.java | 66 +
.../federation/store/records/QueryResult.java | 56 +
.../federation/store/records/RouterState.java | 168 ++
.../store/records/StateStoreVersion.java | 91 +
.../records/impl/pb/MembershipStatePBImpl.java | 335 +++
.../records/impl/pb/MembershipStatsPBImpl.java | 201 ++
.../store/records/impl/pb/MountTablePBImpl.java | 317 +++
.../store/records/impl/pb/PBRecord.java | 47 +
.../records/impl/pb/RouterStatePBImpl.java | 202 ++
.../impl/pb/StateStoreVersionPBImpl.java | 79 +
.../store/records/impl/pb/package-info.java | 29 +
.../federation/store/records/package-info.java | 36 +
.../federation/utils/ConsistentHashRing.java | 144 ++
.../server/federation/utils/package-info.java | 28 +
.../hdfs/tools/federation/RouterAdmin.java | 627 +++++
.../hdfs/tools/federation/package-info.java | 28 +
.../src/main/proto/FederationProtocol.proto | 244 ++
.../src/main/proto/RouterProtocol.proto | 62 +
.../src/main/resources/hdfs-rbf-default.xml | 434 ++++
.../src/main/webapps/proto-web.xml | 16 +
.../main/webapps/router/federationhealth.html | 424 ++++
.../src/main/webapps/router/federationhealth.js | 381 +++
.../src/main/webapps/router/index.html | 24 +
.../src/main/webapps/router/robots.txt | 2 +
.../src/site/markdown/HDFSRouterFederation.md | 373 +++
.../src/site/resources/css/site.css | 30 +
.../site/resources/images/routerfederation.png | Bin 0 -> 24961 bytes
.../hadoop-hdfs-rbf/src/site/site.xml | 28 +
.../server/federation/FederationTestUtils.java | 246 ++
.../hdfs/server/federation/MockResolver.java | 313 +++
.../server/federation/RouterConfigBuilder.java | 163 ++
.../server/federation/RouterDFSCluster.java | 1005 ++++++++
.../server/federation/StateStoreDFSCluster.java | 148 ++
.../metrics/TestFederationMetrics.java | 293 +++
.../federation/metrics/TestMetricsBase.java | 196 ++
.../resolver/TestMountTableResolver.java | 528 +++++
.../TestMultipleDestinationResolver.java | 419 ++++
.../resolver/TestNamenodeResolver.java | 284 +++
.../resolver/order/TestLocalResolver.java | 143 ++
.../router/TestConnectionManager.java | 157 ++
.../router/TestNamenodeHeartbeat.java | 168 ++
.../federation/router/TestRBFConfigFields.java | 51 +
.../server/federation/router/TestRouter.java | 150 ++
.../federation/router/TestRouterAdmin.java | 340 +++
.../federation/router/TestRouterAdminCLI.java | 477 ++++
.../router/TestRouterHeartbeatService.java | 146 ++
.../federation/router/TestRouterMountTable.java | 143 ++
.../router/TestRouterNamenodeMonitoring.java | 143 ++
.../federation/router/TestRouterQuota.java | 451 ++++
.../router/TestRouterQuotaManager.java | 125 +
.../router/TestRouterRPCClientRetries.java | 160 ++
.../server/federation/router/TestRouterRpc.java | 1072 +++++++++
.../router/TestRouterRpcMultiDestination.java | 216 ++
.../federation/router/TestRouterSafemode.java | 200 ++
.../store/FederationStateStoreTestUtils.java | 274 +++
.../federation/store/TestStateStoreBase.java | 81 +
.../store/TestStateStoreMembershipState.java | 463 ++++
.../store/TestStateStoreMountTable.java | 250 ++
.../store/TestStateStoreRouterState.java | 195 ++
.../store/driver/TestStateStoreDriverBase.java | 613 +++++
.../store/driver/TestStateStoreFile.java | 76 +
.../store/driver/TestStateStoreFileBase.java | 47 +
.../store/driver/TestStateStoreFileSystem.java | 94 +
.../store/driver/TestStateStoreZK.java | 105 +
.../store/records/TestMembershipState.java | 129 +
.../store/records/TestMountTable.java | 259 ++
.../store/records/TestRouterState.java | 85 +
.../src/test/resources/hdfs-site.xml | 29 +
.../src/test/resources/log4j.properties | 23 +
.../dev-support/findbugsExcludeFile.xml | 3 -
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 6 -
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 204 --
.../hdfs/protocolPB/RouterAdminProtocolPB.java | 44 -
...uterAdminProtocolServerSideTranslatorPB.java | 211 --
.../RouterAdminProtocolTranslatorPB.java | 208 --
.../federation/metrics/FederationMBean.java | 223 --
.../federation/metrics/FederationMetrics.java | 766 ------
.../federation/metrics/FederationRPCMBean.java | 92 -
.../metrics/FederationRPCMetrics.java | 250 --
.../FederationRPCPerformanceMonitor.java | 221 --
.../federation/metrics/NamenodeBeanMetrics.java | 634 -----
.../federation/metrics/StateStoreMBean.java | 45 -
.../federation/metrics/StateStoreMetrics.java | 149 --
.../server/federation/metrics/package-info.java | 27 -
.../resolver/ActiveNamenodeResolver.java | 118 -
.../resolver/FederationNamenodeContext.java | 87 -
.../FederationNamenodeServiceState.java | 46 -
.../resolver/FederationNamespaceInfo.java | 73 -
.../resolver/FileSubclusterResolver.java | 75 -
.../resolver/MembershipNamenodeResolver.java | 317 ---
.../federation/resolver/MountTableManager.java | 80 -
.../federation/resolver/MountTableResolver.java | 597 -----
.../MultipleDestinationMountTableResolver.java | 116 -
.../resolver/NamenodePriorityComparator.java | 63 -
.../resolver/NamenodeStatusReport.java | 406 ----
.../federation/resolver/PathLocation.java | 212 --
.../federation/resolver/RemoteLocation.java | 76 -
.../resolver/order/DestinationOrder.java | 30 -
.../resolver/order/HashFirstResolver.java | 71 -
.../federation/resolver/order/HashResolver.java | 137 --
.../resolver/order/LocalResolver.java | 297 ---
.../resolver/order/OrderedResolver.java | 37 -
.../resolver/order/RandomResolver.java | 62 -
.../federation/resolver/order/package-info.java | 29 -
.../federation/resolver/package-info.java | 41 -
.../federation/router/ConnectionContext.java | 104 -
.../federation/router/ConnectionManager.java | 436 ----
.../federation/router/ConnectionPool.java | 337 ---
.../federation/router/ConnectionPoolId.java | 123 -
.../server/federation/router/DFSRouter.java | 76 -
.../server/federation/router/ErasureCoding.java | 198 --
.../federation/router/FederationUtil.java | 209 --
.../router/NamenodeHeartbeatService.java | 362 ---
.../federation/router/PeriodicService.java | 198 --
.../hdfs/server/federation/router/Quota.java | 208 --
.../router/RemoteLocationContext.java | 68 -
.../server/federation/router/RemoteMethod.java | 164 --
.../server/federation/router/RemoteParam.java | 71 -
.../hdfs/server/federation/router/Router.java | 656 ------
.../federation/router/RouterAdminServer.java | 298 ---
.../server/federation/router/RouterClient.java | 80 -
.../router/RouterHeartbeatService.java | 167 --
.../federation/router/RouterHttpServer.java | 124 -
.../server/federation/router/RouterMetrics.java | 73 -
.../federation/router/RouterMetricsService.java | 108 -
.../router/RouterPermissionChecker.java | 82 -
.../federation/router/RouterQuotaManager.java | 172 --
.../router/RouterQuotaUpdateService.java | 228 --
.../federation/router/RouterQuotaUsage.java | 119 -
.../federation/router/RouterRpcClient.java | 1022 --------
.../federation/router/RouterRpcMonitor.java | 108 -
.../federation/router/RouterRpcServer.java | 2214 ------------------
.../router/RouterSafeModeException.java | 53 -
.../router/RouterSafemodeService.java | 150 --
.../federation/router/RouterServiceState.java | 31 -
.../federation/router/RouterStateManager.java | 50 -
.../server/federation/router/package-info.java | 31 -
.../federation/store/CachedRecordStore.java | 242 --
.../federation/store/MembershipStore.java | 126 -
.../federation/store/MountTableStore.java | 49 -
.../server/federation/store/RecordStore.java | 100 -
.../server/federation/store/RouterStore.java | 78 -
.../federation/store/StateStoreCache.java | 36 -
.../store/StateStoreCacheUpdateService.java | 70 -
.../StateStoreConnectionMonitorService.java | 67 -
.../federation/store/StateStoreService.java | 450 ----
.../store/StateStoreUnavailableException.java | 33 -
.../federation/store/StateStoreUtils.java | 109 -
.../store/driver/StateStoreDriver.java | 203 --
.../driver/StateStoreRecordOperations.java | 148 --
.../store/driver/StateStoreSerializer.java | 119 -
.../store/driver/impl/StateStoreBaseImpl.java | 86 -
.../driver/impl/StateStoreFileBaseImpl.java | 462 ----
.../store/driver/impl/StateStoreFileImpl.java | 148 --
.../driver/impl/StateStoreFileSystemImpl.java | 178 --
.../driver/impl/StateStoreSerializableImpl.java | 98 -
.../driver/impl/StateStoreSerializerPBImpl.java | 115 -
.../driver/impl/StateStoreZooKeeperImpl.java | 328 ---
.../store/driver/impl/package-info.java | 39 -
.../federation/store/driver/package-info.java | 37 -
.../store/impl/MembershipStoreImpl.java | 310 ---
.../store/impl/MountTableStoreImpl.java | 154 --
.../federation/store/impl/RouterStoreImpl.java | 93 -
.../federation/store/impl/package-info.java | 31 -
.../server/federation/store/package-info.java | 62 -
.../protocol/AddMountTableEntryRequest.java | 47 -
.../protocol/AddMountTableEntryResponse.java | 42 -
.../store/protocol/EnterSafeModeRequest.java | 32 -
.../store/protocol/EnterSafeModeResponse.java | 50 -
.../protocol/GetMountTableEntriesRequest.java | 49 -
.../protocol/GetMountTableEntriesResponse.java | 53 -
.../GetNamenodeRegistrationsRequest.java | 52 -
.../GetNamenodeRegistrationsResponse.java | 55 -
.../store/protocol/GetNamespaceInfoRequest.java | 30 -
.../protocol/GetNamespaceInfoResponse.java | 52 -
.../protocol/GetRouterRegistrationRequest.java | 47 -
.../protocol/GetRouterRegistrationResponse.java | 44 -
.../protocol/GetRouterRegistrationsRequest.java | 34 -
.../GetRouterRegistrationsResponse.java | 55 -
.../store/protocol/GetSafeModeRequest.java | 31 -
.../store/protocol/GetSafeModeResponse.java | 49 -
.../store/protocol/LeaveSafeModeRequest.java | 32 -
.../store/protocol/LeaveSafeModeResponse.java | 50 -
.../protocol/NamenodeHeartbeatRequest.java | 52 -
.../protocol/NamenodeHeartbeatResponse.java | 49 -
.../protocol/RemoveMountTableEntryRequest.java | 49 -
.../protocol/RemoveMountTableEntryResponse.java | 42 -
.../store/protocol/RouterHeartbeatRequest.java | 50 -
.../store/protocol/RouterHeartbeatResponse.java | 49 -
.../protocol/UpdateMountTableEntryRequest.java | 51 -
.../protocol/UpdateMountTableEntryResponse.java | 43 -
.../UpdateNamenodeRegistrationRequest.java | 72 -
.../UpdateNamenodeRegistrationResponse.java | 51 -
.../pb/AddMountTableEntryRequestPBImpl.java | 84 -
.../pb/AddMountTableEntryResponsePBImpl.java | 76 -
.../impl/pb/EnterSafeModeRequestPBImpl.java | 62 -
.../impl/pb/EnterSafeModeResponsePBImpl.java | 73 -
.../impl/pb/FederationProtocolPBTranslator.java | 145 --
.../pb/GetMountTableEntriesRequestPBImpl.java | 76 -
.../pb/GetMountTableEntriesResponsePBImpl.java | 104 -
.../GetNamenodeRegistrationsRequestPBImpl.java | 87 -
.../GetNamenodeRegistrationsResponsePBImpl.java | 99 -
.../impl/pb/GetNamespaceInfoRequestPBImpl.java | 60 -
.../impl/pb/GetNamespaceInfoResponsePBImpl.java | 95 -
.../pb/GetRouterRegistrationRequestPBImpl.java | 70 -
.../pb/GetRouterRegistrationResponsePBImpl.java | 79 -
.../pb/GetRouterRegistrationsRequestPBImpl.java | 61 -
.../GetRouterRegistrationsResponsePBImpl.java | 102 -
.../impl/pb/GetSafeModeRequestPBImpl.java | 62 -
.../impl/pb/GetSafeModeResponsePBImpl.java | 73 -
.../impl/pb/LeaveSafeModeRequestPBImpl.java | 62 -
.../impl/pb/LeaveSafeModeResponsePBImpl.java | 73 -
.../impl/pb/NamenodeHeartbeatRequestPBImpl.java | 93 -
.../pb/NamenodeHeartbeatResponsePBImpl.java | 71 -
.../pb/RemoveMountTableEntryRequestPBImpl.java | 76 -
.../pb/RemoveMountTableEntryResponsePBImpl.java | 76 -
.../impl/pb/RouterHeartbeatRequestPBImpl.java | 78 -
.../impl/pb/RouterHeartbeatResponsePBImpl.java | 70 -
.../pb/UpdateMountTableEntryRequestPBImpl.java | 96 -
.../pb/UpdateMountTableEntryResponsePBImpl.java | 76 -
...UpdateNamenodeRegistrationRequestPBImpl.java | 95 -
...pdateNamenodeRegistrationResponsePBImpl.java | 73 -
.../store/protocol/impl/pb/package-info.java | 29 -
.../federation/store/protocol/package-info.java | 31 -
.../federation/store/records/BaseRecord.java | 215 --
.../store/records/MembershipState.java | 336 ---
.../store/records/MembershipStats.java | 130 -
.../federation/store/records/MountTable.java | 430 ----
.../server/federation/store/records/Query.java | 66 -
.../federation/store/records/QueryResult.java | 56 -
.../federation/store/records/RouterState.java | 168 --
.../store/records/StateStoreVersion.java | 91 -
.../records/impl/pb/MembershipStatePBImpl.java | 335 ---
.../records/impl/pb/MembershipStatsPBImpl.java | 201 --
.../store/records/impl/pb/MountTablePBImpl.java | 317 ---
.../store/records/impl/pb/PBRecord.java | 47 -
.../records/impl/pb/RouterStatePBImpl.java | 202 --
.../impl/pb/StateStoreVersionPBImpl.java | 79 -
.../store/records/impl/pb/package-info.java | 29 -
.../federation/store/records/package-info.java | 36 -
.../federation/utils/ConsistentHashRing.java | 144 --
.../server/federation/utils/package-info.java | 28 -
.../hdfs/tools/federation/RouterAdmin.java | 627 -----
.../hdfs/tools/federation/package-info.java | 28 -
.../src/main/proto/FederationProtocol.proto | 244 --
.../src/main/proto/RouterProtocol.proto | 62 -
.../src/main/resources/hdfs-default.xml | 408 ----
.../main/webapps/router/federationhealth.html | 424 ----
.../src/main/webapps/router/federationhealth.js | 381 ---
.../src/main/webapps/router/index.html | 24 -
.../src/main/webapps/router/robots.txt | 2 -
.../src/site/markdown/HDFSRouterFederation.md | 373 ---
.../site/resources/images/routerfederation.png | Bin 24961 -> 0 bytes
.../server/federation/FederationTestUtils.java | 246 --
.../hdfs/server/federation/MockResolver.java | 313 ---
.../server/federation/RouterConfigBuilder.java | 163 --
.../server/federation/RouterDFSCluster.java | 1005 --------
.../server/federation/StateStoreDFSCluster.java | 148 --
.../metrics/TestFederationMetrics.java | 293 ---
.../federation/metrics/TestMetricsBase.java | 196 --
.../resolver/TestMountTableResolver.java | 528 -----
.../TestMultipleDestinationResolver.java | 419 ----
.../resolver/TestNamenodeResolver.java | 284 ---
.../resolver/order/TestLocalResolver.java | 143 --
.../router/TestConnectionManager.java | 157 --
.../router/TestNamenodeHeartbeat.java | 168 --
.../server/federation/router/TestRouter.java | 150 --
.../federation/router/TestRouterAdmin.java | 340 ---
.../federation/router/TestRouterAdminCLI.java | 478 ----
.../router/TestRouterHeartbeatService.java | 147 --
.../federation/router/TestRouterMountTable.java | 143 --
.../router/TestRouterNamenodeMonitoring.java | 143 --
.../federation/router/TestRouterQuota.java | 452 ----
.../router/TestRouterQuotaManager.java | 125 -
.../router/TestRouterRPCClientRetries.java | 160 --
.../server/federation/router/TestRouterRpc.java | 1072 ---------
.../router/TestRouterRpcMultiDestination.java | 216 --
.../federation/router/TestRouterSafemode.java | 201 --
.../store/FederationStateStoreTestUtils.java | 274 ---
.../federation/store/TestStateStoreBase.java | 81 -
.../store/TestStateStoreMembershipState.java | 463 ----
.../store/TestStateStoreMountTable.java | 250 --
.../store/TestStateStoreRouterState.java | 195 --
.../store/driver/TestStateStoreDriverBase.java | 613 -----
.../store/driver/TestStateStoreFile.java | 76 -
.../store/driver/TestStateStoreFileBase.java | 47 -
.../store/driver/TestStateStoreFileSystem.java | 94 -
.../store/driver/TestStateStoreZK.java | 105 -
.../store/records/TestMembershipState.java | 129 -
.../store/records/TestMountTable.java | 259 --
.../store/records/TestRouterState.java | 85 -
hadoop-hdfs-project/pom.xml | 1 +
hadoop-project/pom.xml | 5 +
hadoop-project/src/site/site.xml | 3 +-
455 files changed, 38400 insertions(+), 37908 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/dev-support/bin/dist-layout-stitching
----------------------------------------------------------------------
diff --git a/dev-support/bin/dist-layout-stitching b/dev-support/bin/dist-layout-stitching
index fdbc83c..f3db542 100755
--- a/dev-support/bin/dist-layout-stitching
+++ b/dev-support/bin/dist-layout-stitching
@@ -129,6 +129,7 @@ run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${VERSION}"
run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-nfs/target/hadoop-hdfs-nfs-${VERSION}" .
run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-${VERSION}" .
run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-native-client/target/hadoop-hdfs-native-client-${VERSION}" .
+run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-rbf/target/hadoop-hdfs-rbf-${VERSION}" .
run copy "${ROOT}/hadoop-yarn-project/target/hadoop-yarn-project-${VERSION}" .
run copy "${ROOT}/hadoop-mapreduce-project/target/hadoop-mapreduce-${VERSION}" .
run copy "${ROOT}/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${VERSION}" .
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/dev-support/findbugsExcludeFile.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/dev-support/findbugsExcludeFile.xml
new file mode 100644
index 0000000..40771bf
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/dev-support/findbugsExcludeFile.xml
@@ -0,0 +1,22 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<FindBugsFilter>
+ <Match>
+ <Package name="org.apache.hadoop.hdfs.federation.protocol.proto" />
+ </Match>
+</FindBugsFilter>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
new file mode 100644
index 0000000..9683744
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
@@ -0,0 +1,236 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-project-dist</artifactId>
+ <version>3.2.0-SNAPSHOT</version>
+ <relativePath>../../hadoop-project-dist</relativePath>
+ </parent>
+ <artifactId>hadoop-hdfs-rbf</artifactId>
+ <version>3.2.0-SNAPSHOT</version>
+ <description>Apache Hadoop HDFS-RBF</description>
+ <name>Apache Hadoop HDFS-RBF</name>
+ <packaging>jar</packaging>
+
+ <properties>
+ <hadoop.component>hdfs</hadoop.component>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-util-ajax</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <configuration>
+ <skipTests>false</skipTests>
+ </configuration>
+ <executions>
+ <execution>
+ <id>create-web-xmls</id>
+ <phase>compile</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <copy file="${basedir}/src/main/webapps/proto-web.xml"
+ tofile="${project.build.directory}/webapps/router/WEB-INF/web.xml"
+ filtering="true"/>
+ <copy toDir="${project.build.directory}/webapps">
+ <fileset dir="${basedir}/src/main/webapps">
+ <exclude name="**/proto-web.xml"/>
+ </fileset>
+ </copy>
+ <replace dir="${project.build.directory}/webapps" value="${release-year}">
+ <include name="**/*.html"/>
+ <replacetoken>{release-year-token}</replacetoken>
+ </replace>
+ </target>
+ </configuration>
+ </execution>
+ <execution>
+ <id>create-log-dir</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <copy todir="${project.build.directory}/test-classes/webapps">
+ <fileset dir="${project.build.directory}/webapps">
+ <exclude name="proto-*-web.xml"/>
+ <exclude name="**/proto-web.xml"/>
+ </fileset>
+ </copy>
+ </target>
+ </configuration>
+ </execution>
+ <execution>
+ <phase>pre-site</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <tasks>
+ <copy file="src/main/resources/hdfs-rbf-default.xml" todir="src/site/resources"/>
+ </tasks>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-maven-plugins</artifactId>
+ <executions>
+ <execution>
+ <id>compile-protoc</id>
+ <goals>
+ <goal>protoc</goal>
+ </goals>
+ <configuration>
+ <protocVersion>${protobuf.version}</protocVersion>
+ <protocCommand>${protoc.path}</protocCommand>
+ <imports>
+ <param>${basedir}/../hadoop-hdfs-client/src/main/proto</param>
+ <param>${basedir}/../../hadoop-common-project/hadoop-common/src/main/proto</param>
+ <param>${basedir}/src/main/proto</param>
+ </imports>
+ <source>
+ <directory>${basedir}/src/main/proto</directory>
+ <includes>
+ <include>FederationProtocol.proto</include>
+ <include>RouterProtocol.proto</include>
+ </includes>
+ </source>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>.gitattributes</exclude>
+ <exclude>.idea/**</exclude>
+ <exclude>dev-support/findbugsExcludeFile.xml</exclude>
+ <exclude>src/main/webapps/router/robots.txt</exclude>
+ <exclude>src/site/resources/images/*</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-clean-plugin</artifactId>
+ <configuration>
+ <filesets>
+ <fileset>
+ <directory>src/site/resources</directory>
+ <includes>
+ <include>hdfs-rbf-default.xml</include>
+ </includes>
+ <followSymlinks>false</followSymlinks>
+ </fileset>
+ </filesets>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolPB.java
new file mode 100644
index 0000000..96fa794
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolPB.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocolPB;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.proto.RouterProtocolProtos.RouterAdminProtocolService;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector;
+import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.security.KerberosInfo;
+import org.apache.hadoop.security.token.TokenInfo;
+
+/**
+ * Protocol that a clients use to communicate with the NameNode.
+ * Note: This extends the protocolbuffer service based interface to
+ * add annotations required for security.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Stable
+@KerberosInfo(
+ serverPrincipal = DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY)
+@TokenInfo(DelegationTokenSelector.class)
+@ProtocolInfo(protocolName = HdfsConstants.CLIENT_NAMENODE_PROTOCOL_NAME,
+ protocolVersion = 1)
+public interface RouterAdminProtocolPB extends
+ RouterAdminProtocolService.BlockingInterface {
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java
new file mode 100644
index 0000000..159d5c2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java
@@ -0,0 +1,211 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocolPB;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.EnterSafeModeRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.EnterSafeModeResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetSafeModeRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetSafeModeResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.LeaveSafeModeRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.LeaveSafeModeResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryResponseProto;
+import org.apache.hadoop.hdfs.server.federation.router.RouterAdminServer;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.AddMountTableEntryRequestPBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.AddMountTableEntryResponsePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.EnterSafeModeRequestPBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.EnterSafeModeResponsePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.GetMountTableEntriesRequestPBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.GetMountTableEntriesResponsePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.GetSafeModeRequestPBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.GetSafeModeResponsePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.LeaveSafeModeRequestPBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.LeaveSafeModeResponsePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryRequestPBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryResponsePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.UpdateMountTableEntryRequestPBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.UpdateMountTableEntryResponsePBImpl;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+
+/**
+ * This class is used on the server side. Calls come across the wire for the for
+ * protocol {@link RouterAdminProtocolPB}. This class translates the PB data
+ * types to the native data types used inside the HDFS Router as specified in
+ * the generic RouterAdminProtocol.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Stable
+public class RouterAdminProtocolServerSideTranslatorPB implements
+ RouterAdminProtocolPB {
+
+ private final RouterAdminServer server;
+
+ /**
+ * Constructor.
+ * @param server The NN server.
+ * @throws IOException
+ */
+ public RouterAdminProtocolServerSideTranslatorPB(RouterAdminServer server)
+ throws IOException {
+ this.server = server;
+ }
+
+ @Override
+ public AddMountTableEntryResponseProto addMountTableEntry(
+ RpcController controller, AddMountTableEntryRequestProto request)
+ throws ServiceException {
+
+ try {
+ AddMountTableEntryRequest req =
+ new AddMountTableEntryRequestPBImpl(request);
+ AddMountTableEntryResponse response = server.addMountTableEntry(req);
+ AddMountTableEntryResponsePBImpl responsePB =
+ (AddMountTableEntryResponsePBImpl)response;
+ return responsePB.getProto();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ /**
+ * Remove an entry from the mount table.
+ */
+ @Override
+ public RemoveMountTableEntryResponseProto removeMountTableEntry(
+ RpcController controller, RemoveMountTableEntryRequestProto request)
+ throws ServiceException {
+ try {
+ RemoveMountTableEntryRequest req =
+ new RemoveMountTableEntryRequestPBImpl(request);
+ RemoveMountTableEntryResponse response =
+ server.removeMountTableEntry(req);
+ RemoveMountTableEntryResponsePBImpl responsePB =
+ (RemoveMountTableEntryResponsePBImpl)response;
+ return responsePB.getProto();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ /**
+ * Get matching mount table entries.
+ */
+ @Override
+ public GetMountTableEntriesResponseProto getMountTableEntries(
+ RpcController controller, GetMountTableEntriesRequestProto request)
+ throws ServiceException {
+ try {
+ GetMountTableEntriesRequest req =
+ new GetMountTableEntriesRequestPBImpl(request);
+ GetMountTableEntriesResponse response = server.getMountTableEntries(req);
+ GetMountTableEntriesResponsePBImpl responsePB =
+ (GetMountTableEntriesResponsePBImpl)response;
+ return responsePB.getProto();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ /**
+ * Update a single mount table entry.
+ */
+ @Override
+ public UpdateMountTableEntryResponseProto updateMountTableEntry(
+ RpcController controller, UpdateMountTableEntryRequestProto request)
+ throws ServiceException {
+ try {
+ UpdateMountTableEntryRequest req =
+ new UpdateMountTableEntryRequestPBImpl(request);
+ UpdateMountTableEntryResponse response =
+ server.updateMountTableEntry(req);
+ UpdateMountTableEntryResponsePBImpl responsePB =
+ (UpdateMountTableEntryResponsePBImpl)response;
+ return responsePB.getProto();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public EnterSafeModeResponseProto enterSafeMode(RpcController controller,
+ EnterSafeModeRequestProto request) throws ServiceException {
+ try {
+ EnterSafeModeRequest req = new EnterSafeModeRequestPBImpl(request);
+ EnterSafeModeResponse response = server.enterSafeMode(req);
+ EnterSafeModeResponsePBImpl responsePB =
+ (EnterSafeModeResponsePBImpl) response;
+ return responsePB.getProto();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public LeaveSafeModeResponseProto leaveSafeMode(RpcController controller,
+ LeaveSafeModeRequestProto request) throws ServiceException {
+ try {
+ LeaveSafeModeRequest req = new LeaveSafeModeRequestPBImpl(request);
+ LeaveSafeModeResponse response = server.leaveSafeMode(req);
+ LeaveSafeModeResponsePBImpl responsePB =
+ (LeaveSafeModeResponsePBImpl) response;
+ return responsePB.getProto();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public GetSafeModeResponseProto getSafeMode(RpcController controller,
+ GetSafeModeRequestProto request) throws ServiceException {
+ try {
+ GetSafeModeRequest req = new GetSafeModeRequestPBImpl(request);
+ GetSafeModeResponse response = server.getSafeMode(req);
+ GetSafeModeResponsePBImpl responsePB =
+ (GetSafeModeResponsePBImpl) response;
+ return responsePB.getProto();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java
new file mode 100644
index 0000000..d6210ce
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java
@@ -0,0 +1,208 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocolPB;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.EnterSafeModeRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.EnterSafeModeResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetSafeModeRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetSafeModeResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.LeaveSafeModeRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.LeaveSafeModeResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryResponseProto;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
+import org.apache.hadoop.hdfs.server.federation.router.RouterStateManager;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.AddMountTableEntryRequestPBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.AddMountTableEntryResponsePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.EnterSafeModeResponsePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.GetMountTableEntriesRequestPBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.GetMountTableEntriesResponsePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.GetSafeModeResponsePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.LeaveSafeModeResponsePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryRequestPBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryResponsePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.UpdateMountTableEntryRequestPBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.UpdateMountTableEntryResponsePBImpl;
+import org.apache.hadoop.ipc.ProtobufHelper;
+import org.apache.hadoop.ipc.ProtocolMetaInterface;
+import org.apache.hadoop.ipc.ProtocolTranslator;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.RpcClientUtil;
+
+import com.google.protobuf.ServiceException;
+
+/**
+ * This class forwards NN's ClientProtocol calls as RPC calls to the NN server
+ * while translating from the parameter types used in ClientProtocol to the
+ * new PB types.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Stable
+public class RouterAdminProtocolTranslatorPB
+ implements ProtocolMetaInterface, MountTableManager,
+ Closeable, ProtocolTranslator, RouterStateManager {
+ final private RouterAdminProtocolPB rpcProxy;
+
+ public RouterAdminProtocolTranslatorPB(RouterAdminProtocolPB proxy) {
+ rpcProxy = proxy;
+ }
+
+ @Override
+ public void close() {
+ RPC.stopProxy(rpcProxy);
+ }
+
+ @Override
+ public Object getUnderlyingProxyObject() {
+ return rpcProxy;
+ }
+
+ @Override
+ public boolean isMethodSupported(String methodName) throws IOException {
+ return RpcClientUtil.isMethodSupported(rpcProxy,
+ RouterAdminProtocolPB.class, RPC.RpcKind.RPC_PROTOCOL_BUFFER,
+ RPC.getProtocolVersion(RouterAdminProtocolPB.class), methodName);
+ }
+
+ @Override
+ public AddMountTableEntryResponse addMountTableEntry(
+ AddMountTableEntryRequest request) throws IOException {
+ AddMountTableEntryRequestPBImpl requestPB =
+ (AddMountTableEntryRequestPBImpl)request;
+ AddMountTableEntryRequestProto proto = requestPB.getProto();
+ try {
+ AddMountTableEntryResponseProto response =
+ rpcProxy.addMountTableEntry(null, proto);
+ return new AddMountTableEntryResponsePBImpl(response);
+ } catch (ServiceException e) {
+ throw new IOException(ProtobufHelper.getRemoteException(e).getMessage());
+ }
+ }
+
+ @Override
+ public UpdateMountTableEntryResponse updateMountTableEntry(
+ UpdateMountTableEntryRequest request) throws IOException {
+ UpdateMountTableEntryRequestPBImpl requestPB =
+ (UpdateMountTableEntryRequestPBImpl)request;
+ UpdateMountTableEntryRequestProto proto = requestPB.getProto();
+ try {
+ UpdateMountTableEntryResponseProto response =
+ rpcProxy.updateMountTableEntry(null, proto);
+ return new UpdateMountTableEntryResponsePBImpl(response);
+ } catch (ServiceException e) {
+ throw new IOException(ProtobufHelper.getRemoteException(e).getMessage());
+ }
+ }
+
+ @Override
+ public RemoveMountTableEntryResponse removeMountTableEntry(
+ RemoveMountTableEntryRequest request) throws IOException {
+ RemoveMountTableEntryRequestPBImpl requestPB =
+ (RemoveMountTableEntryRequestPBImpl)request;
+ RemoveMountTableEntryRequestProto proto = requestPB.getProto();
+ try {
+ RemoveMountTableEntryResponseProto responseProto =
+ rpcProxy.removeMountTableEntry(null, proto);
+ return new RemoveMountTableEntryResponsePBImpl(responseProto);
+ } catch (ServiceException e) {
+ throw new IOException(ProtobufHelper.getRemoteException(e).getMessage());
+ }
+ }
+
+ @Override
+ public GetMountTableEntriesResponse getMountTableEntries(
+ GetMountTableEntriesRequest request) throws IOException {
+ GetMountTableEntriesRequestPBImpl requestPB =
+ (GetMountTableEntriesRequestPBImpl)request;
+ GetMountTableEntriesRequestProto proto = requestPB.getProto();
+ try {
+ GetMountTableEntriesResponseProto response =
+ rpcProxy.getMountTableEntries(null, proto);
+ return new GetMountTableEntriesResponsePBImpl(response);
+ } catch (ServiceException e) {
+ throw new IOException(ProtobufHelper.getRemoteException(e).getMessage());
+ }
+ }
+
+ @Override
+ public EnterSafeModeResponse enterSafeMode(EnterSafeModeRequest request)
+ throws IOException {
+ EnterSafeModeRequestProto proto =
+ EnterSafeModeRequestProto.newBuilder().build();
+ try {
+ EnterSafeModeResponseProto response =
+ rpcProxy.enterSafeMode(null, proto);
+ return new EnterSafeModeResponsePBImpl(response);
+ } catch (ServiceException e) {
+ throw new IOException(ProtobufHelper.getRemoteException(e).getMessage());
+ }
+ }
+
+ @Override
+ public LeaveSafeModeResponse leaveSafeMode(LeaveSafeModeRequest request)
+ throws IOException {
+ LeaveSafeModeRequestProto proto =
+ LeaveSafeModeRequestProto.newBuilder().build();
+ try {
+ LeaveSafeModeResponseProto response =
+ rpcProxy.leaveSafeMode(null, proto);
+ return new LeaveSafeModeResponsePBImpl(response);
+ } catch (ServiceException e) {
+ throw new IOException(ProtobufHelper.getRemoteException(e).getMessage());
+ }
+ }
+
+ @Override
+ public GetSafeModeResponse getSafeMode(GetSafeModeRequest request)
+ throws IOException {
+ GetSafeModeRequestProto proto =
+ GetSafeModeRequestProto.newBuilder().build();
+ try {
+ GetSafeModeResponseProto response =
+ rpcProxy.getSafeMode(null, proto);
+ return new GetSafeModeResponsePBImpl(response);
+ } catch (ServiceException e) {
+ throw new IOException(ProtobufHelper.getRemoteException(e).getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/package-info.java
new file mode 100644
index 0000000..6233024
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocolPB;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java
new file mode 100644
index 0000000..79fb3e4
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java
@@ -0,0 +1,223 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.metrics;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * JMX interface for the federation statistics.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface FederationMBean {
+
+ /**
+ * Get information about all the namenodes in the federation or null if
+ * failure.
+ * @return JSON with all the Namenodes.
+ */
+ String getNamenodes();
+
+ /**
+ * Get the latest info for each registered nameservice.
+ * @return JSON with all the nameservices.
+ */
+ String getNameservices();
+
+ /**
+ * Get the mount table for the federated filesystem or null if failure.
+ * @return JSON with the mount table.
+ */
+ String getMountTable();
+
+ /**
+ * Get the latest state of all routers.
+ * @return JSON with all of the known routers or null if failure.
+ */
+ String getRouters();
+
+ /**
+ * Get the total capacity of the federated cluster.
+ * @return Total capacity of the federated cluster.
+ */
+ long getTotalCapacity();
+
+ /**
+ * Get the used capacity of the federated cluster.
+ * @return Used capacity of the federated cluster.
+ */
+ long getUsedCapacity();
+
+ /**
+ * Get the remaining capacity of the federated cluster.
+ * @return Remaining capacity of the federated cluster.
+ */
+ long getRemainingCapacity();
+
+ /**
+ * Get the total remote storage capacity mounted in the federated cluster.
+ * @return Remote capacity of the federated cluster.
+ */
+ long getProvidedSpace();
+
+ /**
+ * Get the number of nameservices in the federation.
+ * @return Number of nameservices in the federation.
+ */
+ int getNumNameservices();
+
+ /**
+ * Get the number of namenodes.
+ * @return Number of namenodes.
+ */
+ int getNumNamenodes();
+
+ /**
+ * Get the number of expired namenodes.
+ * @return Number of expired namenodes.
+ */
+ int getNumExpiredNamenodes();
+
+ /**
+ * Get the number of live datanodes.
+ * @return Number of live datanodes.
+ */
+ int getNumLiveNodes();
+
+ /**
+ * Get the number of dead datanodes.
+ * @return Number of dead datanodes.
+ */
+ int getNumDeadNodes();
+
+ /**
+ * Get the number of decommissioning datanodes.
+ * @return Number of decommissioning datanodes.
+ */
+ int getNumDecommissioningNodes();
+
+ /**
+ * Get the number of live decommissioned datanodes.
+ * @return Number of live decommissioned datanodes.
+ */
+ int getNumDecomLiveNodes();
+
+ /**
+ * Get the number of dead decommissioned datanodes.
+ * @return Number of dead decommissioned datanodes.
+ */
+ int getNumDecomDeadNodes();
+
+ /**
+ * Get Max, Median, Min and Standard Deviation of DataNodes usage.
+ * @return the DataNode usage information, as a JSON string.
+ */
+ String getNodeUsage();
+
+ /**
+ * Get the number of blocks in the federation.
+ * @return Number of blocks in the federation.
+ */
+ long getNumBlocks();
+
+ /**
+ * Get the number of missing blocks in the federation.
+ * @return Number of missing blocks in the federation.
+ */
+ long getNumOfMissingBlocks();
+
+ /**
+ * Get the number of pending replication blocks in the federation.
+ * @return Number of pending replication blocks in the federation.
+ */
+ long getNumOfBlocksPendingReplication();
+
+ /**
+ * Get the number of under replicated blocks in the federation.
+ * @return Number of under replicated blocks in the federation.
+ */
+ long getNumOfBlocksUnderReplicated();
+
+ /**
+ * Get the number of pending deletion blocks in the federation.
+ * @return Number of pending deletion blocks in the federation.
+ */
+ long getNumOfBlocksPendingDeletion();
+
+ /**
+ * Get the number of files in the federation.
+ * @return Number of files in the federation.
+ */
+ long getNumFiles();
+
+ /**
+ * When the router started.
+ * @return Date as a string the router started.
+ */
+ String getRouterStarted();
+
+ /**
+ * Get the version of the router.
+ * @return Version of the router.
+ */
+ String getVersion();
+
+ /**
+ * Get the compilation date of the router.
+ * @return Compilation date of the router.
+ */
+ String getCompiledDate();
+
+ /**
+ * Get the compilation info of the router.
+ * @return Compilation info of the router.
+ */
+ String getCompileInfo();
+
+ /**
+ * Get the host and port of the router.
+ * @return Host and port of the router.
+ */
+ String getHostAndPort();
+
+ /**
+ * Get the identifier of the router.
+ * @return Identifier of the router.
+ */
+ String getRouterId();
+
+ /**
+ * Get the host and port of the router.
+ * @return Host and port of the router.
+ */
+ String getClusterId();
+
+ /**
+ * Get the host and port of the router.
+ * @return Host and port of the router.
+ */
+ String getBlockPoolId();
+
+ /**
+ * Get the current state of the router.
+ *
+ * @return String label for the current router state.
+ */
+ String getRouterStatus();
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
deleted file mode 100644
index 8f8bd3e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
+++ /dev/null
@@ -1,1005 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation;
-
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_BIND_HOST_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_ADMIN_BIND_HOST_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_CACHE_TIME_TO_LIVE_MS;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HANDLER_COUNT_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HEARTBEAT_INTERVAL_MS;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HTTPS_ADDRESS_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HTTP_ADDRESS_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HTTP_BIND_HOST_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_MONITOR_NAMENODE;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_RPC_ADDRESS_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_RPC_BIND_HOST_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS;
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMENODES;
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.addDirectory;
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.waitNamenodeRegistered;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map.Entry;
-import java.util.Random;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.UnsupportedFileSystemException;
-import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
-import org.apache.hadoop.hdfs.DFSClient;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.MiniDFSCluster.NameNodeInfo;
-import org.apache.hadoop.hdfs.MiniDFSNNTopology;
-import org.apache.hadoop.hdfs.MiniDFSNNTopology.NNConf;
-import org.apache.hadoop.hdfs.MiniDFSNNTopology.NSConf;
-import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
-import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport;
-import org.apache.hadoop.hdfs.server.federation.router.Router;
-import org.apache.hadoop.hdfs.server.federation.router.RouterClient;
-import org.apache.hadoop.hdfs.server.namenode.FSImage;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
-import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.Service.STATE;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Test utility to mimic a federated HDFS cluster with multiple routers.
- */
-public class RouterDFSCluster {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(RouterDFSCluster.class);
-
- public static final String TEST_STRING = "teststring";
- public static final String TEST_DIR = "testdir";
- public static final String TEST_FILE = "testfile";
-
-
- /** Nameservices in the federated cluster. */
- private List<String> nameservices;
- /** Namenodes in the federated cluster. */
- private List<NamenodeContext> namenodes;
- /** Routers in the federated cluster. */
- private List<RouterContext> routers;
- /** If the Namenodes are in high availability.*/
- private boolean highAvailability;
- /** Number of datanodes per nameservice. */
- private int numDatanodesPerNameservice = 2;
-
- /** Mini cluster. */
- private MiniDFSCluster cluster;
-
- protected static final long DEFAULT_HEARTBEAT_INTERVAL_MS =
- TimeUnit.SECONDS.toMillis(5);
- protected static final long DEFAULT_CACHE_INTERVAL_MS =
- TimeUnit.SECONDS.toMillis(5);
- /** Heartbeat interval in milliseconds. */
- private long heartbeatInterval;
- /** Cache flush interval in milliseconds. */
- private long cacheFlushInterval;
-
- /** Router configuration overrides. */
- private Configuration routerOverrides;
- /** Namenode configuration overrides. */
- private Configuration namenodeOverrides;
-
-
- /**
- * Router context.
- */
- public class RouterContext {
- private Router router;
- private FileContext fileContext;
- private String nameserviceId;
- private String namenodeId;
- private int rpcPort;
- private int httpPort;
- private DFSClient client;
- private Configuration conf;
- private RouterClient adminClient;
- private URI fileSystemUri;
-
- public RouterContext(Configuration conf, String nsId, String nnId) {
- this.conf = conf;
- this.nameserviceId = nsId;
- this.namenodeId = nnId;
-
- this.router = new Router();
- this.router.init(conf);
- }
-
- public Router getRouter() {
- return this.router;
- }
-
- public String getNameserviceId() {
- return this.nameserviceId;
- }
-
- public String getNamenodeId() {
- return this.namenodeId;
- }
-
- public int getRpcPort() {
- return this.rpcPort;
- }
-
- public int getHttpPort() {
- return this.httpPort;
- }
-
- public FileContext getFileContext() {
- return this.fileContext;
- }
-
- public void initRouter() throws URISyntaxException {
- // Store the bound points for the router interfaces
- InetSocketAddress rpcAddress = router.getRpcServerAddress();
- if (rpcAddress != null) {
- this.rpcPort = rpcAddress.getPort();
- this.fileSystemUri =
- URI.create("hdfs://" + NetUtils.getHostPortString(rpcAddress));
- // Override the default FS to point to the router RPC
- DistributedFileSystem.setDefaultUri(conf, fileSystemUri);
- try {
- this.fileContext = FileContext.getFileContext(conf);
- } catch (UnsupportedFileSystemException e) {
- this.fileContext = null;
- }
- }
- InetSocketAddress httpAddress = router.getHttpServerAddress();
- if (httpAddress != null) {
- this.httpPort = httpAddress.getPort();
- }
- }
-
- public FileSystem getFileSystem() throws IOException {
- return DistributedFileSystem.get(conf);
- }
-
- public DFSClient getClient(UserGroupInformation user)
- throws IOException, URISyntaxException, InterruptedException {
-
- LOG.info("Connecting to router at {}", fileSystemUri);
- return user.doAs(new PrivilegedExceptionAction<DFSClient>() {
- @Override
- public DFSClient run() throws IOException {
- return new DFSClient(fileSystemUri, conf);
- }
- });
- }
-
- public RouterClient getAdminClient() throws IOException {
- if (adminClient == null) {
- InetSocketAddress routerSocket = router.getAdminServerAddress();
- LOG.info("Connecting to router admin at {}", routerSocket);
- adminClient = new RouterClient(routerSocket, conf);
- }
- return adminClient;
- }
-
- public DFSClient getClient() throws IOException, URISyntaxException {
- if (client == null) {
- LOG.info("Connecting to router at {}", fileSystemUri);
- client = new DFSClient(fileSystemUri, conf);
- }
- return client;
- }
- }
-
- /**
- * Namenode context in the federated cluster.
- */
- public class NamenodeContext {
- private Configuration conf;
- private NameNode namenode;
- private String nameserviceId;
- private String namenodeId;
- private FileContext fileContext;
- private int rpcPort;
- private int servicePort;
- private int lifelinePort;
- private int httpPort;
- private URI fileSystemUri;
- private int index;
- private DFSClient client;
-
- public NamenodeContext(
- Configuration conf, String nsId, String nnId, int index) {
- this.conf = conf;
- this.nameserviceId = nsId;
- this.namenodeId = nnId;
- this.index = index;
- }
-
- public NameNode getNamenode() {
- return this.namenode;
- }
-
- public String getNameserviceId() {
- return this.nameserviceId;
- }
-
- public String getNamenodeId() {
- return this.namenodeId;
- }
-
- public FileContext getFileContext() {
- return this.fileContext;
- }
-
- public void setNamenode(NameNode nn) throws URISyntaxException {
- this.namenode = nn;
-
- // Store the bound ports and override the default FS with the local NN RPC
- this.rpcPort = nn.getNameNodeAddress().getPort();
- this.servicePort = nn.getServiceRpcAddress().getPort();
- this.lifelinePort = nn.getServiceRpcAddress().getPort();
- this.httpPort = nn.getHttpAddress().getPort();
- this.fileSystemUri = new URI("hdfs://" + namenode.getHostAndPort());
- DistributedFileSystem.setDefaultUri(this.conf, this.fileSystemUri);
-
- try {
- this.fileContext = FileContext.getFileContext(this.conf);
- } catch (UnsupportedFileSystemException e) {
- this.fileContext = null;
- }
- }
-
- public String getRpcAddress() {
- return namenode.getNameNodeAddress().getHostName() + ":" + rpcPort;
- }
-
- public String getServiceAddress() {
- return namenode.getServiceRpcAddress().getHostName() + ":" + servicePort;
- }
-
- public String getLifelineAddress() {
- return namenode.getServiceRpcAddress().getHostName() + ":" + lifelinePort;
- }
-
- public String getHttpAddress() {
- return namenode.getHttpAddress().getHostName() + ":" + httpPort;
- }
-
- public FileSystem getFileSystem() throws IOException {
- return DistributedFileSystem.get(conf);
- }
-
- public void resetClient() {
- client = null;
- }
-
- public DFSClient getClient(UserGroupInformation user)
- throws IOException, URISyntaxException, InterruptedException {
-
- LOG.info("Connecting to namenode at {}", fileSystemUri);
- return user.doAs(new PrivilegedExceptionAction<DFSClient>() {
- @Override
- public DFSClient run() throws IOException {
- return new DFSClient(fileSystemUri, conf);
- }
- });
- }
-
- public DFSClient getClient() throws IOException, URISyntaxException {
- if (client == null) {
- LOG.info("Connecting to namenode at {}", fileSystemUri);
- client = new DFSClient(fileSystemUri, conf);
- }
- return client;
- }
-
- public String getConfSuffix() {
- String suffix = nameserviceId;
- if (highAvailability) {
- suffix += "." + namenodeId;
- }
- return suffix;
- }
- }
-
- public RouterDFSCluster(boolean ha, int numNameservices, int numNamenodes,
- long heartbeatInterval, long cacheFlushInterval) {
- this.highAvailability = ha;
- this.heartbeatInterval = heartbeatInterval;
- this.cacheFlushInterval = cacheFlushInterval;
- configureNameservices(numNameservices, numNamenodes);
- }
-
- public RouterDFSCluster(boolean ha, int numNameservices) {
- this(ha, numNameservices, 2,
- DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_CACHE_INTERVAL_MS);
- }
-
- public RouterDFSCluster(boolean ha, int numNameservices, int numNamenodes) {
- this(ha, numNameservices, numNamenodes,
- DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_CACHE_INTERVAL_MS);
- }
-
- /**
- * Add configuration settings to override default Router settings.
- *
- * @param conf Router configuration overrides.
- */
- public void addRouterOverrides(Configuration conf) {
- if (this.routerOverrides == null) {
- this.routerOverrides = conf;
- } else {
- this.routerOverrides.addResource(conf);
- }
- }
-
- /**
- * Add configuration settings to override default Namenode settings.
- *
- * @param conf Namenode configuration overrides.
- */
- public void addNamenodeOverrides(Configuration conf) {
- if (this.namenodeOverrides == null) {
- this.namenodeOverrides = conf;
- } else {
- this.namenodeOverrides.addResource(conf);
- }
- }
-
- /**
- * Generate the configuration for a client.
- *
- * @param nsId Nameservice identifier.
- * @return New namenode configuration.
- */
- public Configuration generateNamenodeConfiguration(String nsId) {
- Configuration conf = new HdfsConfiguration();
-
- conf.set(DFS_NAMESERVICES, getNameservicesKey());
- conf.set(FS_DEFAULT_NAME_KEY, "hdfs://" + nsId);
-
- for (String ns : nameservices) {
- if (highAvailability) {
- conf.set(
- DFS_HA_NAMENODES_KEY_PREFIX + "." + ns,
- NAMENODES[0] + "," + NAMENODES[1]);
- }
-
- for (NamenodeContext context : getNamenodes(ns)) {
- String suffix = context.getConfSuffix();
-
- conf.set(DFS_NAMENODE_RPC_ADDRESS_KEY + "." + suffix,
- "127.0.0.1:" + context.rpcPort);
- conf.set(DFS_NAMENODE_HTTP_ADDRESS_KEY + "." + suffix,
- "127.0.0.1:" + context.httpPort);
- conf.set(DFS_NAMENODE_RPC_BIND_HOST_KEY + "." + suffix,
- "0.0.0.0");
-
- // If the service port is enabled by default, we need to set them up
- boolean servicePortEnabled = false;
- if (servicePortEnabled) {
- conf.set(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY + "." + suffix,
- "127.0.0.1:" + context.servicePort);
- conf.set(DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY + "." + suffix,
- "0.0.0.0");
- }
- }
- }
-
- if (this.namenodeOverrides != null) {
- conf.addResource(this.namenodeOverrides);
- }
- return conf;
- }
-
- /**
- * Generate the configuration for a client.
- *
- * @return New configuration for a client.
- */
- public Configuration generateClientConfiguration() {
- Configuration conf = new HdfsConfiguration(false);
- String ns0 = getNameservices().get(0);
- conf.addResource(generateNamenodeConfiguration(ns0));
- return conf;
- }
-
- /**
- * Generate the configuration for a Router.
- *
- * @param nsId Nameservice identifier.
- * @param nnId Namenode identifier.
- * @return New configuration for a Router.
- */
- public Configuration generateRouterConfiguration(String nsId, String nnId) {
-
- Configuration conf = new HdfsConfiguration(false);
- conf.addResource(generateNamenodeConfiguration(nsId));
-
- conf.setInt(DFS_ROUTER_HANDLER_COUNT_KEY, 10);
- conf.set(DFS_ROUTER_RPC_ADDRESS_KEY, "127.0.0.1:0");
- conf.set(DFS_ROUTER_RPC_BIND_HOST_KEY, "0.0.0.0");
-
- conf.set(DFS_ROUTER_ADMIN_ADDRESS_KEY, "127.0.0.1:0");
- conf.set(DFS_ROUTER_ADMIN_BIND_HOST_KEY, "0.0.0.0");
-
- conf.set(DFS_ROUTER_HTTP_ADDRESS_KEY, "127.0.0.1:0");
- conf.set(DFS_ROUTER_HTTPS_ADDRESS_KEY, "127.0.0.1:0");
- conf.set(DFS_ROUTER_HTTP_BIND_HOST_KEY, "0.0.0.0");
-
- conf.set(DFS_ROUTER_DEFAULT_NAMESERVICE, nameservices.get(0));
- conf.setLong(DFS_ROUTER_HEARTBEAT_INTERVAL_MS, heartbeatInterval);
- conf.setLong(DFS_ROUTER_CACHE_TIME_TO_LIVE_MS, cacheFlushInterval);
-
- // Use mock resolver classes
- conf.setClass(FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS,
- MockResolver.class, ActiveNamenodeResolver.class);
- conf.setClass(FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
- MockResolver.class, FileSubclusterResolver.class);
-
- // Set the nameservice ID for the default NN monitor
- conf.set(DFS_NAMESERVICE_ID, nsId);
- if (nnId != null) {
- conf.set(DFS_HA_NAMENODE_ID_KEY, nnId);
- }
-
- // Namenodes to monitor
- StringBuilder sb = new StringBuilder();
- for (String ns : this.nameservices) {
- for (NamenodeContext context : getNamenodes(ns)) {
- String suffix = context.getConfSuffix();
- if (sb.length() != 0) {
- sb.append(",");
- }
- sb.append(suffix);
- }
- }
- conf.set(DFS_ROUTER_MONITOR_NAMENODE, sb.toString());
-
- // Add custom overrides if available
- if (this.routerOverrides != null) {
- for (Entry<String, String> entry : this.routerOverrides) {
- String confKey = entry.getKey();
- String confValue = entry.getValue();
- conf.set(confKey, confValue);
- }
- }
- return conf;
- }
-
- public void configureNameservices(int numNameservices, int numNamenodes) {
- this.nameservices = new ArrayList<>();
- this.namenodes = new ArrayList<>();
-
- NamenodeContext context = null;
- int nnIndex = 0;
- for (int i=0; i<numNameservices; i++) {
- String ns = "ns" + i;
- this.nameservices.add("ns" + i);
-
- Configuration nnConf = generateNamenodeConfiguration(ns);
- if (!highAvailability) {
- context = new NamenodeContext(nnConf, ns, null, nnIndex++);
- this.namenodes.add(context);
- } else {
- for (int j=0; j<numNamenodes; j++) {
- context = new NamenodeContext(nnConf, ns, NAMENODES[j], nnIndex++);
- this.namenodes.add(context);
- }
- }
- }
- }
-
- public void setNumDatanodesPerNameservice(int num) {
- this.numDatanodesPerNameservice = num;
- }
-
- public String getNameservicesKey() {
- StringBuilder sb = new StringBuilder();
- for (String nsId : this.nameservices) {
- if (sb.length() > 0) {
- sb.append(",");
- }
- sb.append(nsId);
- }
- return sb.toString();
- }
-
- public String getRandomNameservice() {
- Random r = new Random();
- int randIndex = r.nextInt(nameservices.size());
- return nameservices.get(randIndex);
- }
-
- public List<String> getNameservices() {
- return nameservices;
- }
-
- public List<NamenodeContext> getNamenodes(String nameservice) {
- List<NamenodeContext> nns = new ArrayList<>();
- for (NamenodeContext c : namenodes) {
- if (c.nameserviceId.equals(nameservice)) {
- nns.add(c);
- }
- }
- return nns;
- }
-
- public NamenodeContext getRandomNamenode() {
- Random rand = new Random();
- int i = rand.nextInt(this.namenodes.size());
- return this.namenodes.get(i);
- }
-
- public List<NamenodeContext> getNamenodes() {
- return this.namenodes;
- }
-
- public boolean isHighAvailability() {
- return highAvailability;
- }
-
- public NamenodeContext getNamenode(String nameservice, String namenode) {
- for (NamenodeContext c : this.namenodes) {
- if (c.nameserviceId.equals(nameservice)) {
- if (namenode == null || namenode.isEmpty() ||
- c.namenodeId == null || c.namenodeId.isEmpty()) {
- return c;
- } else if (c.namenodeId.equals(namenode)) {
- return c;
- }
- }
- }
- return null;
- }
-
- public List<RouterContext> getRouters(String nameservice) {
- List<RouterContext> nns = new ArrayList<>();
- for (RouterContext c : routers) {
- if (c.nameserviceId.equals(nameservice)) {
- nns.add(c);
- }
- }
- return nns;
- }
-
- public RouterContext getRouterContext(String nsId, String nnId) {
- for (RouterContext c : routers) {
- if (nnId == null) {
- return c;
- }
- if (c.namenodeId.equals(nnId) &&
- c.nameserviceId.equals(nsId)) {
- return c;
- }
- }
- return null;
- }
-
- public RouterContext getRandomRouter() {
- Random rand = new Random();
- return routers.get(rand.nextInt(routers.size()));
- }
-
- public List<RouterContext> getRouters() {
- return routers;
- }
-
- public RouterContext buildRouter(String nsId, String nnId)
- throws URISyntaxException, IOException {
- Configuration config = generateRouterConfiguration(nsId, nnId);
- RouterContext rc = new RouterContext(config, nsId, nnId);
- return rc;
- }
-
- public void startCluster() {
- startCluster(null);
- }
-
- public void startCluster(Configuration overrideConf) {
- try {
- MiniDFSNNTopology topology = new MiniDFSNNTopology();
- for (String ns : nameservices) {
- NSConf conf = new MiniDFSNNTopology.NSConf(ns);
- if (highAvailability) {
- for (int i=0; i<namenodes.size()/nameservices.size(); i++) {
- NNConf nnConf = new MiniDFSNNTopology.NNConf("nn" + i);
- conf.addNN(nnConf);
- }
- } else {
- NNConf nnConf = new MiniDFSNNTopology.NNConf(null);
- conf.addNN(nnConf);
- }
- topology.addNameservice(conf);
- }
- topology.setFederation(true);
-
- // Start mini DFS cluster
- String ns0 = nameservices.get(0);
- Configuration nnConf = generateNamenodeConfiguration(ns0);
- if (overrideConf != null) {
- nnConf.addResource(overrideConf);
- }
- cluster = new MiniDFSCluster.Builder(nnConf)
- .numDataNodes(nameservices.size() * numDatanodesPerNameservice)
- .nnTopology(topology)
- .build();
- cluster.waitActive();
-
- // Store NN pointers
- for (int i = 0; i < namenodes.size(); i++) {
- NameNode nn = cluster.getNameNode(i);
- namenodes.get(i).setNamenode(nn);
- }
-
- } catch (Exception e) {
- LOG.error("Cannot start Router DFS cluster: {}", e.getMessage(), e);
- if (cluster != null) {
- cluster.shutdown();
- }
- }
- }
-
- public void startRouters()
- throws InterruptedException, URISyntaxException, IOException {
-
- // Create one router per nameservice
- this.routers = new ArrayList<>();
- for (String ns : this.nameservices) {
- for (NamenodeContext context : getNamenodes(ns)) {
- RouterContext router = buildRouter(ns, context.namenodeId);
- this.routers.add(router);
- }
- }
-
- // Start all routers
- for (RouterContext router : this.routers) {
- router.router.start();
- }
-
- // Wait until all routers are active and record their ports
- for (RouterContext router : this.routers) {
- waitActive(router);
- router.initRouter();
- }
- }
-
- public void waitActive(NamenodeContext nn) throws IOException {
- cluster.waitActive(nn.index);
- }
-
- public void waitActive(RouterContext router)
- throws InterruptedException {
- for (int loopCount = 0; loopCount < 20; loopCount++) {
- // Validate connection of routers to NNs
- if (router.router.getServiceState() == STATE.STARTED) {
- return;
- }
- Thread.sleep(1000);
- }
- fail("Timeout waiting for " + router.router + " to activate");
- }
-
- public void registerNamenodes() throws IOException {
- for (RouterContext r : this.routers) {
- ActiveNamenodeResolver resolver = r.router.getNamenodeResolver();
- for (NamenodeContext nn : this.namenodes) {
- // Generate a report
- NamenodeStatusReport report = new NamenodeStatusReport(
- nn.nameserviceId, nn.namenodeId,
- nn.getRpcAddress(), nn.getServiceAddress(),
- nn.getLifelineAddress(), nn.getHttpAddress());
- FSImage fsImage = nn.namenode.getNamesystem().getFSImage();
- NamespaceInfo nsInfo = fsImage.getStorage().getNamespaceInfo();
- report.setNamespaceInfo(nsInfo);
-
- // Determine HA state from nn public state string
- String nnState = nn.namenode.getState();
- HAServiceState haState = HAServiceState.ACTIVE;
- for (HAServiceState state : HAServiceState.values()) {
- if (nnState.equalsIgnoreCase(state.name())) {
- haState = state;
- break;
- }
- }
- report.setHAServiceState(haState);
-
- // Register with the resolver
- resolver.registerNamenode(report);
- }
- }
- }
-
- public void waitNamenodeRegistration() throws Exception {
- for (RouterContext r : this.routers) {
- Router router = r.router;
- for (NamenodeContext nn : this.namenodes) {
- ActiveNamenodeResolver nnResolver = router.getNamenodeResolver();
- waitNamenodeRegistered(
- nnResolver, nn.nameserviceId, nn.namenodeId, null);
- }
- }
- }
-
- public void waitRouterRegistrationQuorum(RouterContext router,
- FederationNamenodeServiceState state, String nsId, String nnId)
- throws Exception {
- LOG.info("Waiting for NN {} {} to transition to {}", nsId, nnId, state);
- ActiveNamenodeResolver nnResolver = router.router.getNamenodeResolver();
- waitNamenodeRegistered(nnResolver, nsId, nnId, state);
- }
-
- /**
- * Get the federated path for a nameservice.
- * @param nsId Nameservice identifier.
- * @return Path in the Router.
- */
- public String getFederatedPathForNS(String nsId) {
- return "/" + nsId;
- }
-
- /**
- * Get the namenode path for a nameservice.
- * @param nsId Nameservice identifier.
- * @return Path in the Namenode.
- */
- public String getNamenodePathForNS(String nsId) {
- return "/target-" + nsId;
- }
-
- /**
- * Get the federated test directory for a nameservice.
- * @param nsId Nameservice identifier.
- * @return Example:
- * <ul>
- * <li>/ns0/testdir which maps to ns0->/target-ns0/testdir
- * </ul>
- */
- public String getFederatedTestDirectoryForNS(String nsId) {
- return getFederatedPathForNS(nsId) + "/" + TEST_DIR;
- }
-
- /**
- * Get the namenode test directory for a nameservice.
- * @param nsId Nameservice identifier.
- * @return example:
- * <ul>
- * <li>/target-ns0/testdir
- * </ul>
- */
- public String getNamenodeTestDirectoryForNS(String nsId) {
- return getNamenodePathForNS(nsId) + "/" + TEST_DIR;
- }
-
- /**
- * Get the federated test file for a nameservice.
- * @param nsId Nameservice identifier.
- * @return example:
- * <ul>
- * <li>/ns0/testfile which maps to ns0->/target-ns0/testfile
- * </ul>
- */
- public String getFederatedTestFileForNS(String nsId) {
- return getFederatedPathForNS(nsId) + "/" + TEST_FILE;
- }
-
- /**
- * Get the namenode test file for a nameservice.
- * @param nsId Nameservice identifier.
- * @return example:
- * <ul>
- * <li>/target-ns0/testfile
- * </ul>
- */
- public String getNamenodeTestFileForNS(String nsId) {
- return getNamenodePathForNS(nsId) + "/" + TEST_FILE;
- }
-
- /**
- * Switch a namenode in a nameservice to be the active.
- * @param nsId Nameservice identifier.
- * @param nnId Namenode identifier.
- */
- public void switchToActive(String nsId, String nnId) {
- try {
- int total = cluster.getNumNameNodes();
- NameNodeInfo[] nns = cluster.getNameNodeInfos();
- for (int i = 0; i < total; i++) {
- NameNodeInfo nn = nns[i];
- if (nn.getNameserviceId().equals(nsId) &&
- nn.getNamenodeId().equals(nnId)) {
- cluster.transitionToActive(i);
- }
- }
- } catch (Throwable e) {
- LOG.error("Cannot transition to active", e);
- }
- }
-
- /**
- * Switch a namenode in a nameservice to be in standby.
- * @param nsId Nameservice identifier.
- * @param nnId Namenode identifier.
- */
- public void switchToStandby(String nsId, String nnId) {
- try {
- int total = cluster.getNumNameNodes();
- NameNodeInfo[] nns = cluster.getNameNodeInfos();
- for (int i = 0; i < total; i++) {
- NameNodeInfo nn = nns[i];
- if (nn.getNameserviceId().equals(nsId) &&
- nn.getNamenodeId().equals(nnId)) {
- cluster.transitionToStandby(i);
- }
- }
- } catch (Throwable e) {
- LOG.error("Cannot transition to standby", e);
- }
- }
-
- /**
- * Stop the federated HDFS cluster.
- */
- public void shutdown() {
- if (cluster != null) {
- cluster.shutdown();
- }
- if (routers != null) {
- for (RouterContext context : routers) {
- stopRouter(context);
- }
- }
- }
-
- /**
- * Stop a router.
- * @param router Router context.
- */
- public void stopRouter(RouterContext router) {
- try {
- router.router.shutDown();
-
- int loopCount = 0;
- while (router.router.getServiceState() != STATE.STOPPED) {
- loopCount++;
- Thread.sleep(1000);
- if (loopCount > 20) {
- LOG.error("Cannot shutdown router {}", router.rpcPort);
- break;
- }
- }
- } catch (InterruptedException e) {
- }
- }
-
- /////////////////////////////////////////////////////////////////////////////
- // Namespace Test Fixtures
- /////////////////////////////////////////////////////////////////////////////
-
- /**
- * Creates test directories via the namenode.
- * 1) /target-ns0/testfile
- * 2) /target-ns1/testfile
- * @throws IOException
- */
- public void createTestDirectoriesNamenode() throws IOException {
- // Add a test dir to each NS and verify
- for (String ns : getNameservices()) {
- NamenodeContext context = getNamenode(ns, null);
- if (!createTestDirectoriesNamenode(context)) {
- throw new IOException("Cannot create test directory for ns " + ns);
- }
- }
- }
-
- public boolean createTestDirectoriesNamenode(NamenodeContext nn)
- throws IOException {
- FileSystem fs = nn.getFileSystem();
- String testDir = getNamenodeTestDirectoryForNS(nn.nameserviceId);
- return addDirectory(fs, testDir);
- }
-
- public void deleteAllFiles() throws IOException {
- // Delete all files via the NNs and verify
- for (NamenodeContext context : getNamenodes()) {
- FileSystem fs = context.getFileSystem();
- FileStatus[] status = fs.listStatus(new Path("/"));
- for (int i = 0; i <status.length; i++) {
- Path p = status[i].getPath();
- fs.delete(p, true);
- }
- status = fs.listStatus(new Path("/"));
- assertEquals(status.length, 0);
- }
- }
-
- /////////////////////////////////////////////////////////////////////////////
- // MockRouterResolver Test Fixtures
- /////////////////////////////////////////////////////////////////////////////
-
- /**
- * <ul>
- * <li>/ -> [ns0->/].
- * <li>/nso -> ns0->/target-ns0.
- * <li>/ns1 -> ns1->/target-ns1.
- * </ul>
- */
- public void installMockLocations() {
- for (RouterContext r : routers) {
- MockResolver resolver =
- (MockResolver) r.router.getSubclusterResolver();
- // create table entries
- for (String nsId : nameservices) {
- // Direct path
- String routerPath = getFederatedPathForNS(nsId);
- String nnPath = getNamenodePathForNS(nsId);
- resolver.addLocation(routerPath, nsId, nnPath);
- }
-
- // Root path points to both first nameservice
- String ns0 = nameservices.get(0);
- resolver.addLocation("/", ns0, "/");
- }
- }
-
- public MiniDFSCluster getCluster() {
- return cluster;
- }
-
- /**
- * Wait until the federated cluster is up and ready.
- * @throws IOException If we cannot wait for the cluster to be up.
- */
- public void waitClusterUp() throws IOException {
- cluster.waitClusterUp();
- registerNamenodes();
- try {
- waitNamenodeRegistration();
- } catch (Exception e) {
- throw new IOException("Cannot wait for the namenodes", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/StateStoreDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/StateStoreDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/StateStoreDFSCluster.java
deleted file mode 100644
index e42ab50..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/StateStoreDFSCluster.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation;
-
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.createMockRegistrationForNamenode;
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.getStateStoreConfiguration;
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.synchronizeRecords;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
-import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
-import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-
-/**
- * Test utility to mimic a federated HDFS cluster with a router and a state
- * store.
- */
-public class StateStoreDFSCluster extends RouterDFSCluster {
-
- private static final Class<?> DEFAULT_FILE_RESOLVER =
- MountTableResolver.class;
- private static final Class<?> DEFAULT_NAMENODE_RESOLVER =
- MembershipNamenodeResolver.class;
-
- public StateStoreDFSCluster(boolean ha, int numNameservices, int numNamenodes,
- long heartbeatInterval, long cacheFlushInterval)
- throws IOException, InterruptedException {
- this(ha, numNameservices, numNamenodes, heartbeatInterval,
- cacheFlushInterval, DEFAULT_FILE_RESOLVER);
- }
-
- public StateStoreDFSCluster(boolean ha, int numNameservices, int numNamenodes,
- long heartbeatInterval, long cacheFlushInterval, Class<?> fileResolver)
- throws IOException, InterruptedException {
- super(ha, numNameservices, numNamenodes, heartbeatInterval,
- cacheFlushInterval);
-
- // Attach state store and resolvers to router
- Configuration stateStoreConfig = getStateStoreConfiguration();
- // Use state store backed resolvers
- stateStoreConfig.setClass(
- DFSConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS,
- DEFAULT_NAMENODE_RESOLVER, ActiveNamenodeResolver.class);
- stateStoreConfig.setClass(
- DFSConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
- fileResolver, FileSubclusterResolver.class);
- this.addRouterOverrides(stateStoreConfig);
- }
-
- public StateStoreDFSCluster(boolean ha, int numNameservices,
- Class<?> fileResolver) throws IOException, InterruptedException {
- this(ha, numNameservices, 2,
- DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_CACHE_INTERVAL_MS, fileResolver);
- }
-
- public StateStoreDFSCluster(boolean ha, int numNameservices)
- throws IOException, InterruptedException {
- this(ha, numNameservices, 2,
- DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_CACHE_INTERVAL_MS);
- }
-
- public StateStoreDFSCluster(boolean ha, int numNameservices,
- int numNamnodes) throws IOException, InterruptedException {
- this(ha, numNameservices, numNamnodes,
- DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_CACHE_INTERVAL_MS);
- }
-
- /////////////////////////////////////////////////////////////////////////////
- // State Store Test Fixtures
- /////////////////////////////////////////////////////////////////////////////
-
- /**
- * Adds test fixtures for NN registation for each NN nameservice -> NS
- * namenode -> NN rpcAddress -> 0.0.0.0:0 webAddress -> 0.0.0.0:0 state ->
- * STANDBY safeMode -> false blockPool -> test.
- *
- * @param stateStore State Store.
- * @throws IOException If it cannot register.
- */
- public void createTestRegistration(StateStoreService stateStore)
- throws IOException {
- List<MembershipState> entries = new ArrayList<MembershipState>();
- for (NamenodeContext nn : this.getNamenodes()) {
- MembershipState entry = createMockRegistrationForNamenode(
- nn.getNameserviceId(), nn.getNamenodeId(),
- FederationNamenodeServiceState.STANDBY);
- entries.add(entry);
- }
- synchronizeRecords(
- stateStore, entries, MembershipState.class);
- }
-
- public void createTestMountTable(StateStoreService stateStore)
- throws IOException {
- List<MountTable> mounts = generateMockMountTable();
- synchronizeRecords(stateStore, mounts, MountTable.class);
- stateStore.refreshCaches();
- }
-
- public List<MountTable> generateMockMountTable() throws IOException {
- // create table entries
- List<MountTable> entries = new ArrayList<>();
- for (String ns : this.getNameservices()) {
- Map<String, String> destMap = new HashMap<>();
- destMap.put(ns, getNamenodePathForNS(ns));
-
- // Direct path
- String fedPath = getFederatedPathForNS(ns);
- MountTable entry = MountTable.newInstance(fedPath, destMap);
- entries.add(entry);
- }
-
- // Root path goes to nameservice 1
- Map<String, String> destMap = new HashMap<>();
- String ns0 = this.getNameservices().get(0);
- destMap.put(ns0, "/");
- MountTable entry = MountTable.newInstance("/", destMap);
- entries.add(entry);
- return entries;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestFederationMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestFederationMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestFederationMetrics.java
deleted file mode 100644
index 94799f3..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestFederationMetrics.java
+++ /dev/null
@@ -1,293 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.metrics;
-
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.getBean;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.List;
-
-import javax.management.MalformedObjectNameException;
-
-import org.apache.commons.collections.ListUtils;
-import org.apache.hadoop.hdfs.server.federation.router.Router;
-import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
-import org.apache.hadoop.hdfs.server.federation.store.records.MembershipStats;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
-import org.apache.hadoop.hdfs.server.federation.store.records.StateStoreVersion;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-import org.junit.Test;
-
-/**
- * Test the JMX interface for the {@link Router}.
- */
-public class TestFederationMetrics extends TestMetricsBase {
-
- public static final String FEDERATION_BEAN =
- "Hadoop:service=Router,name=FederationState";
- public static final String STATE_STORE_BEAN =
- "Hadoop:service=Router,name=StateStore";
- public static final String RPC_BEAN =
- "Hadoop:service=Router,name=FederationRPC";
-
- @Test
- public void testClusterStatsJMX()
- throws MalformedObjectNameException, IOException {
-
- FederationMBean bean = getBean(FEDERATION_BEAN, FederationMBean.class);
- validateClusterStatsBean(bean);
- }
-
- @Test
- public void testClusterStatsDataSource() throws IOException {
- FederationMetrics metrics = getRouter().getMetrics();
- validateClusterStatsBean(metrics);
- }
-
- @Test
- public void testMountTableStatsDataSource()
- throws IOException, JSONException {
-
- FederationMetrics metrics = getRouter().getMetrics();
- String jsonString = metrics.getMountTable();
- JSONArray jsonArray = new JSONArray(jsonString);
- assertEquals(jsonArray.length(), getMockMountTable().size());
-
- int match = 0;
- for (int i = 0; i < jsonArray.length(); i++) {
- JSONObject json = jsonArray.getJSONObject(i);
- String src = json.getString("sourcePath");
-
- for (MountTable entry : getMockMountTable()) {
- if (entry.getSourcePath().equals(src)) {
- assertEquals(entry.getDefaultLocation().getNameserviceId(),
- json.getString("nameserviceId"));
- assertEquals(entry.getDefaultLocation().getDest(),
- json.getString("path"));
- assertEquals(entry.getOwnerName(), json.getString("ownerName"));
- assertEquals(entry.getGroupName(), json.getString("groupName"));
- assertEquals(entry.getMode().toString(), json.getString("mode"));
- assertEquals(entry.getQuota().toString(), json.getString("quota"));
- assertNotNullAndNotEmpty(json.getString("dateCreated"));
- assertNotNullAndNotEmpty(json.getString("dateModified"));
- match++;
- }
- }
- }
- assertEquals(match, getMockMountTable().size());
- }
-
- private MembershipState findMockNamenode(String nsId, String nnId) {
-
- @SuppressWarnings("unchecked")
- List<MembershipState> namenodes =
- ListUtils.union(getActiveMemberships(), getStandbyMemberships());
- for (MembershipState nn : namenodes) {
- if (nn.getNamenodeId().equals(nnId)
- && nn.getNameserviceId().equals(nsId)) {
- return nn;
- }
- }
- return null;
- }
-
- @Test
- public void testNamenodeStatsDataSource() throws IOException, JSONException {
-
- FederationMetrics metrics = getRouter().getMetrics();
- String jsonString = metrics.getNamenodes();
- JSONObject jsonObject = new JSONObject(jsonString);
- Iterator<?> keys = jsonObject.keys();
- int nnsFound = 0;
- while (keys.hasNext()) {
- // Validate each entry against our mocks
- JSONObject json = jsonObject.getJSONObject((String) keys.next());
- String nameserviceId = json.getString("nameserviceId");
- String namenodeId = json.getString("namenodeId");
-
- MembershipState mockEntry =
- this.findMockNamenode(nameserviceId, namenodeId);
- assertNotNull(mockEntry);
-
- assertEquals(json.getString("state"), mockEntry.getState().toString());
- MembershipStats stats = mockEntry.getStats();
- assertEquals(json.getLong("numOfActiveDatanodes"),
- stats.getNumOfActiveDatanodes());
- assertEquals(json.getLong("numOfDeadDatanodes"),
- stats.getNumOfDeadDatanodes());
- assertEquals(json.getLong("numOfDecommissioningDatanodes"),
- stats.getNumOfDecommissioningDatanodes());
- assertEquals(json.getLong("numOfDecomActiveDatanodes"),
- stats.getNumOfDecomActiveDatanodes());
- assertEquals(json.getLong("numOfDecomDeadDatanodes"),
- stats.getNumOfDecomDeadDatanodes());
- assertEquals(json.getLong("numOfBlocks"), stats.getNumOfBlocks());
- assertEquals(json.getString("rpcAddress"), mockEntry.getRpcAddress());
- assertEquals(json.getString("webAddress"), mockEntry.getWebAddress());
- nnsFound++;
- }
- // Validate all memberships are present
- assertEquals(getActiveMemberships().size() + getStandbyMemberships().size(),
- nnsFound);
- }
-
- @Test
- public void testNameserviceStatsDataSource()
- throws IOException, JSONException {
-
- FederationMetrics metrics = getRouter().getMetrics();
- String jsonString = metrics.getNameservices();
- JSONObject jsonObject = new JSONObject(jsonString);
- Iterator<?> keys = jsonObject.keys();
- int nameservicesFound = 0;
- while (keys.hasNext()) {
- JSONObject json = jsonObject.getJSONObject((String) keys.next());
- String nameserviceId = json.getString("nameserviceId");
- String namenodeId = json.getString("namenodeId");
-
- MembershipState mockEntry =
- this.findMockNamenode(nameserviceId, namenodeId);
- assertNotNull(mockEntry);
-
- // NS should report the active NN
- assertEquals(mockEntry.getState().toString(), json.getString("state"));
- assertEquals("ACTIVE", json.getString("state"));
-
- // Stats in the NS should reflect the stats for the most active NN
- MembershipStats stats = mockEntry.getStats();
- assertEquals(stats.getNumOfFiles(), json.getLong("numOfFiles"));
- assertEquals(stats.getTotalSpace(), json.getLong("totalSpace"));
- assertEquals(stats.getAvailableSpace(),
- json.getLong("availableSpace"));
- assertEquals(stats.getNumOfBlocksMissing(),
- json.getLong("numOfBlocksMissing"));
- assertEquals(stats.getNumOfActiveDatanodes(),
- json.getLong("numOfActiveDatanodes"));
- assertEquals(stats.getNumOfDeadDatanodes(),
- json.getLong("numOfDeadDatanodes"));
- assertEquals(stats.getNumOfDecommissioningDatanodes(),
- json.getLong("numOfDecommissioningDatanodes"));
- assertEquals(stats.getNumOfDecomActiveDatanodes(),
- json.getLong("numOfDecomActiveDatanodes"));
- assertEquals(stats.getNumOfDecomDeadDatanodes(),
- json.getLong("numOfDecomDeadDatanodes"));
- assertEquals(stats.getProvidedSpace(),
- json.getLong("providedSpace"));
- nameservicesFound++;
- }
- assertEquals(getNameservices().size(), nameservicesFound);
- }
-
- @Test
- public void testRouterStatsDataSource() throws IOException, JSONException {
-
- FederationMetrics metrics = getRouter().getMetrics();
- String jsonString = metrics.getRouters();
- JSONObject jsonObject = new JSONObject(jsonString);
- Iterator<?> keys = jsonObject.keys();
- int routersFound = 0;
- while (keys.hasNext()) {
- JSONObject json = jsonObject.getJSONObject((String) keys.next());
- String address = json.getString("address");
- assertNotNullAndNotEmpty(address);
- RouterState router = findMockRouter(address);
- assertNotNull(router);
-
- assertEquals(router.getStatus().toString(), json.getString("status"));
- assertEquals(router.getCompileInfo(), json.getString("compileInfo"));
- assertEquals(router.getVersion(), json.getString("version"));
- assertEquals(router.getDateStarted(), json.getLong("dateStarted"));
- assertEquals(router.getDateCreated(), json.getLong("dateCreated"));
- assertEquals(router.getDateModified(), json.getLong("dateModified"));
-
- StateStoreVersion version = router.getStateStoreVersion();
- assertEquals(
- FederationMetrics.getDateString(version.getMembershipVersion()),
- json.get("lastMembershipUpdate"));
- assertEquals(
- FederationMetrics.getDateString(version.getMountTableVersion()),
- json.get("lastMountTableUpdate"));
- assertEquals(version.getMembershipVersion(),
- json.get("membershipVersion"));
- assertEquals(version.getMountTableVersion(),
- json.get("mountTableVersion"));
- routersFound++;
- }
-
- assertEquals(getMockRouters().size(), routersFound);
- }
-
- private void assertNotNullAndNotEmpty(String field) {
- assertNotNull(field);
- assertTrue(field.length() > 0);
- }
-
- private RouterState findMockRouter(String routerId) {
- for (RouterState router : getMockRouters()) {
- if (router.getAddress().equals(routerId)) {
- return router;
- }
- }
- return null;
- }
-
- private void validateClusterStatsBean(FederationMBean bean)
- throws IOException {
-
- // Determine aggregates
- long numBlocks = 0;
- long numLive = 0;
- long numDead = 0;
- long numDecom = 0;
- long numDecomLive = 0;
- long numDecomDead = 0;
- long numFiles = 0;
- for (MembershipState mock : getActiveMemberships()) {
- MembershipStats stats = mock.getStats();
- numBlocks += stats.getNumOfBlocks();
- numLive += stats.getNumOfActiveDatanodes();
- numDead += stats.getNumOfDeadDatanodes();
- numDecom += stats.getNumOfDecommissioningDatanodes();
- numDecomLive += stats.getNumOfDecomActiveDatanodes();
- numDecomDead += stats.getNumOfDecomDeadDatanodes();
- }
-
- assertEquals(numBlocks, bean.getNumBlocks());
- assertEquals(numLive, bean.getNumLiveNodes());
- assertEquals(numDead, bean.getNumDeadNodes());
- assertEquals(numDecom, bean.getNumDecommissioningNodes());
- assertEquals(numDecomLive, bean.getNumDecomLiveNodes());
- assertEquals(numDecomDead, bean.getNumDecomDeadNodes());
- assertEquals(numFiles, bean.getNumFiles());
- assertEquals(getActiveMemberships().size() + getStandbyMemberships().size(),
- bean.getNumNamenodes());
- assertEquals(getNameservices().size(), bean.getNumNameservices());
- assertTrue(bean.getVersion().length() > 0);
- assertTrue(bean.getCompiledDate().length() > 0);
- assertTrue(bean.getCompileInfo().length() > 0);
- assertTrue(bean.getRouterStarted().length() > 0);
- assertTrue(bean.getHostAndPort().length() > 0);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestMetricsBase.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestMetricsBase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestMetricsBase.java
deleted file mode 100644
index 2169b21..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestMetricsBase.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.metrics;
-
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMENODES;
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMESERVICES;
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.clearAllRecords;
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.createMockMountTable;
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.createMockRegistrationForNamenode;
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.synchronizeRecords;
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.waitStateStore;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
-import org.apache.hadoop.hdfs.server.federation.router.Router;
-import org.apache.hadoop.hdfs.server.federation.router.RouterServiceState;
-import org.apache.hadoop.hdfs.server.federation.store.MembershipStore;
-import org.apache.hadoop.hdfs.server.federation.store.RouterStore;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatRequest;
-import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
-import org.apache.hadoop.hdfs.server.federation.store.records.StateStoreVersion;
-import org.apache.hadoop.util.Time;
-import org.junit.After;
-import org.junit.Before;
-
-/**
- * Test the basic metrics functionality.
- */
-public class TestMetricsBase {
-
- private StateStoreService stateStore;
- private MembershipStore membershipStore;
- private RouterStore routerStore;
- private Router router;
- private Configuration routerConfig;
-
- private List<MembershipState> activeMemberships;
- private List<MembershipState> standbyMemberships;
- private List<MountTable> mockMountTable;
- private List<RouterState> mockRouters;
- private List<String> nameservices;
-
- @Before
- public void setupBase() throws Exception {
-
- if (router == null) {
- routerConfig = new RouterConfigBuilder()
- .stateStore()
- .metrics()
- .http()
- .build();
- router = new Router();
- router.init(routerConfig);
- router.setRouterId("routerId");
- router.start();
- stateStore = router.getStateStore();
-
- membershipStore =
- stateStore.getRegisteredRecordStore(MembershipStore.class);
- routerStore = stateStore.getRegisteredRecordStore(RouterStore.class);
-
- // Read all data and load all caches
- waitStateStore(stateStore, 10000);
- createFixtures();
- stateStore.refreshCaches(true);
- Thread.sleep(1000);
- }
- }
-
- @After
- public void tearDownBase() throws IOException {
- if (router != null) {
- router.stop();
- router.close();
- router = null;
- }
- }
-
- private void createFixtures() throws IOException {
- // Clear all records
- clearAllRecords(stateStore);
-
- nameservices = new ArrayList<>();
- nameservices.add(NAMESERVICES[0]);
- nameservices.add(NAMESERVICES[1]);
-
- // 2 NNs per NS
- activeMemberships = new ArrayList<>();
- standbyMemberships = new ArrayList<>();
-
- for (String nameservice : nameservices) {
- MembershipState namenode1 = createMockRegistrationForNamenode(
- nameservice, NAMENODES[0], FederationNamenodeServiceState.ACTIVE);
- NamenodeHeartbeatRequest request1 =
- NamenodeHeartbeatRequest.newInstance(namenode1);
- assertTrue(membershipStore.namenodeHeartbeat(request1).getResult());
- activeMemberships.add(namenode1);
-
- MembershipState namenode2 = createMockRegistrationForNamenode(
- nameservice, NAMENODES[1], FederationNamenodeServiceState.STANDBY);
- NamenodeHeartbeatRequest request2 =
- NamenodeHeartbeatRequest.newInstance(namenode2);
- assertTrue(membershipStore.namenodeHeartbeat(request2).getResult());
- standbyMemberships.add(namenode2);
- }
-
- // Add 2 mount table memberships
- mockMountTable = createMockMountTable(nameservices);
- synchronizeRecords(stateStore, mockMountTable, MountTable.class);
-
- // Add 2 router memberships in addition to the running router.
- long t1 = Time.now();
- mockRouters = new ArrayList<>();
- RouterState router1 = RouterState.newInstance(
- "router1", t1, RouterServiceState.RUNNING);
- router1.setStateStoreVersion(StateStoreVersion.newInstance(
- t1 - 1000, t1 - 2000));
- RouterHeartbeatRequest heartbeatRequest =
- RouterHeartbeatRequest.newInstance(router1);
- assertTrue(routerStore.routerHeartbeat(heartbeatRequest).getStatus());
-
- GetRouterRegistrationRequest getRequest =
- GetRouterRegistrationRequest.newInstance("router1");
- GetRouterRegistrationResponse getResponse =
- routerStore.getRouterRegistration(getRequest);
- RouterState routerState1 = getResponse.getRouter();
- mockRouters.add(routerState1);
-
- long t2 = Time.now();
- RouterState router2 = RouterState.newInstance(
- "router2", t2, RouterServiceState.RUNNING);
- router2.setStateStoreVersion(StateStoreVersion.newInstance(
- t2 - 6000, t2 - 7000));
- heartbeatRequest.setRouter(router2);
- assertTrue(routerStore.routerHeartbeat(heartbeatRequest).getStatus());
- getRequest.setRouterId("router2");
- getResponse = routerStore.getRouterRegistration(getRequest);
- RouterState routerState2 = getResponse.getRouter();
- mockRouters.add(routerState2);
- }
-
- protected Router getRouter() {
- return router;
- }
-
- protected List<MountTable> getMockMountTable() {
- return mockMountTable;
- }
-
- protected List<MembershipState> getActiveMemberships() {
- return activeMemberships;
- }
-
- protected List<MembershipState> getStandbyMemberships() {
- return standbyMemberships;
- }
-
- protected List<String> getNameservices() {
- return nameservices;
- }
-
- protected List<RouterState> getMockRouters() {
- return mockRouters;
- }
-
- protected StateStoreService getStateStore() {
- return stateStore;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
deleted file mode 100644
index e38443e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
+++ /dev/null
@@ -1,528 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.resolver;
-
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.server.federation.router.Router;
-import org.apache.hadoop.hdfs.server.federation.store.MountTableStore;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Test the {@link MountTableStore} from the {@link Router}.
- */
-public class TestMountTableResolver {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(TestMountTableResolver.class);
-
- private static final int TEST_MAX_CACHE_SIZE = 10;
-
- private MountTableResolver mountTable;
-
- private Map<String, String> getMountTableEntry(
- String subcluster, String path) {
- Map<String, String> ret = new HashMap<>();
- ret.put(subcluster, path);
- return ret;
- }
-
- /**
- * Setup the mount table.
- * / -> 1:/
- * __tmp -> 2:/tmp
- * __user -> 3:/user
- * ____a -> 2:/user/test
- * ______demo
- * ________test
- * __________a -> 1:/user/test
- * __________b -> 3:/user/test
- * ____b
- * ______file1.txt -> 4:/user/file1.txt
- * __usr
- * ____bin -> 2:/bin
- * __readonly -> 2:/tmp
- *
- * @throws IOException If it cannot set the mount table.
- */
- private void setupMountTable() throws IOException {
- Configuration conf = new Configuration();
- conf.setInt(
- FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE, TEST_MAX_CACHE_SIZE);
- conf.setStrings(DFS_ROUTER_DEFAULT_NAMESERVICE, "0");
- mountTable = new MountTableResolver(conf);
-
- // Root mount point
- Map<String, String> map = getMountTableEntry("1", "/");
- mountTable.addEntry(MountTable.newInstance("/", map));
-
- // /tmp
- map = getMountTableEntry("2", "/");
- mountTable.addEntry(MountTable.newInstance("/tmp", map));
-
- // /user
- map = getMountTableEntry("3", "/user");
- mountTable.addEntry(MountTable.newInstance("/user", map));
-
- // /usr/bin
- map = getMountTableEntry("2", "/bin");
- mountTable.addEntry(MountTable.newInstance("/usr/bin", map));
-
- // /user/a
- map = getMountTableEntry("2", "/user/test");
- mountTable.addEntry(MountTable.newInstance("/user/a", map));
-
- // /user/b/file1.txt
- map = getMountTableEntry("4", "/user/file1.txt");
- mountTable.addEntry(MountTable.newInstance("/user/b/file1.txt", map));
-
- // /user/a/demo/test/a
- map = getMountTableEntry("1", "/user/test");
- mountTable.addEntry(MountTable.newInstance("/user/a/demo/test/a", map));
-
- // /user/a/demo/test/b
- map = getMountTableEntry("3", "/user/test");
- mountTable.addEntry(MountTable.newInstance("/user/a/demo/test/b", map));
-
- // /readonly
- map = getMountTableEntry("2", "/tmp");
- MountTable readOnlyEntry = MountTable.newInstance("/readonly", map);
- readOnlyEntry.setReadOnly(true);
- mountTable.addEntry(readOnlyEntry);
- }
-
- @Before
- public void setup() throws IOException {
- setupMountTable();
- }
-
- @Test
- public void testDestination() throws IOException {
-
- // Check files
- assertEquals("1->/tesfile1.txt",
- mountTable.getDestinationForPath("/tesfile1.txt").toString());
-
- assertEquals("3->/user/testfile2.txt",
- mountTable.getDestinationForPath("/user/testfile2.txt").toString());
-
- assertEquals("2->/user/test/testfile3.txt",
- mountTable.getDestinationForPath("/user/a/testfile3.txt").toString());
-
- assertEquals("3->/user/b/testfile4.txt",
- mountTable.getDestinationForPath("/user/b/testfile4.txt").toString());
-
- assertEquals("1->/share/file5.txt",
- mountTable.getDestinationForPath("/share/file5.txt").toString());
-
- assertEquals("2->/bin/file7.txt",
- mountTable.getDestinationForPath("/usr/bin/file7.txt").toString());
-
- assertEquals("1->/usr/file8.txt",
- mountTable.getDestinationForPath("/usr/file8.txt").toString());
-
- assertEquals("2->/user/test/demo/file9.txt",
- mountTable.getDestinationForPath("/user/a/demo/file9.txt").toString());
-
- // Check folders
- assertEquals("3->/user/testfolder",
- mountTable.getDestinationForPath("/user/testfolder").toString());
-
- assertEquals("2->/user/test/b",
- mountTable.getDestinationForPath("/user/a/b").toString());
-
- assertEquals("3->/user/test/a",
- mountTable.getDestinationForPath("/user/test/a").toString());
-
- assertEquals("2->/tmp/tesfile1.txt",
- mountTable.getDestinationForPath("/readonly/tesfile1.txt").toString());
-
- }
-
- private void compareLists(List<String> list1, String[] list2) {
- assertEquals(list1.size(), list2.length);
- for (String item : list2) {
- assertTrue(list1.contains(item));
- }
- }
-
- @Test
- public void testGetMountPoint() throws IOException {
- // Check get the mount table entry for a path
- MountTable mtEntry;
- mtEntry = mountTable.getMountPoint("/");
- assertTrue(mtEntry.getSourcePath().equals("/"));
-
- mtEntry = mountTable.getMountPoint("/user");
- assertTrue(mtEntry.getSourcePath().equals("/user"));
-
- mtEntry = mountTable.getMountPoint("/user/a");
- assertTrue(mtEntry.getSourcePath().equals("/user/a"));
-
- mtEntry = mountTable.getMountPoint("/user/a/");
- assertTrue(mtEntry.getSourcePath().equals("/user/a"));
-
- mtEntry = mountTable.getMountPoint("/user/a/11");
- assertTrue(mtEntry.getSourcePath().equals("/user/a"));
-
- mtEntry = mountTable.getMountPoint("/user/a1");
- assertTrue(mtEntry.getSourcePath().equals("/user"));
- }
-
- @Test
- public void testGetMountPoints() throws IOException {
-
- // Check getting all mount points (virtual and real) beneath a path
- List<String> mounts = mountTable.getMountPoints("/");
- assertEquals(4, mounts.size());
- compareLists(mounts, new String[] {"tmp", "user", "usr", "readonly"});
-
- mounts = mountTable.getMountPoints("/user");
- assertEquals(2, mounts.size());
- compareLists(mounts, new String[] {"a", "b"});
-
- mounts = mountTable.getMountPoints("/user/a");
- assertEquals(1, mounts.size());
- compareLists(mounts, new String[] {"demo"});
-
- mounts = mountTable.getMountPoints("/user/a/demo");
- assertEquals(1, mounts.size());
- compareLists(mounts, new String[] {"test"});
-
- mounts = mountTable.getMountPoints("/user/a/demo/test");
- assertEquals(2, mounts.size());
- compareLists(mounts, new String[] {"a", "b"});
-
- mounts = mountTable.getMountPoints("/tmp");
- assertEquals(0, mounts.size());
-
- mounts = mountTable.getMountPoints("/t");
- assertNull(mounts);
-
- mounts = mountTable.getMountPoints("/unknownpath");
- assertNull(mounts);
- }
-
- private void compareRecords(List<MountTable> list1, String[] list2) {
- assertEquals(list1.size(), list2.length);
- for (String item : list2) {
- for (MountTable record : list1) {
- if (record.getSourcePath().equals(item)) {
- return;
- }
- }
- }
- fail();
- }
-
- @Test
- public void testGetMounts() throws IOException {
-
- // Check listing the mount table records at or beneath a path
- List<MountTable> records = mountTable.getMounts("/");
- assertEquals(9, records.size());
- compareRecords(records, new String[] {"/", "/tmp", "/user", "/usr/bin",
- "user/a", "/user/a/demo/a", "/user/a/demo/b", "/user/b/file1.txt",
- "readonly"});
-
- records = mountTable.getMounts("/user");
- assertEquals(5, records.size());
- compareRecords(records, new String[] {"/user", "/user/a/demo/a",
- "/user/a/demo/b", "user/a", "/user/b/file1.txt"});
-
- records = mountTable.getMounts("/user/a");
- assertEquals(3, records.size());
- compareRecords(records,
- new String[] {"/user/a/demo/a", "/user/a/demo/b", "/user/a"});
-
- records = mountTable.getMounts("/tmp");
- assertEquals(1, records.size());
- compareRecords(records, new String[] {"/tmp"});
-
- records = mountTable.getMounts("/readonly");
- assertEquals(1, records.size());
- compareRecords(records, new String[] {"/readonly"});
- assertTrue(records.get(0).isReadOnly());
- }
-
- @Test
- public void testRemoveSubTree()
- throws UnsupportedOperationException, IOException {
-
- // 3 mount points are present /tmp, /user, /usr
- compareLists(mountTable.getMountPoints("/"),
- new String[] {"user", "usr", "tmp", "readonly"});
-
- // /tmp currently points to namespace 2
- assertEquals("2", mountTable.getDestinationForPath("/tmp/testfile.txt")
- .getDefaultLocation().getNameserviceId());
-
- // Remove tmp
- mountTable.removeEntry("/tmp");
-
- // Now 2 mount points are present /user, /usr
- compareLists(mountTable.getMountPoints("/"),
- new String[] {"user", "usr", "readonly"});
-
- // /tmp no longer exists, uses default namespace for mapping /
- assertEquals("1", mountTable.getDestinationForPath("/tmp/testfile.txt")
- .getDefaultLocation().getNameserviceId());
- }
-
- @Test
- public void testRemoveVirtualNode()
- throws UnsupportedOperationException, IOException {
-
- // 3 mount points are present /tmp, /user, /usr
- compareLists(mountTable.getMountPoints("/"),
- new String[] {"user", "usr", "tmp", "readonly"});
-
- // /usr is virtual, uses namespace 1->/
- assertEquals("1", mountTable.getDestinationForPath("/usr/testfile.txt")
- .getDefaultLocation().getNameserviceId());
-
- // Attempt to remove /usr
- mountTable.removeEntry("/usr");
-
- // Verify the remove failed
- compareLists(mountTable.getMountPoints("/"),
- new String[] {"user", "usr", "tmp", "readonly"});
- }
-
- @Test
- public void testRemoveLeafNode()
- throws UnsupportedOperationException, IOException {
-
- // /user/a/demo/test/a currently points to namespace 1
- assertEquals("1", mountTable.getDestinationForPath("/user/a/demo/test/a")
- .getDefaultLocation().getNameserviceId());
-
- // Remove /user/a/demo/test/a
- mountTable.removeEntry("/user/a/demo/test/a");
-
- // Now /user/a/demo/test/a points to namespace 2 using the entry for /user/a
- assertEquals("2", mountTable.getDestinationForPath("/user/a/demo/test/a")
- .getDefaultLocation().getNameserviceId());
-
- // Verify the virtual node at /user/a/demo still exists and was not deleted
- compareLists(mountTable.getMountPoints("/user/a"), new String[] {"demo"});
-
- // Verify the sibling node was unaffected and still points to ns 3
- assertEquals("3", mountTable.getDestinationForPath("/user/a/demo/test/b")
- .getDefaultLocation().getNameserviceId());
- }
-
- @Test
- public void testRefreshEntries()
- throws UnsupportedOperationException, IOException {
-
- // Initial table loaded
- testDestination();
- assertEquals(9, mountTable.getMounts("/").size());
-
- // Replace table with /1 and /2
- List<MountTable> records = new ArrayList<>();
- Map<String, String> map1 = getMountTableEntry("1", "/");
- records.add(MountTable.newInstance("/1", map1));
- Map<String, String> map2 = getMountTableEntry("2", "/");
- records.add(MountTable.newInstance("/2", map2));
- mountTable.refreshEntries(records);
-
- // Verify addition
- PathLocation destination1 = mountTable.getDestinationForPath("/1");
- RemoteLocation defaultLoc1 = destination1.getDefaultLocation();
- assertEquals("1", defaultLoc1.getNameserviceId());
-
- PathLocation destination2 = mountTable.getDestinationForPath("/2");
- RemoteLocation defaultLoc2 = destination2.getDefaultLocation();
- assertEquals("2", defaultLoc2.getNameserviceId());
-
- // Verify existing entries were removed
- assertEquals(2, mountTable.getMounts("/").size());
- boolean assertionThrown = false;
- try {
- testDestination();
- fail();
- } catch (AssertionError e) {
- // The / entry was removed, so it triggers an exception
- assertionThrown = true;
- }
- assertTrue(assertionThrown);
- }
-
- @Test
- public void testMountTableScalability() throws IOException {
-
- List<MountTable> emptyList = new ArrayList<>();
- mountTable.refreshEntries(emptyList);
-
- // Add 100,000 entries in flat list
- for (int i = 0; i < 100000; i++) {
- Map<String, String> map = getMountTableEntry("1", "/" + i);
- MountTable record = MountTable.newInstance("/" + i, map);
- mountTable.addEntry(record);
- if (i % 10000 == 0) {
- LOG.info("Adding flat mount record {}: {}", i, record);
- }
- }
-
- assertEquals(100000, mountTable.getMountPoints("/").size());
- assertEquals(100000, mountTable.getMounts("/").size());
-
- // Add 1000 entries in deep list
- mountTable.refreshEntries(emptyList);
- String parent = "/";
- for (int i = 0; i < 1000; i++) {
- final int index = i;
- Map<String, String> map = getMountTableEntry("1", "/" + index);
- if (i > 0) {
- parent = parent + "/";
- }
- parent = parent + i;
- MountTable record = MountTable.newInstance(parent, map);
- mountTable.addEntry(record);
- }
-
- assertEquals(1, mountTable.getMountPoints("/").size());
- assertEquals(1000, mountTable.getMounts("/").size());
-
- // Add 100,000 entries in deep and wide tree
- mountTable.refreshEntries(emptyList);
- Random rand = new Random();
- parent = "/" + Integer.toString(rand.nextInt());
- int numRootTrees = 1;
- for (int i = 0; i < 100000; i++) {
- final int index = i;
- Map<String, String> map = getMountTableEntry("1", "/" + index);
- parent = parent + "/" + i;
- if (parent.length() > 2000) {
- // Start new tree
- parent = "/" + Integer.toString(rand.nextInt());
- numRootTrees++;
- }
- MountTable record = MountTable.newInstance(parent, map);
- mountTable.addEntry(record);
- }
-
- assertEquals(numRootTrees, mountTable.getMountPoints("/").size());
- assertEquals(100000, mountTable.getMounts("/").size());
- }
-
- @Test
- public void testUpdate() throws IOException {
-
- // Add entry to update later
- Map<String, String> map = getMountTableEntry("1", "/");
- mountTable.addEntry(MountTable.newInstance("/testupdate", map));
-
- MountTable entry = mountTable.getMountPoint("/testupdate");
- List<RemoteLocation> dests = entry.getDestinations();
- assertEquals(1, dests.size());
- RemoteLocation dest = dests.get(0);
- assertEquals("1", dest.getNameserviceId());
-
- // Update entry
- Collection<MountTable> entries = Collections.singletonList(
- MountTable.newInstance("/testupdate", getMountTableEntry("2", "/")));
- mountTable.refreshEntries(entries);
-
- MountTable entry1 = mountTable.getMountPoint("/testupdate");
- List<RemoteLocation> dests1 = entry1.getDestinations();
- assertEquals(1, dests1.size());
- RemoteLocation dest1 = dests1.get(0);
- assertEquals("2", dest1.getNameserviceId());
-
- // Remove the entry to test updates and check
- mountTable.removeEntry("/testupdate");
- MountTable entry2 = mountTable.getMountPoint("/testupdate");
- assertNull(entry2);
- }
-
- @Test
- public void testCacheCleaning() throws Exception {
- for (int i = 0; i < 1000; i++) {
- String filename = String.format("/user/a/file-%04d.txt", i);
- mountTable.getDestinationForPath(filename);
- }
- long cacheSize = mountTable.getCacheSize();
- assertTrue(cacheSize <= TEST_MAX_CACHE_SIZE);
- }
-
- @Test
- public void testLocationCache() throws Exception {
- List<MountTable> entries = new ArrayList<>();
-
- // Add entry and test location cache
- Map<String, String> map1 = getMountTableEntry("1", "/testlocationcache");
- MountTable entry1 = MountTable.newInstance("/testlocationcache", map1);
- entries.add(entry1);
-
- Map<String, String> map2 = getMountTableEntry("2",
- "/anothertestlocationcache");
- MountTable entry2 = MountTable.newInstance("/anothertestlocationcache",
- map2);
- entries.add(entry2);
- mountTable.refreshEntries(entries);
- assertEquals("1->/testlocationcache",
- mountTable.getDestinationForPath("/testlocationcache").toString());
- assertEquals("2->/anothertestlocationcache",
- mountTable.getDestinationForPath("/anothertestlocationcache")
- .toString());
-
- // Remove the entry1
- entries.remove(entry1);
- mountTable.refreshEntries(entries);
-
- // Add the default location and test location cache
- assertEquals("0->/testlocationcache",
- mountTable.getDestinationForPath("/testlocationcache").toString());
-
- // Add the entry again but mount to another ns
- Map<String, String> map3 = getMountTableEntry("3", "/testlocationcache");
- MountTable entry3 = MountTable.newInstance("/testlocationcache", map3);
- entries.add(entry3);
- mountTable.refreshEntries(entries);
-
- // Ensure location cache update correctly
- assertEquals("3->/testlocationcache",
- mountTable.getDestinationForPath("/testlocationcache").toString());
-
- // Cleanup before exit
- mountTable.removeEntry("/testlocationcache");
- mountTable.removeEntry("/anothertestlocationcache");
- }
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMultipleDestinationResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMultipleDestinationResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMultipleDestinationResolver.java
deleted file mode 100644
index 3915c56..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMultipleDestinationResolver.java
+++ /dev/null
@@ -1,419 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.resolver;
-
-import static org.apache.hadoop.hdfs.server.federation.resolver.order.HashResolver.extractTempFileName;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-import java.util.TreeSet;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Test the multiple destination resolver.
- */
-public class TestMultipleDestinationResolver {
-
- private MultipleDestinationMountTableResolver resolver;
-
- @Before
- public void setup() throws IOException {
- Configuration conf = new Configuration();
- resolver = new MultipleDestinationMountTableResolver(conf, null);
-
- // We manually point /tmp to only subcluster0
- Map<String, String> map1 = new HashMap<>();
- map1.put("subcluster0", "/tmp");
- resolver.addEntry(MountTable.newInstance("/tmp", map1));
-
- // We manually point / to subcluster0,1,2 with default order (hash)
- Map<String, String> mapDefault = new HashMap<>();
- mapDefault.put("subcluster0", "/");
- mapDefault.put("subcluster1", "/");
- mapDefault.put("subcluster2", "/");
- MountTable defaultEntry = MountTable.newInstance("/", mapDefault);
- resolver.addEntry(defaultEntry);
-
- // We manually point /hash to subcluster0,1,2 with hashing
- Map<String, String> mapHash = new HashMap<>();
- mapHash.put("subcluster0", "/hash");
- mapHash.put("subcluster1", "/hash");
- mapHash.put("subcluster2", "/hash");
- MountTable hashEntry = MountTable.newInstance("/hash", mapHash);
- hashEntry.setDestOrder(DestinationOrder.HASH);
- resolver.addEntry(hashEntry);
-
- // We manually point /hashall to subcluster0,1,2 with hashing (full tree)
- Map<String, String> mapHashAll = new HashMap<>();
- mapHashAll.put("subcluster0", "/hashall");
- mapHashAll.put("subcluster1", "/hashall");
- mapHashAll.put("subcluster2", "/hashall");
- MountTable hashEntryAll = MountTable.newInstance("/hashall", mapHashAll);
- hashEntryAll.setDestOrder(DestinationOrder.HASH_ALL);
- resolver.addEntry(hashEntryAll);
-
- // We point /local to subclusters 0, 1, 2 with the local order
- Map<String, String> mapLocal = new HashMap<>();
- mapLocal.put("subcluster0", "/local");
- mapLocal.put("subcluster1", "/local");
- mapLocal.put("subcluster2", "/local");
- MountTable localEntry = MountTable.newInstance("/local", mapLocal);
- localEntry.setDestOrder(DestinationOrder.LOCAL);
- resolver.addEntry(localEntry);
-
- // We point /random to subclusters 0, 1, 2 with the random order
- Map<String, String> mapRandom = new HashMap<>();
- mapRandom.put("subcluster0", "/random");
- mapRandom.put("subcluster1", "/random");
- mapRandom.put("subcluster2", "/random");
- MountTable randomEntry = MountTable.newInstance("/random", mapRandom);
- randomEntry.setDestOrder(DestinationOrder.RANDOM);
- resolver.addEntry(randomEntry);
-
- // Read only mount point
- Map<String, String> mapReadOnly = new HashMap<>();
- mapReadOnly.put("subcluster0", "/readonly");
- mapReadOnly.put("subcluster1", "/readonly");
- mapReadOnly.put("subcluster2", "/readonly");
- MountTable readOnlyEntry = MountTable.newInstance("/readonly", mapReadOnly);
- readOnlyEntry.setReadOnly(true);
- resolver.addEntry(readOnlyEntry);
- }
-
- @Test
- public void testHashEqualDistribution() throws IOException {
- // First level
- testEvenDistribution("/hash");
- testEvenDistribution("/hash/folder0", false);
-
- // All levels
- testEvenDistribution("/hashall");
- testEvenDistribution("/hashall/folder0");
- }
-
- @Test
- public void testHashAll() throws IOException {
- // Files should be spread across subclusters
- PathLocation dest0 = resolver.getDestinationForPath("/hashall/file0.txt");
- assertDest("subcluster0", dest0);
- PathLocation dest1 = resolver.getDestinationForPath("/hashall/file1.txt");
- assertDest("subcluster1", dest1);
-
- // Files within folder should be spread across subclusters
- PathLocation dest2 = resolver.getDestinationForPath("/hashall/folder0");
- assertDest("subcluster2", dest2);
- PathLocation dest3 = resolver.getDestinationForPath(
- "/hashall/folder0/file0.txt");
- assertDest("subcluster1", dest3);
- PathLocation dest4 = resolver.getDestinationForPath(
- "/hashall/folder0/file1.txt");
- assertDest("subcluster0", dest4);
-
- PathLocation dest5 = resolver.getDestinationForPath(
- "/hashall/folder0/folder0/file0.txt");
- assertDest("subcluster1", dest5);
- PathLocation dest6 = resolver.getDestinationForPath(
- "/hashall/folder0/folder0/file1.txt");
- assertDest("subcluster1", dest6);
- PathLocation dest7 = resolver.getDestinationForPath(
- "/hashall/folder0/folder0/file2.txt");
- assertDest("subcluster0", dest7);
-
- PathLocation dest8 = resolver.getDestinationForPath("/hashall/folder1");
- assertDest("subcluster1", dest8);
- PathLocation dest9 = resolver.getDestinationForPath(
- "/hashall/folder1/file0.txt");
- assertDest("subcluster0", dest9);
- PathLocation dest10 = resolver.getDestinationForPath(
- "/hashall/folder1/file1.txt");
- assertDest("subcluster1", dest10);
-
- PathLocation dest11 = resolver.getDestinationForPath("/hashall/folder2");
- assertDest("subcluster2", dest11);
- PathLocation dest12 = resolver.getDestinationForPath(
- "/hashall/folder2/file0.txt");
- assertDest("subcluster0", dest12);
- PathLocation dest13 = resolver.getDestinationForPath(
- "/hashall/folder2/file1.txt");
- assertDest("subcluster0", dest13);
- PathLocation dest14 = resolver.getDestinationForPath(
- "/hashall/folder2/file2.txt");
- assertDest("subcluster1", dest14);
- }
-
- @Test
- public void testHashFirst() throws IOException {
- PathLocation dest0 = resolver.getDestinationForPath("/hashall/file0.txt");
- assertDest("subcluster0", dest0);
- PathLocation dest1 = resolver.getDestinationForPath("/hashall/file1.txt");
- assertDest("subcluster1", dest1);
-
- // All these must be in the same location: subcluster0
- PathLocation dest2 = resolver.getDestinationForPath("/hash/folder0");
- assertDest("subcluster0", dest2);
- PathLocation dest3 = resolver.getDestinationForPath(
- "/hash/folder0/file0.txt");
- assertDest("subcluster0", dest3);
- PathLocation dest4 = resolver.getDestinationForPath(
- "/hash/folder0/file1.txt");
- assertDest("subcluster0", dest4);
-
- PathLocation dest5 = resolver.getDestinationForPath(
- "/hash/folder0/folder0/file0.txt");
- assertDest("subcluster0", dest5);
- PathLocation dest6 = resolver.getDestinationForPath(
- "/hash/folder0/folder0/file1.txt");
- assertDest("subcluster0", dest6);
-
- // All these must be in the same location: subcluster2
- PathLocation dest7 = resolver.getDestinationForPath("/hash/folder1");
- assertDest("subcluster2", dest7);
- PathLocation dest8 = resolver.getDestinationForPath(
- "/hash/folder1/file0.txt");
- assertDest("subcluster2", dest8);
- PathLocation dest9 = resolver.getDestinationForPath(
- "/hash/folder1/file1.txt");
- assertDest("subcluster2", dest9);
-
- // All these must be in the same location: subcluster2
- PathLocation dest10 = resolver.getDestinationForPath("/hash/folder2");
- assertDest("subcluster2", dest10);
- PathLocation dest11 = resolver.getDestinationForPath(
- "/hash/folder2/file0.txt");
- assertDest("subcluster2", dest11);
- PathLocation dest12 = resolver.getDestinationForPath(
- "/hash/folder2/file1.txt");
- assertDest("subcluster2", dest12);
- }
-
- @Test
- public void testRandomEqualDistribution() throws IOException {
- testEvenDistribution("/random");
- }
-
- @Test
- public void testSingleDestination() throws IOException {
- // All the files in /tmp should be in subcluster0
- for (int f = 0; f < 100; f++) {
- String filename = "/tmp/b/c/file" + f + ".txt";
- PathLocation destination = resolver.getDestinationForPath(filename);
- RemoteLocation loc = destination.getDefaultLocation();
- assertEquals("subcluster0", loc.getNameserviceId());
- assertEquals(filename, loc.getDest());
- }
- }
-
- @Test
- public void testResolveSubdirectories() throws Exception {
- // Simulate a testdir under a multi-destination mount.
- Random r = new Random();
- String testDir = "/sort/testdir" + r.nextInt();
- String file1 = testDir + "/file1" + r.nextInt();
- String file2 = testDir + "/file2" + r.nextInt();
-
- // Verify both files resolve to the same namespace as the parent dir.
- PathLocation testDirLocation = resolver.getDestinationForPath(testDir);
- RemoteLocation defaultLoc = testDirLocation.getDefaultLocation();
- String testDirNamespace = defaultLoc.getNameserviceId();
-
- PathLocation file1Location = resolver.getDestinationForPath(file1);
- RemoteLocation defaultLoc1 = file1Location.getDefaultLocation();
- assertEquals(testDirNamespace, defaultLoc1.getNameserviceId());
-
- PathLocation file2Location = resolver.getDestinationForPath(file2);
- RemoteLocation defaultLoc2 = file2Location.getDefaultLocation();
- assertEquals(testDirNamespace, defaultLoc2.getNameserviceId());
- }
-
- @Test
- public void testExtractTempFileName() {
- for (String teststring : new String[] {
- "testfile1.txt.COPYING",
- "testfile1.txt._COPYING_",
- "testfile1.txt._COPYING_.attempt_1486662804109_0055_m_000042_0",
- "testfile1.txt.tmp",
- "_temp/testfile1.txt",
- "_temporary/testfile1.txt.af77e2ab-4bc5-4959-ae08-299c880ee6b8",
- "_temporary/0/_temporary/attempt_201706281636_0007_m_000003_46/" +
- "testfile1.txt" }) {
- String finalName = extractTempFileName(teststring);
- assertEquals("testfile1.txt", finalName);
- }
-
- // False cases
- assertEquals(
- "file1.txt.COPYING1", extractTempFileName("file1.txt.COPYING1"));
- assertEquals("file1.txt.tmp2", extractTempFileName("file1.txt.tmp2"));
-
- // Speculation patterns
- String finalName = extractTempFileName(
- "_temporary/part-00007.af77e2ab-4bc5-4959-ae08-299c880ee6b8");
- assertEquals("part-00007", finalName);
- finalName = extractTempFileName(
- "_temporary/0/_temporary/attempt_201706281636_0007_m_000003_46/" +
- "part-00003");
- assertEquals("part-00003", finalName);
-
- // Subfolders
- finalName = extractTempFileName("folder0/testfile1.txt._COPYING_");
- assertEquals("folder0/testfile1.txt", finalName);
- finalName = extractTempFileName(
- "folder0/folder1/testfile1.txt._COPYING_");
- assertEquals("folder0/folder1/testfile1.txt", finalName);
- finalName = extractTempFileName(
- "processedHrsData.txt/_temporary/0/_temporary/" +
- "attempt_201706281636_0007_m_000003_46/part-00003");
- assertEquals("processedHrsData.txt/part-00003", finalName);
- }
-
- @Test
- public void testReadOnly() throws IOException {
- MountTable mount = resolver.getMountPoint("/readonly");
- assertTrue(mount.isReadOnly());
-
- PathLocation dest0 = resolver.getDestinationForPath("/readonly/file0.txt");
- assertDest("subcluster1", dest0);
- PathLocation dest1 = resolver.getDestinationForPath("/readonly/file1.txt");
- assertDest("subcluster2", dest1);
-
- // All these must be in the same location: subcluster0
- PathLocation dest2 = resolver.getDestinationForPath("/readonly/folder0");
- assertDest("subcluster1", dest2);
- PathLocation dest3 = resolver.getDestinationForPath(
- "/readonly/folder0/file0.txt");
- assertDest("subcluster1", dest3);
- PathLocation dest4 = resolver.getDestinationForPath(
- "/readonly/folder0/file1.txt");
- assertDest("subcluster1", dest4);
-
- PathLocation dest5 = resolver.getDestinationForPath(
- "/readonly/folder0/folder0/file0.txt");
- assertDest("subcluster1", dest5);
- PathLocation dest6 = resolver.getDestinationForPath(
- "/readonly/folder0/folder0/file1.txt");
- assertDest("subcluster1", dest6);
-
- // All these must be in the same location: subcluster2
- PathLocation dest7 = resolver.getDestinationForPath("/readonly/folder1");
- assertDest("subcluster2", dest7);
- PathLocation dest8 = resolver.getDestinationForPath(
- "/readonly/folder1/file0.txt");
- assertDest("subcluster2", dest8);
- PathLocation dest9 = resolver.getDestinationForPath(
- "/readonly/folder1/file1.txt");
- assertDest("subcluster2", dest9);
-
- // All these must be in the same location: subcluster2
- PathLocation dest10 = resolver.getDestinationForPath("/readonly/folder2");
- assertDest("subcluster1", dest10);
- PathLocation dest11 = resolver.getDestinationForPath(
- "/readonly/folder2/file0.txt");
- assertDest("subcluster1", dest11);
- PathLocation dest12 = resolver.getDestinationForPath(
- "/readonly/folder2/file1.txt");
- assertDest("subcluster1", dest12);
- }
-
- @Test
- public void testLocalResolver() throws IOException {
- PathLocation dest0 =
- resolver.getDestinationForPath("/local/folder0/file0.txt");
- assertDest("subcluster0", dest0);
- }
-
- @Test
- public void testRandomResolver() throws IOException {
- Set<String> destinations = new HashSet<>();
- for (int i = 0; i < 30; i++) {
- PathLocation dest =
- resolver.getDestinationForPath("/random/folder0/file0.txt");
- RemoteLocation firstDest = dest.getDestinations().get(0);
- String nsId = firstDest.getNameserviceId();
- destinations.add(nsId);
- }
- assertEquals(3, destinations.size());
- }
-
- /**
- * Test that a path has files distributed across destinations evenly.
- * @param path Path to check.
- * @throws IOException
- */
- private void testEvenDistribution(final String path) throws IOException {
- testEvenDistribution(path, true);
- }
-
- /**
- * Test that a path has files distributed across destinations evenly or not.
- * @param path Path to check.
- * @param even If the distribution should be even or not.
- * @throws IOException If it cannot check it.
- */
- private void testEvenDistribution(final String path, final boolean even)
- throws IOException {
-
- // Subcluster -> Files
- Map<String, Set<String>> results = new HashMap<>();
- for (int f = 0; f < 10000; f++) {
- String filename = path + "/file" + f + ".txt";
- PathLocation destination = resolver.getDestinationForPath(filename);
- RemoteLocation loc = destination.getDefaultLocation();
- assertEquals(filename, loc.getDest());
-
- String nsId = loc.getNameserviceId();
- if (!results.containsKey(nsId)) {
- results.put(nsId, new TreeSet<>());
- }
- results.get(nsId).add(filename);
- }
-
- if (!even) {
- // All files should be in one subcluster
- assertEquals(1, results.size());
- } else {
- // Files should be distributed somewhat evenly
- assertEquals(3, results.size());
- int count = 0;
- for (Set<String> files : results.values()) {
- count = count + files.size();
- }
- int avg = count / results.keySet().size();
- for (Set<String> files : results.values()) {
- int filesCount = files.size();
- // Check that the count in each namespace is within 20% of avg
- assertTrue(filesCount > 0);
- assertTrue(Math.abs(filesCount - avg) < (avg / 5));
- }
- }
- }
-
- private static void assertDest(String expectedDest, PathLocation loc) {
- assertEquals(expectedDest, loc.getDestinations().get(0).getNameserviceId());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestNamenodeResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestNamenodeResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestNamenodeResolver.java
deleted file mode 100644
index 2d74505..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestNamenodeResolver.java
+++ /dev/null
@@ -1,284 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.resolver;
-
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMENODES;
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMESERVICES;
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.ROUTERS;
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createNamenodeReport;
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyException;
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.clearRecords;
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.getStateStoreConfiguration;
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.newStateStore;
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.waitStateStore;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreUnavailableException;
-import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- * Test the basic {@link ActiveNamenodeResolver} functionality.
- */
-public class TestNamenodeResolver {
-
- private static StateStoreService stateStore;
- private static ActiveNamenodeResolver namenodeResolver;
-
- @BeforeClass
- public static void create() throws Exception {
-
- Configuration conf = getStateStoreConfiguration();
-
- // Reduce expirations to 5 seconds
- conf.setLong(
- DFSConfigKeys.FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS,
- TimeUnit.SECONDS.toMillis(5));
-
- stateStore = newStateStore(conf);
- assertNotNull(stateStore);
-
- namenodeResolver = new MembershipNamenodeResolver(conf, stateStore);
- namenodeResolver.setRouterId(ROUTERS[0]);
- }
-
- @AfterClass
- public static void destroy() throws Exception {
- stateStore.stop();
- stateStore.close();
- }
-
- @Before
- public void setup() throws IOException, InterruptedException {
- // Wait for state store to connect
- stateStore.loadDriver();
- waitStateStore(stateStore, 10000);
-
- // Clear NN registrations
- boolean cleared = clearRecords(stateStore, MembershipState.class);
- assertTrue(cleared);
- }
-
- @Test
- public void testStateStoreDisconnected() throws Exception {
-
- // Add an entry to the store
- NamenodeStatusReport report = createNamenodeReport(
- NAMESERVICES[0], NAMENODES[0], HAServiceState.ACTIVE);
- assertTrue(namenodeResolver.registerNamenode(report));
-
- // Close the data store driver
- stateStore.closeDriver();
- assertFalse(stateStore.isDriverReady());
-
- // Flush the caches
- stateStore.refreshCaches(true);
-
- // Verify commands fail due to no cached data and no state store
- // connectivity.
- List<? extends FederationNamenodeContext> nns =
- namenodeResolver.getNamenodesForBlockPoolId(NAMESERVICES[0]);
- assertNull(nns);
-
- verifyException(namenodeResolver, "registerNamenode",
- StateStoreUnavailableException.class,
- new Class[] {NamenodeStatusReport.class}, new Object[] {report});
- }
-
- /**
- * Verify the first registration on the resolver.
- *
- * @param nsId Nameservice identifier.
- * @param nnId Namenode identifier within the nemeservice.
- * @param resultsCount Number of results expected.
- * @param state Expected state for the first one.
- * @throws IOException If we cannot get the namenodes.
- */
- private void verifyFirstRegistration(String nsId, String nnId,
- int resultsCount, FederationNamenodeServiceState state)
- throws IOException {
- List<? extends FederationNamenodeContext> namenodes =
- namenodeResolver.getNamenodesForNameserviceId(nsId);
- if (resultsCount == 0) {
- assertNull(namenodes);
- } else {
- assertEquals(resultsCount, namenodes.size());
- if (namenodes.size() > 0) {
- FederationNamenodeContext namenode = namenodes.get(0);
- assertEquals(state, namenode.getState());
- assertEquals(nnId, namenode.getNamenodeId());
- }
- }
- }
-
- @Test
- public void testRegistrationExpired()
- throws InterruptedException, IOException {
-
- // Populate the state store with a single NN element
- // 1) ns0:nn0 - Active
- // Wait for the entry to expire without heartbeating
- // Verify the NN entry is not accessible once expired.
- NamenodeStatusReport report = createNamenodeReport(
- NAMESERVICES[0], NAMENODES[0], HAServiceState.ACTIVE);
- assertTrue(namenodeResolver.registerNamenode(report));
-
- // Load cache
- stateStore.refreshCaches(true);
-
- // Verify
- verifyFirstRegistration(
- NAMESERVICES[0], NAMENODES[0], 1,
- FederationNamenodeServiceState.ACTIVE);
-
- // Wait past expiration (set in conf to 5 seconds)
- Thread.sleep(6000);
- // Reload cache
- stateStore.refreshCaches(true);
-
- // Verify entry is now expired and is no longer in the cache
- verifyFirstRegistration(
- NAMESERVICES[0], NAMENODES[0], 0,
- FederationNamenodeServiceState.ACTIVE);
-
- // Heartbeat again, updates dateModified
- assertTrue(namenodeResolver.registerNamenode(report));
- // Reload cache
- stateStore.refreshCaches(true);
-
- // Verify updated entry is marked active again and accessible to RPC server
- verifyFirstRegistration(
- NAMESERVICES[0], NAMENODES[0], 1,
- FederationNamenodeServiceState.ACTIVE);
- }
-
- @Test
- public void testRegistrationNamenodeSelection()
- throws InterruptedException, IOException {
-
- // 1) ns0:nn0 - Active
- // 2) ns0:nn1 - Standby (newest)
- // Verify the selected entry is the active entry
- assertTrue(namenodeResolver.registerNamenode(
- createNamenodeReport(
- NAMESERVICES[0], NAMENODES[0], HAServiceState.ACTIVE)));
- Thread.sleep(100);
- assertTrue(namenodeResolver.registerNamenode(
- createNamenodeReport(
- NAMESERVICES[0], NAMENODES[1], HAServiceState.STANDBY)));
-
- stateStore.refreshCaches(true);
-
- verifyFirstRegistration(
- NAMESERVICES[0], NAMENODES[0], 2,
- FederationNamenodeServiceState.ACTIVE);
-
- // 1) ns0:nn0 - Expired (stale)
- // 2) ns0:nn1 - Standby (newest)
- // Verify the selected entry is the standby entry as the active entry is
- // stale
- assertTrue(namenodeResolver.registerNamenode(
- createNamenodeReport(
- NAMESERVICES[0], NAMENODES[0], HAServiceState.ACTIVE)));
-
- // Expire active registration
- Thread.sleep(6000);
-
- // Refresh standby registration
- assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
- NAMESERVICES[0], NAMENODES[1], HAServiceState.STANDBY)));
-
- // Verify that standby is selected (active is now expired)
- stateStore.refreshCaches(true);
- verifyFirstRegistration(NAMESERVICES[0], NAMENODES[1], 1,
- FederationNamenodeServiceState.STANDBY);
-
- // 1) ns0:nn0 - Active
- // 2) ns0:nn1 - Unavailable (newest)
- // Verify the selected entry is the active entry
- assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
- NAMESERVICES[0], NAMENODES[0], HAServiceState.ACTIVE)));
- Thread.sleep(100);
- assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
- NAMESERVICES[0], NAMENODES[1], null)));
- stateStore.refreshCaches(true);
- verifyFirstRegistration(NAMESERVICES[0], NAMENODES[0], 2,
- FederationNamenodeServiceState.ACTIVE);
-
- // 1) ns0:nn0 - Unavailable (newest)
- // 2) ns0:nn1 - Standby
- // Verify the selected entry is the standby entry
- assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
- NAMESERVICES[0], NAMENODES[1], HAServiceState.STANDBY)));
- Thread.sleep(1000);
- assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
- NAMESERVICES[0], NAMENODES[0], null)));
-
- stateStore.refreshCaches(true);
- verifyFirstRegistration(NAMESERVICES[0], NAMENODES[1], 2,
- FederationNamenodeServiceState.STANDBY);
-
- // 1) ns0:nn0 - Active (oldest)
- // 2) ns0:nn1 - Standby
- // 3) ns0:nn2 - Active (newest)
- // Verify the selected entry is the newest active entry
- assertTrue(namenodeResolver.registerNamenode(
- createNamenodeReport(NAMESERVICES[0], NAMENODES[0], null)));
- Thread.sleep(100);
- assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
- NAMESERVICES[0], NAMENODES[1], HAServiceState.STANDBY)));
- Thread.sleep(100);
- assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
- NAMESERVICES[0], NAMENODES[2], HAServiceState.ACTIVE)));
-
- stateStore.refreshCaches(true);
- verifyFirstRegistration(NAMESERVICES[0], NAMENODES[2], 3,
- FederationNamenodeServiceState.ACTIVE);
-
- // 1) ns0:nn0 - Standby (oldest)
- // 2) ns0:nn1 - Standby (newest)
- // 3) ns0:nn2 - Standby
- // Verify the selected entry is the newest standby entry
- assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
- NAMESERVICES[0], NAMENODES[0], HAServiceState.STANDBY)));
- assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
- NAMESERVICES[0], NAMENODES[2], HAServiceState.STANDBY)));
- Thread.sleep(1500);
- assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
- NAMESERVICES[0], NAMENODES[1], HAServiceState.STANDBY)));
-
- stateStore.refreshCaches(true);
- verifyFirstRegistration(NAMESERVICES[0], NAMENODES[1], 3,
- FederationNamenodeServiceState.STANDBY);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/order/TestLocalResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/order/TestLocalResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/order/TestLocalResolver.java
deleted file mode 100644
index 42ede62..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/order/TestLocalResolver.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.resolver.order;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
-import org.apache.hadoop.hdfs.server.federation.resolver.MultipleDestinationMountTableResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
-import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
-import org.apache.hadoop.hdfs.server.federation.router.Router;
-import org.apache.hadoop.hdfs.server.federation.store.MembershipStore;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-import org.junit.Test;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-
-/**
- * Test the {@link LocalResolver}.
- */
-public class TestLocalResolver {
-
- @Test
- @SuppressWarnings("unchecked")
- public void testLocalResolver() throws IOException {
-
- // Mock the subcluster mapping
- Configuration conf = new Configuration();
- Router router = mock(Router.class);
- StateStoreService stateStore = mock(StateStoreService.class);
- MembershipStore membership = mock(MembershipStore.class);
- when(router.getStateStore()).thenReturn(stateStore);
- when(stateStore.getRegisteredRecordStore(any(Class.class)))
- .thenReturn(membership);
- GetNamenodeRegistrationsResponse response =
- GetNamenodeRegistrationsResponse.newInstance();
- // Set the mapping for each client
- List<MembershipState> records = new LinkedList<>();
- records.add(newMembershipState("client0", "subcluster0"));
- records.add(newMembershipState("client1", "subcluster1"));
- records.add(newMembershipState("client2", "subcluster2"));
- response.setNamenodeMemberships(records);
- when(membership.getNamenodeRegistrations(
- any(GetNamenodeRegistrationsRequest.class))).thenReturn(response);
-
- // Mock the client resolution: it will be anything in sb
- StringBuilder sb = new StringBuilder("clientX");
- LocalResolver localResolver = new LocalResolver(conf, router);
- LocalResolver spyLocalResolver = spy(localResolver);
- doAnswer(new Answer<String>() {
- @Override
- public String answer(InvocationOnMock invocation) throws Throwable {
- return sb.toString();
- }
- }).when(spyLocalResolver).getClientAddr();
-
- // Add the mocks to the resolver
- MultipleDestinationMountTableResolver resolver =
- new MultipleDestinationMountTableResolver(conf, router);
- resolver.addResolver(DestinationOrder.LOCAL, spyLocalResolver);
-
-
- // We point /local to subclusters 0, 1, 2 with the local order
- Map<String, String> mapLocal = new HashMap<>();
- mapLocal.put("subcluster0", "/local");
- mapLocal.put("subcluster1", "/local");
- mapLocal.put("subcluster2", "/local");
- MountTable localEntry = MountTable.newInstance("/local", mapLocal);
- localEntry.setDestOrder(DestinationOrder.LOCAL);
- resolver.addEntry(localEntry);
-
- // Test first with the default destination
- PathLocation dest = resolver.getDestinationForPath("/local/file0.txt");
- assertDestination("subcluster0", dest);
-
- // We change the client location and verify
- setClient(sb, "client2");
- dest = resolver.getDestinationForPath("/local/file0.txt");
- assertDestination("subcluster2", dest);
-
- setClient(sb, "client1");
- dest = resolver.getDestinationForPath("/local/file0.txt");
- assertDestination("subcluster1", dest);
-
- setClient(sb, "client0");
- dest = resolver.getDestinationForPath("/local/file0.txt");
- assertDestination("subcluster0", dest);
- }
-
- private void assertDestination(String expectedNsId, PathLocation loc) {
- List<RemoteLocation> dests = loc.getDestinations();
- RemoteLocation dest = dests.get(0);
- assertEquals(expectedNsId, dest.getNameserviceId());
- }
-
- private MembershipState newMembershipState(String addr, String nsId) {
- return MembershipState.newInstance(
- "routerId", nsId, "nn0", "cluster0", "blockPool0",
- addr + ":8001", addr + ":8002", addr + ":8003", addr + ":8004",
- FederationNamenodeServiceState.ACTIVE, false);
- }
-
- /**
- * Set the address of the client issuing the request. We use a StringBuilder
- * to modify the value in place for the mock.
- * @param sb StringBuilder to set the client string.
- * @param client Address of the client.
- */
- private static void setClient(StringBuilder sb, String client) {
- sb.replace(0, sb.length(), client);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java
deleted file mode 100644
index 741d1f6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test functionalities of {@link ConnectionManager}, which manages a pool
- * of connections to NameNodes.
- */
-public class TestConnectionManager {
- private Configuration conf;
- private ConnectionManager connManager;
- private static final String[] TEST_GROUP = new String[]{"TEST_GROUP"};
- private static final UserGroupInformation TEST_USER1 =
- UserGroupInformation.createUserForTesting("user1", TEST_GROUP);
- private static final UserGroupInformation TEST_USER2 =
- UserGroupInformation.createUserForTesting("user2", TEST_GROUP);
- private static final UserGroupInformation TEST_USER3 =
- UserGroupInformation.createUserForTesting("user3", TEST_GROUP);
- private static final String TEST_NN_ADDRESS = "nn1:8080";
-
- @Before
- public void setup() throws Exception {
- conf = new Configuration();
- connManager = new ConnectionManager(conf);
- NetUtils.addStaticResolution("nn1", "localhost");
- NetUtils.createSocketAddrForHost("nn1", 8080);
- connManager.start();
- }
-
- @After
- public void shutdown() {
- if (connManager != null) {
- connManager.close();
- }
- }
-
- @Test
- public void testCleanup() throws Exception {
- Map<ConnectionPoolId, ConnectionPool> poolMap = connManager.getPools();
-
- ConnectionPool pool1 = new ConnectionPool(
- conf, TEST_NN_ADDRESS, TEST_USER1, 0, 10);
- addConnectionsToPool(pool1, 9, 4);
- poolMap.put(new ConnectionPoolId(TEST_USER1, TEST_NN_ADDRESS), pool1);
-
- ConnectionPool pool2 = new ConnectionPool(
- conf, TEST_NN_ADDRESS, TEST_USER2, 0, 10);
- addConnectionsToPool(pool2, 10, 10);
- poolMap.put(new ConnectionPoolId(TEST_USER2, TEST_NN_ADDRESS), pool2);
-
- checkPoolConnections(TEST_USER1, 9, 4);
- checkPoolConnections(TEST_USER2, 10, 10);
-
- // Clean up first pool, one connection should be removed, and second pool
- // should remain the same.
- connManager.cleanup(pool1);
- checkPoolConnections(TEST_USER1, 8, 4);
- checkPoolConnections(TEST_USER2, 10, 10);
-
- // Clean up the first pool again, it should have no effect since it reached
- // the MIN_ACTIVE_RATIO.
- connManager.cleanup(pool1);
- checkPoolConnections(TEST_USER1, 8, 4);
- checkPoolConnections(TEST_USER2, 10, 10);
-
- // Make sure the number of connections doesn't go below minSize
- ConnectionPool pool3 = new ConnectionPool(
- conf, TEST_NN_ADDRESS, TEST_USER3, 2, 10);
- addConnectionsToPool(pool3, 10, 0);
- poolMap.put(new ConnectionPoolId(TEST_USER2, TEST_NN_ADDRESS), pool3);
- connManager.cleanup(pool3);
- checkPoolConnections(TEST_USER3, 2, 0);
- // With active connections added to pool, make sure it honors the
- // MIN_ACTIVE_RATIO again
- addConnectionsToPool(pool3, 10, 2);
- connManager.cleanup(pool3);
- checkPoolConnections(TEST_USER3, 4, 2);
- }
-
- @Test
- public void testGetConnection() throws Exception {
- Map<ConnectionPoolId, ConnectionPool> poolMap = connManager.getPools();
- final int totalConns = 10;
- int activeConns = 5;
-
- ConnectionPool pool = new ConnectionPool(
- conf, TEST_NN_ADDRESS, TEST_USER1, 0, 10);
- addConnectionsToPool(pool, totalConns, activeConns);
- poolMap.put(new ConnectionPoolId(TEST_USER1, TEST_NN_ADDRESS), pool);
-
- // All remaining connections should be usable
- final int remainingSlots = totalConns - activeConns;
- for (int i = 0; i < remainingSlots; i++) {
- ConnectionContext cc = pool.getConnection();
- assertTrue(cc.isUsable());
- cc.getClient();
- activeConns++;
- }
-
- checkPoolConnections(TEST_USER1, totalConns, activeConns);
-
- // Ask for more and this returns an active connection
- ConnectionContext cc = pool.getConnection();
- assertTrue(cc.isActive());
- }
-
- private void addConnectionsToPool(ConnectionPool pool, int numTotalConn,
- int numActiveConn) throws IOException {
- for (int i = 0; i < numTotalConn; i++) {
- ConnectionContext cc = pool.newConnection();
- pool.addConnection(cc);
- if (i < numActiveConn) {
- cc.getClient();
- }
- }
- }
-
- private void checkPoolConnections(UserGroupInformation ugi,
- int numOfConns, int numOfActiveConns) {
- for (Map.Entry<ConnectionPoolId, ConnectionPool> e :
- connManager.getPools().entrySet()) {
- if (e.getKey().getUgi() == ugi) {
- assertEquals(numOfConns, e.getValue().getNumConnections());
- assertEquals(numOfActiveConns, e.getValue().getNumActiveConnections());
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestNamenodeHeartbeat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestNamenodeHeartbeat.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestNamenodeHeartbeat.java
deleted file mode 100644
index 877fb02..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestNamenodeHeartbeat.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMENODES;
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMESERVICES;
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.server.federation.MockResolver;
-import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster;
-import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
-import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
-import org.apache.hadoop.service.Service.STATE;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestName;
-
-/**
- * Test the service that heartbeats the state of the namenodes to the State
- * Store.
- */
-public class TestNamenodeHeartbeat {
-
- private static RouterDFSCluster cluster;
- private static ActiveNamenodeResolver namenodeResolver;
- private static List<NamenodeHeartbeatService> services;
-
- @Rule
- public TestName name = new TestName();
-
- @BeforeClass
- public static void globalSetUp() throws Exception {
-
- cluster = new RouterDFSCluster(true, 2);
-
- // Start NNs and DNs and wait until ready
- cluster.startCluster();
-
- // Mock locator that records the heartbeats
- List<String> nss = cluster.getNameservices();
- String ns = nss.get(0);
- Configuration conf = cluster.generateNamenodeConfiguration(ns);
- namenodeResolver = new MockResolver(conf);
- namenodeResolver.setRouterId("testrouter");
-
- // Create one heartbeat service per NN
- services = new ArrayList<>();
- for (NamenodeContext nn : cluster.getNamenodes()) {
- String nsId = nn.getNameserviceId();
- String nnId = nn.getNamenodeId();
- NamenodeHeartbeatService service = new NamenodeHeartbeatService(
- namenodeResolver, nsId, nnId);
- service.init(conf);
- service.start();
- services.add(service);
- }
- }
-
- @AfterClass
- public static void tearDown() throws IOException {
- cluster.shutdown();
- for (NamenodeHeartbeatService service: services) {
- service.stop();
- service.close();
- }
- }
-
- @Test
- public void testNamenodeHeartbeatService() throws IOException {
-
- RouterDFSCluster testCluster = new RouterDFSCluster(true, 1);
- Configuration heartbeatConfig = testCluster.generateNamenodeConfiguration(
- NAMESERVICES[0]);
- NamenodeHeartbeatService server = new NamenodeHeartbeatService(
- namenodeResolver, NAMESERVICES[0], NAMENODES[0]);
- server.init(heartbeatConfig);
- assertEquals(STATE.INITED, server.getServiceState());
- server.start();
- assertEquals(STATE.STARTED, server.getServiceState());
- server.stop();
- assertEquals(STATE.STOPPED, server.getServiceState());
- server.close();
- }
-
- @Test
- public void testHearbeat() throws InterruptedException, IOException {
-
- // Set NAMENODE1 to active for all nameservices
- if (cluster.isHighAvailability()) {
- for (String ns : cluster.getNameservices()) {
- cluster.switchToActive(ns, NAMENODES[0]);
- cluster.switchToStandby(ns, NAMENODES[1]);
- }
- }
-
- // Wait for heartbeats to record
- Thread.sleep(5000);
-
- // Verify the locator has matching NN entries for each NS
- for (String ns : cluster.getNameservices()) {
- List<? extends FederationNamenodeContext> nns =
- namenodeResolver.getNamenodesForNameserviceId(ns);
-
- // Active
- FederationNamenodeContext active = nns.get(0);
- assertEquals(NAMENODES[0], active.getNamenodeId());
-
- // Standby
- FederationNamenodeContext standby = nns.get(1);
- assertEquals(NAMENODES[1], standby.getNamenodeId());
- }
-
- // Switch active NNs in 1/2 nameservices
- List<String> nss = cluster.getNameservices();
- String failoverNS = nss.get(0);
- String normalNs = nss.get(1);
-
- cluster.switchToStandby(failoverNS, NAMENODES[0]);
- cluster.switchToActive(failoverNS, NAMENODES[1]);
-
- // Wait for heartbeats to record
- Thread.sleep(5000);
-
- // Verify the locator has recorded the failover for the failover NS
- List<? extends FederationNamenodeContext> failoverNSs =
- namenodeResolver.getNamenodesForNameserviceId(failoverNS);
- // Active
- FederationNamenodeContext active = failoverNSs.get(0);
- assertEquals(NAMENODES[1], active.getNamenodeId());
-
- // Standby
- FederationNamenodeContext standby = failoverNSs.get(1);
- assertEquals(NAMENODES[0], standby.getNamenodeId());
-
- // Verify the locator has the same records for the other ns
- List<? extends FederationNamenodeContext> normalNss =
- namenodeResolver.getNamenodesForNameserviceId(normalNs);
- // Active
- active = normalNss.get(0);
- assertEquals(NAMENODES[0], active.getNamenodeId());
- // Standby
- standby = normalNss.get(1);
- assertEquals(NAMENODES[1], standby.getNamenodeId());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
deleted file mode 100644
index 77d1698..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.server.federation.MockResolver;
-import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
-import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
-import org.apache.hadoop.service.Service.STATE;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- * The the safe mode for the {@link Router} controlled by
- * {@link SafeModeTimer}.
- */
-public class TestRouter {
-
- private static Configuration conf;
-
- @BeforeClass
- public static void create() throws IOException {
- // Basic configuration without the state store
- conf = new Configuration();
- // 1 sec cache refresh
- conf.setInt(DFSConfigKeys.DFS_ROUTER_CACHE_TIME_TO_LIVE_MS, 1);
- // Mock resolver classes
- conf.setClass(DFSConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS,
- MockResolver.class, ActiveNamenodeResolver.class);
- conf.setClass(DFSConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
- MockResolver.class, FileSubclusterResolver.class);
-
- // Bind to any available port
- conf.set(DFSConfigKeys.DFS_ROUTER_RPC_BIND_HOST_KEY, "0.0.0.0");
- conf.set(DFSConfigKeys.DFS_ROUTER_RPC_ADDRESS_KEY, "127.0.0.1:0");
- conf.set(DFSConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_KEY, "127.0.0.1:0");
- conf.set(DFSConfigKeys.DFS_ROUTER_ADMIN_BIND_HOST_KEY, "0.0.0.0");
- conf.set(DFSConfigKeys.DFS_ROUTER_HTTP_ADDRESS_KEY, "127.0.0.1:0");
- conf.set(DFSConfigKeys.DFS_ROUTER_HTTPS_ADDRESS_KEY, "127.0.0.1:0");
- conf.set(DFSConfigKeys.DFS_ROUTER_HTTP_BIND_HOST_KEY, "0.0.0.0");
-
- // Simulate a co-located NN
- conf.set(DFSConfigKeys.DFS_NAMESERVICES, "ns0");
- conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://" + "ns0");
- conf.set(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + "ns0",
- "127.0.0.1:0" + 0);
- conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY + "." + "ns0",
- "127.0.0.1:" + 0);
- conf.set(DFSConfigKeys.DFS_NAMENODE_RPC_BIND_HOST_KEY + "." + "ns0",
- "0.0.0.0");
- }
-
- @AfterClass
- public static void destroy() {
- }
-
- @Before
- public void setup() throws IOException, URISyntaxException {
- }
-
- @After
- public void cleanup() {
- }
-
- private static void testRouterStartup(Configuration routerConfig)
- throws InterruptedException, IOException {
- Router router = new Router();
- assertEquals(STATE.NOTINITED, router.getServiceState());
- router.init(routerConfig);
- assertEquals(STATE.INITED, router.getServiceState());
- router.start();
- assertEquals(STATE.STARTED, router.getServiceState());
- router.stop();
- assertEquals(STATE.STOPPED, router.getServiceState());
- router.close();
- }
-
- @Test
- public void testRouterService() throws InterruptedException, IOException {
-
- // Admin only
- testRouterStartup(new RouterConfigBuilder(conf).admin().build());
-
- // Http only
- testRouterStartup(new RouterConfigBuilder(conf).http().build());
-
- // Rpc only
- testRouterStartup(new RouterConfigBuilder(conf).rpc().build());
-
- // Metrics only
- testRouterStartup(new RouterConfigBuilder(conf).metrics().build());
-
- // Statestore only
- testRouterStartup(new RouterConfigBuilder(conf).stateStore().build());
-
- // Heartbeat only
- testRouterStartup(new RouterConfigBuilder(conf).heartbeat().build());
-
- // Run with all services
- testRouterStartup(new RouterConfigBuilder(conf).all().build());
- }
-
- @Test
- public void testRouterRestartRpcService() throws IOException {
-
- // Start
- Router router = new Router();
- router.init(new RouterConfigBuilder(conf).rpc().build());
- router.start();
-
- // Verify RPC server is running
- assertNotNull(router.getRpcServerAddress());
- RouterRpcServer rpcServer = router.getRpcServer();
- assertNotNull(rpcServer);
- assertEquals(STATE.STARTED, rpcServer.getServiceState());
-
- // Stop router and RPC server
- router.stop();
- assertEquals(STATE.STOPPED, rpcServer.getServiceState());
-
- router.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
deleted file mode 100644
index a8ffded..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
+++ /dev/null
@@ -1,340 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.synchronizeRecords;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
-import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
-import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
-import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
-import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
-import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
-import org.apache.hadoop.hdfs.server.federation.store.impl.MountTableStoreImpl;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-import org.apache.hadoop.util.Time;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- * The administrator interface of the {@link Router} implemented by
- * {@link RouterAdminServer}.
- */
-public class TestRouterAdmin {
-
- private static StateStoreDFSCluster cluster;
- private static RouterContext routerContext;
- public static final String RPC_BEAN =
- "Hadoop:service=Router,name=FederationRPC";
- private static List<MountTable> mockMountTable;
- private static StateStoreService stateStore;
-
- @BeforeClass
- public static void globalSetUp() throws Exception {
- cluster = new StateStoreDFSCluster(false, 1);
- // Build and start a router with State Store + admin + RPC
- Configuration conf = new RouterConfigBuilder()
- .stateStore()
- .admin()
- .rpc()
- .build();
- cluster.addRouterOverrides(conf);
- cluster.startRouters();
- routerContext = cluster.getRandomRouter();
- mockMountTable = cluster.generateMockMountTable();
- Router router = routerContext.getRouter();
- stateStore = router.getStateStore();
- }
-
- @AfterClass
- public static void tearDown() {
- cluster.stopRouter(routerContext);
- }
-
- @Before
- public void testSetup() throws Exception {
- assertTrue(
- synchronizeRecords(stateStore, mockMountTable, MountTable.class));
- }
-
- @Test
- public void testAddMountTable() throws IOException {
- MountTable newEntry = MountTable.newInstance(
- "/testpath", Collections.singletonMap("ns0", "/testdir"),
- Time.now(), Time.now());
-
- RouterClient client = routerContext.getAdminClient();
- MountTableManager mountTable = client.getMountTableManager();
-
- // Existing mount table size
- List<MountTable> records = getMountTableEntries(mountTable);
- assertEquals(records.size(), mockMountTable.size());
-
- // Add
- AddMountTableEntryRequest addRequest =
- AddMountTableEntryRequest.newInstance(newEntry);
- AddMountTableEntryResponse addResponse =
- mountTable.addMountTableEntry(addRequest);
- assertTrue(addResponse.getStatus());
-
- // New mount table size
- List<MountTable> records2 = getMountTableEntries(mountTable);
- assertEquals(records2.size(), mockMountTable.size() + 1);
- }
-
- @Test
- public void testAddDuplicateMountTable() throws IOException {
- MountTable newEntry = MountTable.newInstance("/testpath",
- Collections.singletonMap("ns0", "/testdir"), Time.now(), Time.now());
-
- RouterClient client = routerContext.getAdminClient();
- MountTableManager mountTable = client.getMountTableManager();
-
- // Existing mount table size
- List<MountTable> entries1 = getMountTableEntries(mountTable);
- assertEquals(entries1.size(), mockMountTable.size());
-
- // Add
- AddMountTableEntryRequest addRequest =
- AddMountTableEntryRequest.newInstance(newEntry);
- AddMountTableEntryResponse addResponse =
- mountTable.addMountTableEntry(addRequest);
- assertTrue(addResponse.getStatus());
-
- // New mount table size
- List<MountTable> entries2 = getMountTableEntries(mountTable);
- assertEquals(entries2.size(), mockMountTable.size() + 1);
-
- // Add again, should fail
- AddMountTableEntryResponse addResponse2 =
- mountTable.addMountTableEntry(addRequest);
- assertFalse(addResponse2.getStatus());
- }
-
- @Test
- public void testAddReadOnlyMountTable() throws IOException {
- MountTable newEntry = MountTable.newInstance(
- "/readonly", Collections.singletonMap("ns0", "/testdir"),
- Time.now(), Time.now());
- newEntry.setReadOnly(true);
-
- RouterClient client = routerContext.getAdminClient();
- MountTableManager mountTable = client.getMountTableManager();
-
- // Existing mount table size
- List<MountTable> records = getMountTableEntries(mountTable);
- assertEquals(records.size(), mockMountTable.size());
-
- // Add
- AddMountTableEntryRequest addRequest =
- AddMountTableEntryRequest.newInstance(newEntry);
- AddMountTableEntryResponse addResponse =
- mountTable.addMountTableEntry(addRequest);
- assertTrue(addResponse.getStatus());
-
- // New mount table size
- List<MountTable> records2 = getMountTableEntries(mountTable);
- assertEquals(records2.size(), mockMountTable.size() + 1);
-
- // Check that we have the read only entry
- MountTable record = getMountTableEntry("/readonly");
- assertEquals("/readonly", record.getSourcePath());
- assertTrue(record.isReadOnly());
-
- // Removing the new entry
- RemoveMountTableEntryRequest removeRequest =
- RemoveMountTableEntryRequest.newInstance("/readonly");
- RemoveMountTableEntryResponse removeResponse =
- mountTable.removeMountTableEntry(removeRequest);
- assertTrue(removeResponse.getStatus());
- }
-
- @Test
- public void testAddOrderMountTable() throws IOException {
- testAddOrderMountTable(DestinationOrder.HASH);
- testAddOrderMountTable(DestinationOrder.LOCAL);
- testAddOrderMountTable(DestinationOrder.RANDOM);
- testAddOrderMountTable(DestinationOrder.HASH_ALL);
- }
-
- private void testAddOrderMountTable(final DestinationOrder order)
- throws IOException {
- final String mnt = "/" + order;
- MountTable newEntry = MountTable.newInstance(
- mnt, Collections.singletonMap("ns0", "/testdir"),
- Time.now(), Time.now());
- newEntry.setDestOrder(order);
-
- RouterClient client = routerContext.getAdminClient();
- MountTableManager mountTable = client.getMountTableManager();
-
- // Add
- AddMountTableEntryRequest addRequest;
- AddMountTableEntryResponse addResponse;
- addRequest = AddMountTableEntryRequest.newInstance(newEntry);
- addResponse = mountTable.addMountTableEntry(addRequest);
- assertTrue(addResponse.getStatus());
-
- // Check that we have the read only entry
- MountTable record = getMountTableEntry(mnt);
- assertEquals(mnt, record.getSourcePath());
- assertEquals(order, record.getDestOrder());
-
- // Removing the new entry
- RemoveMountTableEntryRequest removeRequest =
- RemoveMountTableEntryRequest.newInstance(mnt);
- RemoveMountTableEntryResponse removeResponse =
- mountTable.removeMountTableEntry(removeRequest);
- assertTrue(removeResponse.getStatus());
- }
-
- @Test
- public void testRemoveMountTable() throws IOException {
-
- RouterClient client = routerContext.getAdminClient();
- MountTableManager mountTable = client.getMountTableManager();
-
- // Existing mount table size
- List<MountTable> entries1 = getMountTableEntries(mountTable);
- assertEquals(entries1.size(), mockMountTable.size());
-
- // Remove an entry
- RemoveMountTableEntryRequest removeRequest =
- RemoveMountTableEntryRequest.newInstance("/");
- mountTable.removeMountTableEntry(removeRequest);
-
- // New mount table size
- List<MountTable> entries2 = getMountTableEntries(mountTable);
- assertEquals(entries2.size(), mockMountTable.size() - 1);
- }
-
- @Test
- public void testEditMountTable() throws IOException {
-
- RouterClient client = routerContext.getAdminClient();
- MountTableManager mountTable = client.getMountTableManager();
-
- // Verify starting condition
- MountTable entry = getMountTableEntry("/");
- assertEquals(
- Collections.singletonList(new RemoteLocation("ns0", "/")),
- entry.getDestinations());
-
- // Edit the entry for /
- MountTable updatedEntry = MountTable.newInstance(
- "/", Collections.singletonMap("ns1", "/"), Time.now(), Time.now());
- UpdateMountTableEntryRequest updateRequest =
- UpdateMountTableEntryRequest.newInstance(updatedEntry);
- mountTable.updateMountTableEntry(updateRequest);
-
- // Verify edited condition
- entry = getMountTableEntry("/");
- assertEquals(
- Collections.singletonList(new RemoteLocation("ns1", "/")),
- entry.getDestinations());
- }
-
- @Test
- public void testGetMountTable() throws IOException {
-
- RouterClient client = routerContext.getAdminClient();
- MountTableManager mountTable = client.getMountTableManager();
-
- // Verify size of table
- List<MountTable> entries = getMountTableEntries(mountTable);
- assertEquals(mockMountTable.size(), entries.size());
-
- // Verify all entries are present
- int matches = 0;
- for (MountTable e : entries) {
- for (MountTable entry : mockMountTable) {
- assertEquals(e.getDestinations().size(), 1);
- assertNotNull(e.getDateCreated());
- assertNotNull(e.getDateModified());
- if (entry.getSourcePath().equals(e.getSourcePath())) {
- matches++;
- }
- }
- }
- assertEquals(matches, mockMountTable.size());
- }
-
- @Test
- public void testGetSingleMountTableEntry() throws IOException {
- MountTable entry = getMountTableEntry("/ns0");
- assertNotNull(entry);
- assertEquals(entry.getSourcePath(), "/ns0");
- }
-
- /**
- * Gets an existing mount table record in the state store.
- *
- * @param mount The mount point of the record to remove.
- * @return The matching record if found, null if it is not found.
- * @throws IOException If the state store could not be accessed.
- */
- private MountTable getMountTableEntry(final String mount) throws IOException {
- // Refresh the cache
- stateStore.loadCache(MountTableStoreImpl.class, true);
-
- GetMountTableEntriesRequest request =
- GetMountTableEntriesRequest.newInstance(mount);
- RouterClient client = routerContext.getAdminClient();
- MountTableManager mountTable = client.getMountTableManager();
- List<MountTable> results = getMountTableEntries(mountTable, request);
- if (results.size() > 0) {
- // First result is sorted to have the shortest mount string length
- return results.get(0);
- }
- return null;
- }
-
- private List<MountTable> getMountTableEntries(MountTableManager mountTable)
- throws IOException {
- GetMountTableEntriesRequest request =
- GetMountTableEntriesRequest.newInstance("/");
- return getMountTableEntries(mountTable, request);
- }
-
- private List<MountTable> getMountTableEntries(MountTableManager mountTable,
- GetMountTableEntriesRequest request) throws IOException {
- stateStore.loadCache(MountTableStoreImpl.class, true);
- GetMountTableEntriesResponse response =
- mountTable.getMountTableEntries(request);
- return response.getEntries();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
deleted file mode 100644
index 20353c3..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
+++ /dev/null
@@ -1,478 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.io.ByteArrayOutputStream;
-
-import java.io.PrintStream;
-import java.net.InetSocketAddress;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
-import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
-import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
-import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
-import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
-import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
-import org.apache.hadoop.hdfs.server.federation.store.impl.MountTableStoreImpl;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-import org.apache.hadoop.hdfs.tools.federation.RouterAdmin;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.util.ToolRunner;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import com.google.common.base.Supplier;
-
-/**
- * Tests Router admin commands.
- */
-public class TestRouterAdminCLI {
- private static StateStoreDFSCluster cluster;
- private static RouterContext routerContext;
- private static StateStoreService stateStore;
-
- private static RouterAdmin admin;
- private static RouterClient client;
-
- private static final String TEST_USER = "test-user";
-
- private final ByteArrayOutputStream out = new ByteArrayOutputStream();
- private static final PrintStream OLD_OUT = System.out;
-
- @BeforeClass
- public static void globalSetUp() throws Exception {
- cluster = new StateStoreDFSCluster(false, 1);
- // Build and start a router with State Store + admin + RPC
- Configuration conf = new RouterConfigBuilder()
- .stateStore()
- .admin()
- .rpc()
- .build();
- cluster.addRouterOverrides(conf);
-
- // Start routers
- cluster.startRouters();
-
- routerContext = cluster.getRandomRouter();
- Router router = routerContext.getRouter();
- stateStore = router.getStateStore();
-
- Configuration routerConf = new Configuration();
- InetSocketAddress routerSocket = router.getAdminServerAddress();
- routerConf.setSocketAddr(DFSConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_KEY,
- routerSocket);
- admin = new RouterAdmin(routerConf);
- client = routerContext.getAdminClient();
- }
-
- @AfterClass
- public static void tearDownCluster() {
- cluster.stopRouter(routerContext);
- cluster.shutdown();
- cluster = null;
- }
-
- @After
- public void tearDown() {
- // set back system out
- System.setOut(OLD_OUT);
- }
-
- @Test
- public void testAddMountTable() throws Exception {
- String nsId = "ns0";
- String src = "/test-addmounttable";
- String dest = "/addmounttable";
- String[] argv = new String[] {"-add", src, nsId, dest};
- assertEquals(0, ToolRunner.run(admin, argv));
-
- stateStore.loadCache(MountTableStoreImpl.class, true);
- GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest
- .newInstance(src);
- GetMountTableEntriesResponse getResponse = client.getMountTableManager()
- .getMountTableEntries(getRequest);
- MountTable mountTable = getResponse.getEntries().get(0);
-
- List<RemoteLocation> destinations = mountTable.getDestinations();
- assertEquals(1, destinations.size());
-
- assertEquals(src, mountTable.getSourcePath());
- assertEquals(nsId, destinations.get(0).getNameserviceId());
- assertEquals(dest, destinations.get(0).getDest());
- assertFalse(mountTable.isReadOnly());
-
- // test mount table update behavior
- dest = dest + "-new";
- argv = new String[] {"-add", src, nsId, dest, "-readonly"};
- assertEquals(0, ToolRunner.run(admin, argv));
- stateStore.loadCache(MountTableStoreImpl.class, true);
-
- getResponse = client.getMountTableManager()
- .getMountTableEntries(getRequest);
- mountTable = getResponse.getEntries().get(0);
- assertEquals(2, mountTable.getDestinations().size());
- assertEquals(nsId, mountTable.getDestinations().get(1).getNameserviceId());
- assertEquals(dest, mountTable.getDestinations().get(1).getDest());
- assertTrue(mountTable.isReadOnly());
- }
-
- @Test
- public void testAddOrderMountTable() throws Exception {
- testAddOrderMountTable(DestinationOrder.HASH);
- testAddOrderMountTable(DestinationOrder.LOCAL);
- testAddOrderMountTable(DestinationOrder.RANDOM);
- testAddOrderMountTable(DestinationOrder.HASH_ALL);
- }
-
- private void testAddOrderMountTable(DestinationOrder order)
- throws Exception {
- final String mnt = "/" + order;
- final String nsId = "ns0,ns1";
- final String dest = "/";
- String[] argv = new String[] {
- "-add", mnt, nsId, dest, "-order", order.toString()};
- assertEquals(0, ToolRunner.run(admin, argv));
-
- // Check the state in the State Store
- stateStore.loadCache(MountTableStoreImpl.class, true);
- MountTableManager mountTable = client.getMountTableManager();
- GetMountTableEntriesRequest request =
- GetMountTableEntriesRequest.newInstance(mnt);
- GetMountTableEntriesResponse response =
- mountTable.getMountTableEntries(request);
- List<MountTable> entries = response.getEntries();
- assertEquals(1, entries.size());
- assertEquals(2, entries.get(0).getDestinations().size());
- assertEquals(order, response.getEntries().get(0).getDestOrder());
- }
-
- @Test
- public void testListMountTable() throws Exception {
- String nsId = "ns0";
- String src = "/test-lsmounttable";
- String dest = "/lsmounttable";
- String[] argv = new String[] {"-add", src, nsId, dest};
- assertEquals(0, ToolRunner.run(admin, argv));
-
- // re-set system out for testing
- System.setOut(new PrintStream(out));
- stateStore.loadCache(MountTableStoreImpl.class, true);
- argv = new String[] {"-ls", src};
- assertEquals(0, ToolRunner.run(admin, argv));
- assertTrue(out.toString().contains(src));
-
- out.reset();
- GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest
- .newInstance("/");
- GetMountTableEntriesResponse getResponse = client.getMountTableManager()
- .getMountTableEntries(getRequest);
-
- // Test ls command without input path, it will list
- // mount table under root path.
- argv = new String[] {"-ls"};
- assertEquals(0, ToolRunner.run(admin, argv));
- assertTrue(out.toString().contains(src));
- String outStr = out.toString();
- // verify if all the mount table are listed
- for(MountTable entry: getResponse.getEntries()) {
- assertTrue(outStr.contains(entry.getSourcePath()));
- }
- }
-
- @Test
- public void testRemoveMountTable() throws Exception {
- String nsId = "ns0";
- String src = "/test-rmmounttable";
- String dest = "/rmmounttable";
- String[] argv = new String[] {"-add", src, nsId, dest};
- assertEquals(0, ToolRunner.run(admin, argv));
-
- stateStore.loadCache(MountTableStoreImpl.class, true);
- GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest
- .newInstance(src);
- GetMountTableEntriesResponse getResponse = client.getMountTableManager()
- .getMountTableEntries(getRequest);
- // ensure mount table added successfully
- MountTable mountTable = getResponse.getEntries().get(0);
- assertEquals(src, mountTable.getSourcePath());
-
- argv = new String[] {"-rm", src};
- assertEquals(0, ToolRunner.run(admin, argv));
-
- stateStore.loadCache(MountTableStoreImpl.class, true);
- getResponse = client.getMountTableManager()
- .getMountTableEntries(getRequest);
- assertEquals(0, getResponse.getEntries().size());
-
- // remove an invalid mount table
- String invalidPath = "/invalid";
- System.setOut(new PrintStream(out));
- argv = new String[] {"-rm", invalidPath};
- assertEquals(0, ToolRunner.run(admin, argv));
- assertTrue(out.toString().contains(
- "Cannot remove mount point " + invalidPath));
- }
-
- @Test
- public void testMountTableDefaultACL() throws Exception {
- String[] argv = new String[] {"-add", "/testpath0", "ns0", "/testdir0"};
- assertEquals(0, ToolRunner.run(admin, argv));
-
- stateStore.loadCache(MountTableStoreImpl.class, true);
- GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest
- .newInstance("/testpath0");
- GetMountTableEntriesResponse getResponse = client.getMountTableManager()
- .getMountTableEntries(getRequest);
- MountTable mountTable = getResponse.getEntries().get(0);
-
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- String group = ugi.getGroups().isEmpty() ? ugi.getShortUserName()
- : ugi.getPrimaryGroupName();
- assertEquals(ugi.getShortUserName(), mountTable.getOwnerName());
- assertEquals(group, mountTable.getGroupName());
- assertEquals((short) 0755, mountTable.getMode().toShort());
- }
-
- @Test
- public void testMountTablePermissions() throws Exception {
- // re-set system out for testing
- System.setOut(new PrintStream(out));
- // use superuser to add new mount table with only read permission
- String[] argv = new String[] {"-add", "/testpath2-1", "ns0", "/testdir2-1",
- "-owner", TEST_USER, "-group", TEST_USER, "-mode", "0455"};
- assertEquals(0, ToolRunner.run(admin, argv));
-
- String superUser = UserGroupInformation.
- getCurrentUser().getShortUserName();
- // use normal user as current user to test
- UserGroupInformation remoteUser = UserGroupInformation
- .createRemoteUser(TEST_USER);
- UserGroupInformation.setLoginUser(remoteUser);
-
- // verify read permission by executing other commands
- verifyExecutionResult("/testpath2-1", true, -1, -1);
-
- // add new mount table with only write permission
- argv = new String[] {"-add", "/testpath2-2", "ns0", "/testdir2-2",
- "-owner", TEST_USER, "-group", TEST_USER, "-mode", "0255"};
- assertEquals(0, ToolRunner.run(admin, argv));
- verifyExecutionResult("/testpath2-2", false, 0, 0);
-
- // set mount table entry with read and write permission
- argv = new String[] {"-add", "/testpath2-3", "ns0", "/testdir2-3",
- "-owner", TEST_USER, "-group", TEST_USER, "-mode", "0755"};
- assertEquals(0, ToolRunner.run(admin, argv));
- verifyExecutionResult("/testpath2-3", true, 0, 0);
-
- // set back login user
- remoteUser = UserGroupInformation.createRemoteUser(superUser);
- UserGroupInformation.setLoginUser(remoteUser);
- }
-
- /**
- * Verify router admin commands execution result.
- *
- * @param mount
- * target mount table
- * @param canRead
- * whether can list mount tables under specified mount
- * @param addCommandCode
- * expected return code of add command executed for specified mount
- * @param rmCommandCode
- * expected return code of rm command executed for specified mount
- * @throws Exception
- */
- private void verifyExecutionResult(String mount, boolean canRead,
- int addCommandCode, int rmCommandCode) throws Exception {
- String[] argv = null;
- stateStore.loadCache(MountTableStoreImpl.class, true);
-
- out.reset();
- // execute ls command
- argv = new String[] {"-ls", mount};
- assertEquals(0, ToolRunner.run(admin, argv));
- assertEquals(canRead, out.toString().contains(mount));
-
- // execute add/update command
- argv = new String[] {"-add", mount, "ns0", mount + "newdir"};
- assertEquals(addCommandCode, ToolRunner.run(admin, argv));
-
- stateStore.loadCache(MountTableStoreImpl.class, true);
- // execute remove command
- argv = new String[] {"-rm", mount};
- assertEquals(rmCommandCode, ToolRunner.run(admin, argv));
- }
-
- @Test
- public void testSetAndClearQuota() throws Exception {
- String nsId = "ns0";
- String src = "/test-QuotaMounttable";
- String dest = "/QuotaMounttable";
- String[] argv = new String[] {"-add", src, nsId, dest};
- assertEquals(0, ToolRunner.run(admin, argv));
-
- stateStore.loadCache(MountTableStoreImpl.class, true);
- GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest
- .newInstance(src);
- GetMountTableEntriesResponse getResponse = client.getMountTableManager()
- .getMountTableEntries(getRequest);
- MountTable mountTable = getResponse.getEntries().get(0);
- RouterQuotaUsage quotaUsage = mountTable.getQuota();
-
- // verify the default quota set
- assertEquals(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT,
- quotaUsage.getFileAndDirectoryCount());
- assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaUsage.getQuota());
- assertEquals(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT,
- quotaUsage.getSpaceConsumed());
- assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaUsage.getSpaceQuota());
-
- long nsQuota = 50;
- long ssQuota = 100;
- argv = new String[] {"-setQuota", src, "-nsQuota", String.valueOf(nsQuota),
- "-ssQuota", String.valueOf(ssQuota)};
- assertEquals(0, ToolRunner.run(admin, argv));
-
- stateStore.loadCache(MountTableStoreImpl.class, true);
- getResponse = client.getMountTableManager()
- .getMountTableEntries(getRequest);
- mountTable = getResponse.getEntries().get(0);
- quotaUsage = mountTable.getQuota();
-
- // verify if the quota is set
- assertEquals(nsQuota, quotaUsage.getQuota());
- assertEquals(ssQuota, quotaUsage.getSpaceQuota());
-
- // use quota string for setting ss quota
- String newSsQuota = "2m";
- argv = new String[] {"-setQuota", src, "-ssQuota", newSsQuota};
- assertEquals(0, ToolRunner.run(admin, argv));
-
- stateStore.loadCache(MountTableStoreImpl.class, true);
- getResponse = client.getMountTableManager()
- .getMountTableEntries(getRequest);
- mountTable = getResponse.getEntries().get(0);
- quotaUsage = mountTable.getQuota();
- // verify if ss quota is correctly set
- assertEquals(2 * 1024 * 1024, quotaUsage.getSpaceQuota());
-
- // test clrQuota command
- argv = new String[] {"-clrQuota", src};
- assertEquals(0, ToolRunner.run(admin, argv));
-
- stateStore.loadCache(MountTableStoreImpl.class, true);
- getResponse = client.getMountTableManager()
- .getMountTableEntries(getRequest);
- mountTable = getResponse.getEntries().get(0);
- quotaUsage = mountTable.getQuota();
-
- // verify if quota unset successfully
- assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaUsage.getQuota());
- assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaUsage.getSpaceQuota());
- }
-
- @Test
- public void testManageSafeMode() throws Exception {
- // ensure the Router become RUNNING state
- waitState(RouterServiceState.RUNNING);
- assertFalse(routerContext.getRouter().getRpcServer().isInSafeMode());
- assertEquals(0, ToolRunner.run(admin,
- new String[] {"-safemode", "enter"}));
- // verify state
- assertEquals(RouterServiceState.SAFEMODE,
- routerContext.getRouter().getRouterState());
- assertTrue(routerContext.getRouter().getRpcServer().isInSafeMode());
-
- System.setOut(new PrintStream(out));
- assertEquals(0, ToolRunner.run(admin,
- new String[] {"-safemode", "get"}));
- assertTrue(out.toString().contains("true"));
-
- assertEquals(0, ToolRunner.run(admin,
- new String[] {"-safemode", "leave"}));
- // verify state
- assertEquals(RouterServiceState.RUNNING,
- routerContext.getRouter().getRouterState());
- assertFalse(routerContext.getRouter().getRpcServer().isInSafeMode());
-
- out.reset();
- assertEquals(0, ToolRunner.run(admin,
- new String[] {"-safemode", "get"}));
- assertTrue(out.toString().contains("false"));
- }
-
- @Test
- public void testCreateInvalidEntry() throws Exception {
- String[] argv = new String[] {
- "-add", "test-createInvalidEntry", "ns0", "/createInvalidEntry"};
- assertEquals(-1, ToolRunner.run(admin, argv));
-
- argv = new String[] {
- "-add", "/test-createInvalidEntry", "ns0", "createInvalidEntry"};
- assertEquals(-1, ToolRunner.run(admin, argv));
-
- argv = new String[] {
- "-add", null, "ns0", "/createInvalidEntry"};
- assertEquals(-1, ToolRunner.run(admin, argv));
-
- argv = new String[] {
- "-add", "/test-createInvalidEntry", "ns0", null};
- assertEquals(-1, ToolRunner.run(admin, argv));
-
- argv = new String[] {
- "-add", "", "ns0", "/createInvalidEntry"};
- assertEquals(-1, ToolRunner.run(admin, argv));
-
- argv = new String[] {
- "-add", "/test-createInvalidEntry", null, "/createInvalidEntry"};
- assertEquals(-1, ToolRunner.run(admin, argv));
-
- argv = new String[] {
- "-add", "/test-createInvalidEntry", "", "/createInvalidEntry"};
- assertEquals(-1, ToolRunner.run(admin, argv));
- }
-
- /**
- * Wait for the Router transforming to expected state.
- * @param expectedState Expected Router state.
- * @throws Exception
- */
- private void waitState(final RouterServiceState expectedState)
- throws Exception {
- GenericTestUtils.waitFor(new Supplier<Boolean>() {
- @Override
- public Boolean get() {
- return expectedState == routerContext.getRouter().getRouterState();
- }
- }, 1000, 30000);
- }
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
new file mode 100644
index 0000000..e3a1834
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
@@ -0,0 +1,2214 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HANDLER_COUNT_DEFAULT;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HANDLER_COUNT_KEY;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HANDLER_QUEUE_SIZE_DEFAULT;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HANDLER_QUEUE_SIZE_KEY;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_READER_COUNT_DEFAULT;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_READER_COUNT_KEY;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_READER_QUEUE_SIZE_DEFAULT;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_READER_QUEUE_SIZE_KEY;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.lang.reflect.Array;
+import java.net.InetSocketAddress;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.CryptoProtocolVersion;
+import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries;
+import org.apache.hadoop.fs.CacheFlag;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FsServerDefaults;
+import org.apache.hadoop.fs.Options;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.QuotaUsage;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.fs.XAttr;
+import org.apache.hadoop.fs.XAttrSetFlag;
+import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.AddBlockFlag;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.inotify.EventBatchList;
+import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
+import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
+import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
+import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats;
+import org.apache.hadoop.hdfs.protocol.EncryptionZone;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
+import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
+import org.apache.hadoop.hdfs.protocol.OpenFilesIterator;
+import org.apache.hadoop.hdfs.protocol.OpenFilesIterator.OpenFilesType;
+import org.apache.hadoop.hdfs.protocol.ReplicatedBlockStats;
+import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing;
+import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
+import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ClientNamenodeProtocol;
+import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB;
+import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB;
+import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCMetrics;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException;
+import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
+import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException;
+import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
+import org.apache.hadoop.io.EnumSetWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.RPC.Server;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.ipc.StandbyException;
+import org.apache.hadoop.net.NodeBase;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.protobuf.BlockingService;
+
+/**
+ * This class is responsible for handling all of the RPC calls to the It is
+ * created, started, and stopped by {@link Router}. It implements the
+ * {@link ClientProtocol} to mimic a
+ * {@link org.apache.hadoop.hdfs.server.namenode.NameNode NameNode} and proxies
+ * the requests to the active
+ * {@link org.apache.hadoop.hdfs.server.namenode.NameNode NameNode}.
+ */
+public class RouterRpcServer extends AbstractService implements ClientProtocol {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RouterRpcServer.class);
+
+
+ /** Configuration for the RPC server. */
+ private Configuration conf;
+
+ /** Identifier for the super user. */
+ private final String superUser;
+ /** Identifier for the super group. */
+ private final String superGroup;
+
+ /** Router using this RPC server. */
+ private final Router router;
+
+ /** The RPC server that listens to requests from clients. */
+ private final Server rpcServer;
+ /** The address for this RPC server. */
+ private final InetSocketAddress rpcAddress;
+
+ /** RPC clients to connect to the Namenodes. */
+ private final RouterRpcClient rpcClient;
+
+ /** Monitor metrics for the RPC calls. */
+ private final RouterRpcMonitor rpcMonitor;
+
+
+ /** Interface to identify the active NN for a nameservice or blockpool ID. */
+ private final ActiveNamenodeResolver namenodeResolver;
+
+ /** Interface to map global name space to HDFS subcluster name spaces. */
+ private final FileSubclusterResolver subclusterResolver;
+
+ /** If we are in safe mode, fail requests as if a standby NN. */
+ private volatile boolean safeMode;
+
+ /** Category of the operation that a thread is executing. */
+ private final ThreadLocal<OperationCategory> opCategory = new ThreadLocal<>();
+
+ // Modules implementing groups of RPC calls
+ /** Router Quota calls. */
+ private final Quota quotaCall;
+ /** Erasure coding calls. */
+ private final ErasureCoding erasureCoding;
+
+
+ /**
+ * Construct a router RPC server.
+ *
+ * @param configuration HDFS Configuration.
+ * @param nnResolver The NN resolver instance to determine active NNs in HA.
+ * @param fileResolver File resolver to resolve file paths to subclusters.
+ * @throws IOException If the RPC server could not be created.
+ */
+ public RouterRpcServer(Configuration configuration, Router router,
+ ActiveNamenodeResolver nnResolver, FileSubclusterResolver fileResolver)
+ throws IOException {
+ super(RouterRpcServer.class.getName());
+
+ this.conf = configuration;
+ this.router = router;
+ this.namenodeResolver = nnResolver;
+ this.subclusterResolver = fileResolver;
+
+ // User and group for reporting
+ this.superUser = System.getProperty("user.name");
+ this.superGroup = this.conf.get(
+ DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_KEY,
+ DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT);
+
+ // RPC server settings
+ int handlerCount = this.conf.getInt(DFS_ROUTER_HANDLER_COUNT_KEY,
+ DFS_ROUTER_HANDLER_COUNT_DEFAULT);
+
+ int readerCount = this.conf.getInt(DFS_ROUTER_READER_COUNT_KEY,
+ DFS_ROUTER_READER_COUNT_DEFAULT);
+
+ int handlerQueueSize = this.conf.getInt(DFS_ROUTER_HANDLER_QUEUE_SIZE_KEY,
+ DFS_ROUTER_HANDLER_QUEUE_SIZE_DEFAULT);
+
+ // Override Hadoop Common IPC setting
+ int readerQueueSize = this.conf.getInt(DFS_ROUTER_READER_QUEUE_SIZE_KEY,
+ DFS_ROUTER_READER_QUEUE_SIZE_DEFAULT);
+ this.conf.setInt(
+ CommonConfigurationKeys.IPC_SERVER_RPC_READ_CONNECTION_QUEUE_SIZE_KEY,
+ readerQueueSize);
+
+ RPC.setProtocolEngine(this.conf, ClientNamenodeProtocolPB.class,
+ ProtobufRpcEngine.class);
+
+ ClientNamenodeProtocolServerSideTranslatorPB
+ clientProtocolServerTranslator =
+ new ClientNamenodeProtocolServerSideTranslatorPB(this);
+ BlockingService clientNNPbService = ClientNamenodeProtocol
+ .newReflectiveBlockingService(clientProtocolServerTranslator);
+
+ InetSocketAddress confRpcAddress = conf.getSocketAddr(
+ RBFConfigKeys.DFS_ROUTER_RPC_BIND_HOST_KEY,
+ RBFConfigKeys.DFS_ROUTER_RPC_ADDRESS_KEY,
+ RBFConfigKeys.DFS_ROUTER_RPC_ADDRESS_DEFAULT,
+ RBFConfigKeys.DFS_ROUTER_RPC_PORT_DEFAULT);
+ LOG.info("RPC server binding to {} with {} handlers for Router {}",
+ confRpcAddress, handlerCount, this.router.getRouterId());
+
+ this.rpcServer = new RPC.Builder(this.conf)
+ .setProtocol(ClientNamenodeProtocolPB.class)
+ .setInstance(clientNNPbService)
+ .setBindAddress(confRpcAddress.getHostName())
+ .setPort(confRpcAddress.getPort())
+ .setNumHandlers(handlerCount)
+ .setnumReaders(readerCount)
+ .setQueueSizePerHandler(handlerQueueSize)
+ .setVerbose(false)
+ .build();
+ // We don't want the server to log the full stack trace for some exceptions
+ this.rpcServer.addTerseExceptions(
+ RemoteException.class,
+ StandbyException.class,
+ SafeModeException.class,
+ FileNotFoundException.class,
+ FileAlreadyExistsException.class,
+ AccessControlException.class,
+ LeaseExpiredException.class,
+ NotReplicatedYetException.class,
+ IOException.class);
+
+ // The RPC-server port can be ephemeral... ensure we have the correct info
+ InetSocketAddress listenAddress = this.rpcServer.getListenerAddress();
+ this.rpcAddress = new InetSocketAddress(
+ confRpcAddress.getHostName(), listenAddress.getPort());
+
+ // Create metrics monitor
+ Class<? extends RouterRpcMonitor> rpcMonitorClass = this.conf.getClass(
+ RBFConfigKeys.DFS_ROUTER_METRICS_CLASS,
+ RBFConfigKeys.DFS_ROUTER_METRICS_CLASS_DEFAULT,
+ RouterRpcMonitor.class);
+ this.rpcMonitor = ReflectionUtils.newInstance(rpcMonitorClass, conf);
+
+ // Create the client
+ this.rpcClient = new RouterRpcClient(this.conf, this.router.getRouterId(),
+ this.namenodeResolver, this.rpcMonitor);
+
+ // Initialize modules
+ this.quotaCall = new Quota(this.router, this);
+ this.erasureCoding = new ErasureCoding(this);
+ }
+
+ @Override
+ protected void serviceInit(Configuration configuration) throws Exception {
+ this.conf = configuration;
+
+ if (this.rpcMonitor == null) {
+ LOG.error("Cannot instantiate Router RPC metrics class");
+ } else {
+ this.rpcMonitor.init(this.conf, this, this.router.getStateStore());
+ }
+
+ super.serviceInit(configuration);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ if (this.rpcServer != null) {
+ this.rpcServer.start();
+ LOG.info("Router RPC up at: {}", this.getRpcAddress());
+ }
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (this.rpcServer != null) {
+ this.rpcServer.stop();
+ }
+ if (rpcMonitor != null) {
+ this.rpcMonitor.close();
+ }
+ super.serviceStop();
+ }
+
+ /**
+ * Get the RPC client to the Namenode.
+ *
+ * @return RPC clients to the Namenodes.
+ */
+ public RouterRpcClient getRPCClient() {
+ return rpcClient;
+ }
+
+ /**
+ * Get the RPC monitor and metrics.
+ *
+ * @return RPC monitor and metrics.
+ */
+ public RouterRpcMonitor getRPCMonitor() {
+ return rpcMonitor;
+ }
+
+ /**
+ * Allow access to the client RPC server for testing.
+ *
+ * @return The RPC server.
+ */
+ @VisibleForTesting
+ public Server getServer() {
+ return rpcServer;
+ }
+
+ /**
+ * Get the RPC address of the service.
+ *
+ * @return RPC service address.
+ */
+ public InetSocketAddress getRpcAddress() {
+ return rpcAddress;
+ }
+
+ /**
+ * Check if the Router is in safe mode. We should only see READ, WRITE, and
+ * UNCHECKED. It includes a default handler when we haven't implemented an
+ * operation. If not supported, it always throws an exception reporting the
+ * operation.
+ *
+ * @param op Category of the operation to check.
+ * @param supported If the operation is supported or not. If not, it will
+ * throw an UnsupportedOperationException.
+ * @throws SafeModeException If the Router is in safe mode and cannot serve
+ * client requests.
+ * @throws UnsupportedOperationException If the operation is not supported.
+ */
+ protected void checkOperation(OperationCategory op, boolean supported)
+ throws RouterSafeModeException, UnsupportedOperationException {
+ checkOperation(op);
+
+ if (!supported) {
+ if (rpcMonitor != null) {
+ rpcMonitor.proxyOpNotImplemented();
+ }
+ String methodName = getMethodName();
+ throw new UnsupportedOperationException(
+ "Operation \"" + methodName + "\" is not supported");
+ }
+ }
+
+ /**
+ * Check if the Router is in safe mode. We should only see READ, WRITE, and
+ * UNCHECKED. This function should be called by all ClientProtocol functions.
+ *
+ * @param op Category of the operation to check.
+ * @throws SafeModeException If the Router is in safe mode and cannot serve
+ * client requests.
+ */
+ protected void checkOperation(OperationCategory op)
+ throws RouterSafeModeException {
+ // Log the function we are currently calling.
+ if (rpcMonitor != null) {
+ rpcMonitor.startOp();
+ }
+ // Log the function we are currently calling.
+ if (LOG.isDebugEnabled()) {
+ String methodName = getMethodName();
+ LOG.debug("Proxying operation: {}", methodName);
+ }
+
+ // Store the category of the operation category for this thread
+ opCategory.set(op);
+
+ // We allow unchecked and read operations
+ if (op == OperationCategory.UNCHECKED || op == OperationCategory.READ) {
+ return;
+ }
+
+ if (safeMode) {
+ // Throw standby exception, router is not available
+ if (rpcMonitor != null) {
+ rpcMonitor.routerFailureSafemode();
+ }
+ throw new RouterSafeModeException(router.getRouterId(), op);
+ }
+ }
+
+ /**
+ * In safe mode all RPC requests will fail and return a standby exception.
+ * The client will try another Router, similar to the client retry logic for
+ * HA.
+ *
+ * @param mode True if enabled, False if disabled.
+ */
+ public void setSafeMode(boolean mode) {
+ this.safeMode = mode;
+ }
+
+ /**
+ * Check if the Router is in safe mode and cannot serve RPC calls.
+ *
+ * @return If the Router is in safe mode.
+ */
+ public boolean isInSafeMode() {
+ return this.safeMode;
+ }
+
+ @Override // ClientProtocol
+ public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ return null;
+ }
+
+ /**
+ * The the delegation token from each name service.
+ * @param renewer
+ * @return Name service -> Token.
+ * @throws IOException
+ */
+ public Map<FederationNamespaceInfo, Token<DelegationTokenIdentifier>>
+ getDelegationTokens(Text renewer) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ return null;
+ }
+
+ @Override // ClientProtocol
+ public long renewDelegationToken(Token<DelegationTokenIdentifier> token)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ return 0;
+ }
+
+ @Override // ClientProtocol
+ public void cancelDelegationToken(Token<DelegationTokenIdentifier> token)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // ClientProtocol
+ public LocatedBlocks getBlockLocations(String src, final long offset,
+ final long length) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod remoteMethod = new RemoteMethod("getBlockLocations",
+ new Class<?>[] {String.class, long.class, long.class},
+ new RemoteParam(), offset, length);
+ return (LocatedBlocks) rpcClient.invokeSequential(locations, remoteMethod,
+ LocatedBlocks.class, null);
+ }
+
+ @Override // ClientProtocol
+ public FsServerDefaults getServerDefaults() throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ RemoteMethod method = new RemoteMethod("getServerDefaults");
+ String ns = subclusterResolver.getDefaultNamespace();
+ return (FsServerDefaults) rpcClient.invokeSingle(ns, method);
+ }
+
+ @Override // ClientProtocol
+ public HdfsFileStatus create(String src, FsPermission masked,
+ String clientName, EnumSetWritable<CreateFlag> flag,
+ boolean createParent, short replication, long blockSize,
+ CryptoProtocolVersion[] supportedVersions, String ecPolicyName)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ RemoteLocation createLocation = getCreateLocation(src);
+ RemoteMethod method = new RemoteMethod("create",
+ new Class<?>[] {String.class, FsPermission.class, String.class,
+ EnumSetWritable.class, boolean.class, short.class,
+ long.class, CryptoProtocolVersion[].class,
+ String.class},
+ createLocation.getDest(), masked, clientName, flag, createParent,
+ replication, blockSize, supportedVersions, ecPolicyName);
+ return (HdfsFileStatus) rpcClient.invokeSingle(createLocation, method);
+ }
+
+ /**
+ * Get the location to create a file. It checks if the file already existed
+ * in one of the locations.
+ *
+ * @param src Path of the file to check.
+ * @return The remote location for this file.
+ * @throws IOException If the file has no creation location.
+ */
+ private RemoteLocation getCreateLocation(final String src)
+ throws IOException {
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ if (locations == null || locations.isEmpty()) {
+ throw new IOException("Cannot get locations to create " + src);
+ }
+
+ RemoteLocation createLocation = locations.get(0);
+ if (locations.size() > 1) {
+ try {
+ // Check if this file already exists in other subclusters
+ LocatedBlocks existingLocation = getBlockLocations(src, 0, 1);
+ if (existingLocation != null) {
+ // Forward to the existing location and let the NN handle the error
+ LocatedBlock existingLocationLastLocatedBlock =
+ existingLocation.getLastLocatedBlock();
+ if (existingLocationLastLocatedBlock == null) {
+ // The block has no blocks yet, check for the meta data
+ for (RemoteLocation location : locations) {
+ RemoteMethod method = new RemoteMethod("getFileInfo",
+ new Class<?>[] {String.class}, new RemoteParam());
+ if (rpcClient.invokeSingle(location, method) != null) {
+ createLocation = location;
+ break;
+ }
+ }
+ } else {
+ ExtendedBlock existingLocationLastBlock =
+ existingLocationLastLocatedBlock.getBlock();
+ String blockPoolId = existingLocationLastBlock.getBlockPoolId();
+ createLocation = getLocationForPath(src, true, blockPoolId);
+ }
+ }
+ } catch (FileNotFoundException fne) {
+ // Ignore if the file is not found
+ }
+ }
+ return createLocation;
+ }
+
+ // Medium
+ @Override // ClientProtocol
+ public LastBlockWithStatus append(String src, final String clientName,
+ final EnumSetWritable<CreateFlag> flag) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("append",
+ new Class<?>[] {String.class, String.class, EnumSetWritable.class},
+ new RemoteParam(), clientName, flag);
+ return (LastBlockWithStatus) rpcClient.invokeSequential(
+ locations, method, LastBlockWithStatus.class, null);
+ }
+
+ // Low
+ @Override // ClientProtocol
+ public boolean recoverLease(String src, String clientName)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("recoverLease",
+ new Class<?>[] {String.class, String.class}, new RemoteParam(),
+ clientName);
+ Object result = rpcClient.invokeSequential(
+ locations, method, Boolean.class, Boolean.TRUE);
+ return (boolean) result;
+ }
+
+ @Override // ClientProtocol
+ public boolean setReplication(String src, short replication)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("setReplication",
+ new Class<?>[] {String.class, short.class}, new RemoteParam(),
+ replication);
+ Object result = rpcClient.invokeSequential(
+ locations, method, Boolean.class, Boolean.TRUE);
+ return (boolean) result;
+ }
+
+ @Override
+ public void setStoragePolicy(String src, String policyName)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("setStoragePolicy",
+ new Class<?>[] {String.class, String.class},
+ new RemoteParam(), policyName);
+ rpcClient.invokeSequential(locations, method, null, null);
+ }
+
+ @Override
+ public BlockStoragePolicy[] getStoragePolicies() throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ RemoteMethod method = new RemoteMethod("getStoragePolicies");
+ String ns = subclusterResolver.getDefaultNamespace();
+ return (BlockStoragePolicy[]) rpcClient.invokeSingle(ns, method);
+ }
+
+ @Override // ClientProtocol
+ public void setPermission(String src, FsPermission permissions)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("setPermission",
+ new Class<?>[] {String.class, FsPermission.class},
+ new RemoteParam(), permissions);
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @Override // ClientProtocol
+ public void setOwner(String src, String username, String groupname)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("setOwner",
+ new Class<?>[] {String.class, String.class, String.class},
+ new RemoteParam(), username, groupname);
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ /**
+ * Excluded and favored nodes are not verified and will be ignored by
+ * placement policy if they are not in the same nameservice as the file.
+ */
+ @Override // ClientProtocol
+ public LocatedBlock addBlock(String src, String clientName,
+ ExtendedBlock previous, DatanodeInfo[] excludedNodes, long fileId,
+ String[] favoredNodes, EnumSet<AddBlockFlag> addBlockFlags)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("addBlock",
+ new Class<?>[] {String.class, String.class, ExtendedBlock.class,
+ DatanodeInfo[].class, long.class, String[].class,
+ EnumSet.class},
+ new RemoteParam(), clientName, previous, excludedNodes, fileId,
+ favoredNodes, addBlockFlags);
+ // TODO verify the excludedNodes and favoredNodes are acceptable to this NN
+ return (LocatedBlock) rpcClient.invokeSequential(
+ locations, method, LocatedBlock.class, null);
+ }
+
+ /**
+ * Excluded nodes are not verified and will be ignored by placement if they
+ * are not in the same nameservice as the file.
+ */
+ @Override // ClientProtocol
+ public LocatedBlock getAdditionalDatanode(final String src, final long fileId,
+ final ExtendedBlock blk, final DatanodeInfo[] existings,
+ final String[] existingStorageIDs, final DatanodeInfo[] excludes,
+ final int numAdditionalNodes, final String clientName)
+ throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod method = new RemoteMethod("getAdditionalDatanode",
+ new Class<?>[] {String.class, long.class, ExtendedBlock.class,
+ DatanodeInfo[].class, String[].class,
+ DatanodeInfo[].class, int.class, String.class},
+ new RemoteParam(), fileId, blk, existings, existingStorageIDs, excludes,
+ numAdditionalNodes, clientName);
+ return (LocatedBlock) rpcClient.invokeSequential(
+ locations, method, LocatedBlock.class, null);
+ }
+
+ @Override // ClientProtocol
+ public void abandonBlock(ExtendedBlock b, long fileId, String src,
+ String holder) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ RemoteMethod method = new RemoteMethod("abandonBlock",
+ new Class<?>[] {ExtendedBlock.class, long.class, String.class,
+ String.class},
+ b, fileId, new RemoteParam(), holder);
+ rpcClient.invokeSingle(b, method);
+ }
+
+ @Override // ClientProtocol
+ public boolean complete(String src, String clientName, ExtendedBlock last,
+ long fileId) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("complete",
+ new Class<?>[] {String.class, String.class, ExtendedBlock.class,
+ long.class},
+ new RemoteParam(), clientName, last, fileId);
+ // Complete can return true/false, so don't expect a result
+ return ((Boolean) rpcClient.invokeSequential(
+ locations, method, Boolean.class, null)).booleanValue();
+ }
+
+ @Override // ClientProtocol
+ public LocatedBlock updateBlockForPipeline(
+ ExtendedBlock block, String clientName) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ RemoteMethod method = new RemoteMethod("updateBlockForPipeline",
+ new Class<?>[] {ExtendedBlock.class, String.class},
+ block, clientName);
+ return (LocatedBlock) rpcClient.invokeSingle(block, method);
+ }
+
+ /**
+ * Datanode are not verified to be in the same nameservice as the old block.
+ * TODO This may require validation.
+ */
+ @Override // ClientProtocol
+ public void updatePipeline(String clientName, ExtendedBlock oldBlock,
+ ExtendedBlock newBlock, DatanodeID[] newNodes, String[] newStorageIDs)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ RemoteMethod method = new RemoteMethod("updatePipeline",
+ new Class<?>[] {String.class, ExtendedBlock.class, ExtendedBlock.class,
+ DatanodeID[].class, String[].class},
+ clientName, oldBlock, newBlock, newNodes, newStorageIDs);
+ rpcClient.invokeSingle(oldBlock, method);
+ }
+
+ @Override // ClientProtocol
+ public long getPreferredBlockSize(String src) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("getPreferredBlockSize",
+ new Class<?>[] {String.class}, new RemoteParam());
+ return ((Long) rpcClient.invokeSequential(
+ locations, method, Long.class, null)).longValue();
+ }
+
+ /**
+ * Determines combinations of eligible src/dst locations for a rename. A
+ * rename cannot change the namespace. Renames are only allowed if there is an
+ * eligible dst location in the same namespace as the source.
+ *
+ * @param srcLocations List of all potential source destinations where the
+ * path may be located. On return this list is trimmed to include
+ * only the paths that have corresponding destinations in the same
+ * namespace.
+ * @param dst The destination path
+ * @return A map of all eligible source namespaces and their corresponding
+ * replacement value.
+ * @throws IOException If the dst paths could not be determined.
+ */
+ private RemoteParam getRenameDestinations(
+ final List<RemoteLocation> srcLocations, final String dst)
+ throws IOException {
+
+ final List<RemoteLocation> dstLocations = getLocationsForPath(dst, true);
+ final Map<RemoteLocation, String> dstMap = new HashMap<>();
+
+ Iterator<RemoteLocation> iterator = srcLocations.iterator();
+ while (iterator.hasNext()) {
+ RemoteLocation srcLocation = iterator.next();
+ RemoteLocation eligibleDst =
+ getFirstMatchingLocation(srcLocation, dstLocations);
+ if (eligibleDst != null) {
+ // Use this dst for this source location
+ dstMap.put(srcLocation, eligibleDst.getDest());
+ } else {
+ // This src destination is not valid, remove from the source list
+ iterator.remove();
+ }
+ }
+ return new RemoteParam(dstMap);
+ }
+
+ /**
+ * Get first matching location.
+ *
+ * @param location Location we are looking for.
+ * @param locations List of locations.
+ * @return The first matchin location in the list.
+ */
+ private RemoteLocation getFirstMatchingLocation(RemoteLocation location,
+ List<RemoteLocation> locations) {
+ for (RemoteLocation loc : locations) {
+ if (loc.getNameserviceId().equals(location.getNameserviceId())) {
+ // Return first matching location
+ return loc;
+ }
+ }
+ return null;
+ }
+
+ @Deprecated
+ @Override // ClientProtocol
+ public boolean rename(final String src, final String dst)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> srcLocations = getLocationsForPath(src, true);
+ // srcLocations may be trimmed by getRenameDestinations()
+ final List<RemoteLocation> locs = new LinkedList<>(srcLocations);
+ RemoteParam dstParam = getRenameDestinations(locs, dst);
+ if (locs.isEmpty()) {
+ throw new IOException(
+ "Rename of " + src + " to " + dst + " is not allowed," +
+ " no eligible destination in the same namespace was found.");
+ }
+ RemoteMethod method = new RemoteMethod("rename",
+ new Class<?>[] {String.class, String.class},
+ new RemoteParam(), dstParam);
+ return ((Boolean) rpcClient.invokeSequential(
+ locs, method, Boolean.class, Boolean.TRUE)).booleanValue();
+ }
+
+ @Override // ClientProtocol
+ public void rename2(final String src, final String dst,
+ final Options.Rename... options) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> srcLocations = getLocationsForPath(src, true);
+ // srcLocations may be trimmed by getRenameDestinations()
+ final List<RemoteLocation> locs = new LinkedList<>(srcLocations);
+ RemoteParam dstParam = getRenameDestinations(locs, dst);
+ if (locs.isEmpty()) {
+ throw new IOException(
+ "Rename of " + src + " to " + dst + " is not allowed," +
+ " no eligible destination in the same namespace was found.");
+ }
+ RemoteMethod method = new RemoteMethod("rename2",
+ new Class<?>[] {String.class, String.class, options.getClass()},
+ new RemoteParam(), dstParam, options);
+ rpcClient.invokeSequential(locs, method, null, null);
+ }
+
+ @Override // ClientProtocol
+ public void concat(String trg, String[] src) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // See if the src and target files are all in the same namespace
+ LocatedBlocks targetBlocks = getBlockLocations(trg, 0, 1);
+ if (targetBlocks == null) {
+ throw new IOException("Cannot locate blocks for target file - " + trg);
+ }
+ LocatedBlock lastLocatedBlock = targetBlocks.getLastLocatedBlock();
+ String targetBlockPoolId = lastLocatedBlock.getBlock().getBlockPoolId();
+ for (String source : src) {
+ LocatedBlocks sourceBlocks = getBlockLocations(source, 0, 1);
+ if (sourceBlocks == null) {
+ throw new IOException(
+ "Cannot located blocks for source file " + source);
+ }
+ String sourceBlockPoolId =
+ sourceBlocks.getLastLocatedBlock().getBlock().getBlockPoolId();
+ if (!sourceBlockPoolId.equals(targetBlockPoolId)) {
+ throw new IOException("Cannot concatenate source file " + source
+ + " because it is located in a different namespace"
+ + " with block pool id " + sourceBlockPoolId
+ + " from the target file with block pool id "
+ + targetBlockPoolId);
+ }
+ }
+
+ // Find locations in the matching namespace.
+ final RemoteLocation targetDestination =
+ getLocationForPath(trg, true, targetBlockPoolId);
+ String[] sourceDestinations = new String[src.length];
+ for (int i = 0; i < src.length; i++) {
+ String sourceFile = src[i];
+ RemoteLocation location =
+ getLocationForPath(sourceFile, true, targetBlockPoolId);
+ sourceDestinations[i] = location.getDest();
+ }
+ // Invoke
+ RemoteMethod method = new RemoteMethod("concat",
+ new Class<?>[] {String.class, String[].class},
+ targetDestination.getDest(), sourceDestinations);
+ rpcClient.invokeSingle(targetDestination, method);
+ }
+
+ @Override // ClientProtocol
+ public boolean truncate(String src, long newLength, String clientName)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("truncate",
+ new Class<?>[] {String.class, long.class, String.class},
+ new RemoteParam(), newLength, clientName);
+ return ((Boolean) rpcClient.invokeSequential(locations, method,
+ Boolean.class, Boolean.TRUE)).booleanValue();
+ }
+
+ @Override // ClientProtocol
+ public boolean delete(String src, boolean recursive) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("delete",
+ new Class<?>[] {String.class, boolean.class}, new RemoteParam(),
+ recursive);
+ return ((Boolean) rpcClient.invokeSequential(locations, method,
+ Boolean.class, Boolean.TRUE)).booleanValue();
+ }
+
+ @Override // ClientProtocol
+ public boolean mkdirs(String src, FsPermission masked, boolean createParent)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ if (locations.size() > 1) {
+ // Check if this directory already exists
+ try {
+ HdfsFileStatus fileStatus = getFileInfo(src);
+ if (fileStatus != null) {
+ // When existing, the NN doesn't return an exception; return true
+ return true;
+ }
+ } catch (IOException ioe) {
+ // Can't query if this file exists or not.
+ LOG.error("Error requesting file info for path {} while proxing mkdirs",
+ src, ioe);
+ }
+ }
+
+ RemoteLocation firstLocation = locations.get(0);
+ RemoteMethod method = new RemoteMethod("mkdirs",
+ new Class<?>[] {String.class, FsPermission.class, boolean.class},
+ new RemoteParam(), masked, createParent);
+ return ((Boolean) rpcClient.invokeSingle(firstLocation, method))
+ .booleanValue();
+ }
+
+ @Override // ClientProtocol
+ public void renewLease(String clientName) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ RemoteMethod method = new RemoteMethod("renewLease",
+ new Class<?>[] {String.class}, clientName);
+ Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ rpcClient.invokeConcurrent(nss, method, false, false);
+ }
+
+ @Override // ClientProtocol
+ public DirectoryListing getListing(String src, byte[] startAfter,
+ boolean needLocation) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ // Locate the dir and fetch the listing
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("getListing",
+ new Class<?>[] {String.class, startAfter.getClass(), boolean.class},
+ new RemoteParam(), startAfter, needLocation);
+ Map<RemoteLocation, DirectoryListing> listings =
+ rpcClient.invokeConcurrent(
+ locations, method, false, false, DirectoryListing.class);
+
+ Map<String, HdfsFileStatus> nnListing = new TreeMap<>();
+ int totalRemainingEntries = 0;
+ int remainingEntries = 0;
+ boolean namenodeListingExists = false;
+ if (listings != null) {
+ // Check the subcluster listing with the smallest name
+ String lastName = null;
+ for (Entry<RemoteLocation, DirectoryListing> entry :
+ listings.entrySet()) {
+ RemoteLocation location = entry.getKey();
+ DirectoryListing listing = entry.getValue();
+ if (listing == null) {
+ LOG.debug("Cannot get listing from {}", location);
+ } else {
+ totalRemainingEntries += listing.getRemainingEntries();
+ HdfsFileStatus[] partialListing = listing.getPartialListing();
+ int length = partialListing.length;
+ if (length > 0) {
+ HdfsFileStatus lastLocalEntry = partialListing[length-1];
+ String lastLocalName = lastLocalEntry.getLocalName();
+ if (lastName == null || lastName.compareTo(lastLocalName) > 0) {
+ lastName = lastLocalName;
+ }
+ }
+ }
+ }
+
+ // Add existing entries
+ for (Object value : listings.values()) {
+ DirectoryListing listing = (DirectoryListing) value;
+ if (listing != null) {
+ namenodeListingExists = true;
+ for (HdfsFileStatus file : listing.getPartialListing()) {
+ String filename = file.getLocalName();
+ if (totalRemainingEntries > 0 && filename.compareTo(lastName) > 0) {
+ // Discarding entries further than the lastName
+ remainingEntries++;
+ } else {
+ nnListing.put(filename, file);
+ }
+ }
+ remainingEntries += listing.getRemainingEntries();
+ }
+ }
+ }
+
+ // Add mount points at this level in the tree
+ final List<String> children = subclusterResolver.getMountPoints(src);
+ if (children != null) {
+ // Get the dates for each mount point
+ Map<String, Long> dates = getMountPointDates(src);
+
+ // Create virtual folder with the mount name
+ for (String child : children) {
+ long date = 0;
+ if (dates != null && dates.containsKey(child)) {
+ date = dates.get(child);
+ }
+ // TODO add number of children
+ HdfsFileStatus dirStatus = getMountPointStatus(child, 0, date);
+
+ // This may overwrite existing listing entries with the mount point
+ // TODO don't add if already there?
+ nnListing.put(child, dirStatus);
+ }
+ }
+
+ if (!namenodeListingExists && nnListing.size() == 0) {
+ // NN returns a null object if the directory cannot be found and has no
+ // listing. If we didn't retrieve any NN listing data, and there are no
+ // mount points here, return null.
+ return null;
+ }
+
+ // Generate combined listing
+ HdfsFileStatus[] combinedData = new HdfsFileStatus[nnListing.size()];
+ combinedData = nnListing.values().toArray(combinedData);
+ return new DirectoryListing(combinedData, remainingEntries);
+ }
+
+ @Override // ClientProtocol
+ public HdfsFileStatus getFileInfo(String src) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod method = new RemoteMethod("getFileInfo",
+ new Class<?>[] {String.class}, new RemoteParam());
+ HdfsFileStatus ret = (HdfsFileStatus) rpcClient.invokeSequential(
+ locations, method, HdfsFileStatus.class, null);
+
+ // If there is no real path, check mount points
+ if (ret == null) {
+ List<String> children = subclusterResolver.getMountPoints(src);
+ if (children != null && !children.isEmpty()) {
+ Map<String, Long> dates = getMountPointDates(src);
+ long date = 0;
+ if (dates != null && dates.containsKey(src)) {
+ date = dates.get(src);
+ }
+ ret = getMountPointStatus(src, children.size(), date);
+ }
+ }
+
+ return ret;
+ }
+
+ @Override // ClientProtocol
+ public boolean isFileClosed(String src) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod method = new RemoteMethod("isFileClosed",
+ new Class<?>[] {String.class}, new RemoteParam());
+ return ((Boolean) rpcClient.invokeSequential(
+ locations, method, Boolean.class, Boolean.TRUE)).booleanValue();
+ }
+
+ @Override // ClientProtocol
+ public HdfsFileStatus getFileLinkInfo(String src) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod method = new RemoteMethod("getFileLinkInfo",
+ new Class<?>[] {String.class}, new RemoteParam());
+ return (HdfsFileStatus) rpcClient.invokeSequential(
+ locations, method, HdfsFileStatus.class, null);
+ }
+
+ @Override
+ public HdfsLocatedFileStatus getLocatedFileInfo(String src,
+ boolean needBlockToken) throws IOException {
+ checkOperation(OperationCategory.READ);
+ final List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod method = new RemoteMethod("getLocatedFileInfo",
+ new Class<?>[] {String.class, boolean.class}, new RemoteParam(),
+ Boolean.valueOf(needBlockToken));
+ return (HdfsLocatedFileStatus) rpcClient.invokeSequential(
+ locations, method, HdfsFileStatus.class, null);
+ }
+
+ @Override // ClientProtocol
+ public long[] getStats() throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+
+ RemoteMethod method = new RemoteMethod("getStats");
+ Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, long[]> results =
+ rpcClient.invokeConcurrent(nss, method, true, false, long[].class);
+ long[] combinedData = new long[STATS_ARRAY_LENGTH];
+ for (long[] data : results.values()) {
+ for (int i = 0; i < combinedData.length && i < data.length; i++) {
+ if (data[i] >= 0) {
+ combinedData[i] += data[i];
+ }
+ }
+ }
+ return combinedData;
+ }
+
+ @Override // ClientProtocol
+ public DatanodeInfo[] getDatanodeReport(DatanodeReportType type)
+ throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+ return getDatanodeReport(type, 0);
+ }
+
+ /**
+ * Get the datanode report with a timeout.
+ * @param type Type of the datanode.
+ * @param timeOutMs Time out for the reply in milliseconds.
+ * @return List of datanodes.
+ * @throws IOException If it cannot get the report.
+ */
+ public DatanodeInfo[] getDatanodeReport(
+ DatanodeReportType type, long timeOutMs) throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+
+ Map<String, DatanodeInfo> datanodesMap = new LinkedHashMap<>();
+ RemoteMethod method = new RemoteMethod("getDatanodeReport",
+ new Class<?>[] {DatanodeReportType.class}, type);
+
+ Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, DatanodeInfo[]> results =
+ rpcClient.invokeConcurrent(
+ nss, method, true, false, timeOutMs, DatanodeInfo[].class);
+ for (Entry<FederationNamespaceInfo, DatanodeInfo[]> entry :
+ results.entrySet()) {
+ FederationNamespaceInfo ns = entry.getKey();
+ DatanodeInfo[] result = entry.getValue();
+ for (DatanodeInfo node : result) {
+ String nodeId = node.getXferAddr();
+ if (!datanodesMap.containsKey(nodeId)) {
+ // Add the subcluster as a suffix to the network location
+ node.setNetworkLocation(
+ NodeBase.PATH_SEPARATOR_STR + ns.getNameserviceId() +
+ node.getNetworkLocation());
+ datanodesMap.put(nodeId, node);
+ } else {
+ LOG.debug("{} is in multiple subclusters", nodeId);
+ }
+ }
+ }
+ // Map -> Array
+ Collection<DatanodeInfo> datanodes = datanodesMap.values();
+ DatanodeInfo[] combinedData = new DatanodeInfo[datanodes.size()];
+ combinedData = datanodes.toArray(combinedData);
+ return combinedData;
+ }
+
+ @Override // ClientProtocol
+ public DatanodeStorageReport[] getDatanodeStorageReport(
+ DatanodeReportType type) throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+
+ Map<String, DatanodeStorageReport[]> dnSubcluster =
+ getDatanodeStorageReportMap(type);
+
+ // Avoid repeating machines in multiple subclusters
+ Map<String, DatanodeStorageReport> datanodesMap = new LinkedHashMap<>();
+ for (DatanodeStorageReport[] dns : dnSubcluster.values()) {
+ for (DatanodeStorageReport dn : dns) {
+ DatanodeInfo dnInfo = dn.getDatanodeInfo();
+ String nodeId = dnInfo.getXferAddr();
+ if (!datanodesMap.containsKey(nodeId)) {
+ datanodesMap.put(nodeId, dn);
+ }
+ // TODO merge somehow, right now it just takes the first one
+ }
+ }
+
+ Collection<DatanodeStorageReport> datanodes = datanodesMap.values();
+ DatanodeStorageReport[] combinedData =
+ new DatanodeStorageReport[datanodes.size()];
+ combinedData = datanodes.toArray(combinedData);
+ return combinedData;
+ }
+
+ /**
+ * Get the list of datanodes per subcluster.
+ *
+ * @param type Type of the datanodes to get.
+ * @return nsId -> datanode list.
+ * @throws IOException
+ */
+ public Map<String, DatanodeStorageReport[]> getDatanodeStorageReportMap(
+ DatanodeReportType type) throws IOException {
+
+ Map<String, DatanodeStorageReport[]> ret = new LinkedHashMap<>();
+ RemoteMethod method = new RemoteMethod("getDatanodeStorageReport",
+ new Class<?>[] {DatanodeReportType.class}, type);
+ Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, DatanodeStorageReport[]> results =
+ rpcClient.invokeConcurrent(
+ nss, method, true, false, DatanodeStorageReport[].class);
+ for (Entry<FederationNamespaceInfo, DatanodeStorageReport[]> entry :
+ results.entrySet()) {
+ FederationNamespaceInfo ns = entry.getKey();
+ String nsId = ns.getNameserviceId();
+ DatanodeStorageReport[] result = entry.getValue();
+ ret.put(nsId, result);
+ }
+ return ret;
+ }
+
+ @Override // ClientProtocol
+ public boolean setSafeMode(SafeModeAction action, boolean isChecked)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // Set safe mode in all the name spaces
+ RemoteMethod method = new RemoteMethod("setSafeMode",
+ new Class<?>[] {SafeModeAction.class, boolean.class},
+ action, isChecked);
+ Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, Boolean> results =
+ rpcClient.invokeConcurrent(nss, method, true, true, boolean.class);
+
+ // We only report true if all the name space are in safe mode
+ int numSafemode = 0;
+ for (boolean safemode : results.values()) {
+ if (safemode) {
+ numSafemode++;
+ }
+ }
+ return numSafemode == results.size();
+ }
+
+ @Override // ClientProtocol
+ public boolean restoreFailedStorage(String arg) throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+
+ RemoteMethod method = new RemoteMethod("restoreFailedStorage",
+ new Class<?>[] {String.class}, arg);
+ final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, Boolean> ret =
+ rpcClient.invokeConcurrent(nss, method, true, false, boolean.class);
+
+ boolean success = true;
+ for (boolean s : ret.values()) {
+ if (!s) {
+ success = false;
+ break;
+ }
+ }
+ return success;
+ }
+
+ @Override // ClientProtocol
+ public boolean saveNamespace(long timeWindow, long txGap) throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+
+ RemoteMethod method = new RemoteMethod("saveNamespace",
+ new Class<?>[] {Long.class, Long.class}, timeWindow, txGap);
+ final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, Boolean> ret =
+ rpcClient.invokeConcurrent(nss, method, true, false, boolean.class);
+
+ boolean success = true;
+ for (boolean s : ret.values()) {
+ if (!s) {
+ success = false;
+ break;
+ }
+ }
+ return success;
+ }
+
+ @Override // ClientProtocol
+ public long rollEdits() throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ RemoteMethod method = new RemoteMethod("rollEdits", new Class<?>[] {});
+ final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, Long> ret =
+ rpcClient.invokeConcurrent(nss, method, true, false, long.class);
+
+ // Return the maximum txid
+ long txid = 0;
+ for (long t : ret.values()) {
+ if (t > txid) {
+ txid = t;
+ }
+ }
+ return txid;
+ }
+
+ @Override // ClientProtocol
+ public void refreshNodes() throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+
+ RemoteMethod method = new RemoteMethod("refreshNodes", new Class<?>[] {});
+ final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ rpcClient.invokeConcurrent(nss, method, true, true);
+ }
+
+ @Override // ClientProtocol
+ public void finalizeUpgrade() throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+
+ RemoteMethod method = new RemoteMethod("finalizeUpgrade",
+ new Class<?>[] {});
+ final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ rpcClient.invokeConcurrent(nss, method, true, false);
+ }
+
+ @Override // ClientProtocol
+ public RollingUpgradeInfo rollingUpgrade(RollingUpgradeAction action)
+ throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ RemoteMethod method = new RemoteMethod("rollingUpgrade",
+ new Class<?>[] {RollingUpgradeAction.class}, action);
+ final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, RollingUpgradeInfo> ret =
+ rpcClient.invokeConcurrent(
+ nss, method, true, false, RollingUpgradeInfo.class);
+
+ // Return the first rolling upgrade info
+ RollingUpgradeInfo info = null;
+ for (RollingUpgradeInfo infoNs : ret.values()) {
+ if (info == null && infoNs != null) {
+ info = infoNs;
+ }
+ }
+ return info;
+ }
+
+ @Override // ClientProtocol
+ public void metaSave(String filename) throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+
+ RemoteMethod method = new RemoteMethod("metaSave",
+ new Class<?>[] {String.class}, filename);
+ final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ rpcClient.invokeConcurrent(nss, method, true, false);
+ }
+
+ @Override // ClientProtocol
+ public CorruptFileBlocks listCorruptFileBlocks(String path, String cookie)
+ throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ final List<RemoteLocation> locations = getLocationsForPath(path, false);
+ RemoteMethod method = new RemoteMethod("listCorruptFileBlocks",
+ new Class<?>[] {String.class, String.class},
+ new RemoteParam(), cookie);
+ return (CorruptFileBlocks) rpcClient.invokeSequential(
+ locations, method, CorruptFileBlocks.class, null);
+ }
+
+ @Override // ClientProtocol
+ public void setBalancerBandwidth(long bandwidth) throws IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+
+ RemoteMethod method = new RemoteMethod("setBalancerBandwidth",
+ new Class<?>[] {Long.class}, bandwidth);
+ final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ rpcClient.invokeConcurrent(nss, method, true, false);
+ }
+
+ @Override // ClientProtocol
+ public ContentSummary getContentSummary(String path) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ // Get the summaries from regular files
+ Collection<ContentSummary> summaries = new LinkedList<>();
+ FileNotFoundException notFoundException = null;
+ try {
+ final List<RemoteLocation> locations = getLocationsForPath(path, false);
+ RemoteMethod method = new RemoteMethod("getContentSummary",
+ new Class<?>[] {String.class}, new RemoteParam());
+ Map<RemoteLocation, ContentSummary> results =
+ rpcClient.invokeConcurrent(
+ locations, method, false, false, ContentSummary.class);
+ summaries.addAll(results.values());
+ } catch (FileNotFoundException e) {
+ notFoundException = e;
+ }
+
+ // Add mount points at this level in the tree
+ final List<String> children = subclusterResolver.getMountPoints(path);
+ if (children != null) {
+ for (String child : children) {
+ Path childPath = new Path(path, child);
+ try {
+ ContentSummary mountSummary = getContentSummary(childPath.toString());
+ if (mountSummary != null) {
+ summaries.add(mountSummary);
+ }
+ } catch (Exception e) {
+ LOG.error("Cannot get content summary for mount {}: {}",
+ childPath, e.getMessage());
+ }
+ }
+ }
+
+ // Throw original exception if no original nor mount points
+ if (summaries.isEmpty() && notFoundException != null) {
+ throw notFoundException;
+ }
+
+ return aggregateContentSummary(summaries);
+ }
+
+ /**
+ * Aggregate content summaries for each subcluster.
+ *
+ * @param summaries Collection of individual summaries.
+ * @return Aggregated content summary.
+ */
+ private ContentSummary aggregateContentSummary(
+ Collection<ContentSummary> summaries) {
+ if (summaries.size() == 1) {
+ return summaries.iterator().next();
+ }
+
+ long length = 0;
+ long fileCount = 0;
+ long directoryCount = 0;
+ long quota = 0;
+ long spaceConsumed = 0;
+ long spaceQuota = 0;
+
+ for (ContentSummary summary : summaries) {
+ length += summary.getLength();
+ fileCount += summary.getFileCount();
+ directoryCount += summary.getDirectoryCount();
+ quota += summary.getQuota();
+ spaceConsumed += summary.getSpaceConsumed();
+ spaceQuota += summary.getSpaceQuota();
+ }
+
+ ContentSummary ret = new ContentSummary.Builder()
+ .length(length)
+ .fileCount(fileCount)
+ .directoryCount(directoryCount)
+ .quota(quota)
+ .spaceConsumed(spaceConsumed)
+ .spaceQuota(spaceQuota)
+ .build();
+ return ret;
+ }
+
+ @Override // ClientProtocol
+ public void fsync(String src, long fileId, String clientName,
+ long lastBlockLength) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("fsync",
+ new Class<?>[] {String.class, long.class, String.class, long.class },
+ new RemoteParam(), fileId, clientName, lastBlockLength);
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @Override // ClientProtocol
+ public void setTimes(String src, long mtime, long atime) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("setTimes",
+ new Class<?>[] {String.class, long.class, long.class},
+ new RemoteParam(), mtime, atime);
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @Override // ClientProtocol
+ public void createSymlink(String target, String link, FsPermission dirPerms,
+ boolean createParent) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO Verify that the link location is in the same NS as the targets
+ final List<RemoteLocation> targetLocations =
+ getLocationsForPath(target, true);
+ final List<RemoteLocation> linkLocations =
+ getLocationsForPath(link, true);
+ RemoteLocation linkLocation = linkLocations.get(0);
+ RemoteMethod method = new RemoteMethod("createSymlink",
+ new Class<?>[] {String.class, String.class, FsPermission.class,
+ boolean.class},
+ new RemoteParam(), linkLocation.getDest(), dirPerms, createParent);
+ rpcClient.invokeSequential(targetLocations, method);
+ }
+
+ @Override // ClientProtocol
+ public String getLinkTarget(String path) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ final List<RemoteLocation> locations = getLocationsForPath(path, true);
+ RemoteMethod method = new RemoteMethod("getLinkTarget",
+ new Class<?>[] {String.class}, new RemoteParam());
+ return (String) rpcClient.invokeSequential(
+ locations, method, String.class, null);
+ }
+
+ @Override // Client Protocol
+ public void allowSnapshot(String snapshotRoot) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // Client Protocol
+ public void disallowSnapshot(String snapshot) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // ClientProtocol
+ public void renameSnapshot(String snapshotRoot, String snapshotOldName,
+ String snapshotNewName) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // Client Protocol
+ public SnapshottableDirectoryStatus[] getSnapshottableDirListing()
+ throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override // ClientProtocol
+ public SnapshotDiffReport getSnapshotDiffReport(String snapshotRoot,
+ String earlierSnapshotName, String laterSnapshotName) throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override // ClientProtocol
+ public SnapshotDiffReportListing getSnapshotDiffReportListing(
+ String snapshotRoot, String earlierSnapshotName, String laterSnapshotName,
+ byte[] startPath, int index) throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override // ClientProtocol
+ public long addCacheDirective(CacheDirectiveInfo path,
+ EnumSet<CacheFlag> flags) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ return 0;
+ }
+
+ @Override // ClientProtocol
+ public void modifyCacheDirective(CacheDirectiveInfo directive,
+ EnumSet<CacheFlag> flags) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // ClientProtocol
+ public void removeCacheDirective(long id) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // ClientProtocol
+ public BatchedEntries<CacheDirectiveEntry> listCacheDirectives(
+ long prevId, CacheDirectiveInfo filter) throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override // ClientProtocol
+ public void addCachePool(CachePoolInfo info) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // ClientProtocol
+ public void modifyCachePool(CachePoolInfo info) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // ClientProtocol
+ public void removeCachePool(String cachePoolName) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // ClientProtocol
+ public BatchedEntries<CachePoolEntry> listCachePools(String prevKey)
+ throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override // ClientProtocol
+ public void modifyAclEntries(String src, List<AclEntry> aclSpec)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("modifyAclEntries",
+ new Class<?>[] {String.class, List.class},
+ new RemoteParam(), aclSpec);
+ rpcClient.invokeSequential(locations, method, null, null);
+ }
+
+ @Override // ClienProtocol
+ public void removeAclEntries(String src, List<AclEntry> aclSpec)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("removeAclEntries",
+ new Class<?>[] {String.class, List.class},
+ new RemoteParam(), aclSpec);
+ rpcClient.invokeSequential(locations, method, null, null);
+ }
+
+ @Override // ClientProtocol
+ public void removeDefaultAcl(String src) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("removeDefaultAcl",
+ new Class<?>[] {String.class}, new RemoteParam());
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @Override // ClientProtocol
+ public void removeAcl(String src) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("removeAcl",
+ new Class<?>[] {String.class}, new RemoteParam());
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @Override // ClientProtocol
+ public void setAcl(String src, List<AclEntry> aclSpec) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod(
+ "setAcl", new Class<?>[] {String.class, List.class},
+ new RemoteParam(), aclSpec);
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @Override // ClientProtocol
+ public AclStatus getAclStatus(String src) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod method = new RemoteMethod("getAclStatus",
+ new Class<?>[] {String.class}, new RemoteParam());
+ return (AclStatus) rpcClient.invokeSequential(
+ locations, method, AclStatus.class, null);
+ }
+
+ @Override // ClientProtocol
+ public void createEncryptionZone(String src, String keyName)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("createEncryptionZone",
+ new Class<?>[] {String.class, String.class},
+ new RemoteParam(), keyName);
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @Override // ClientProtocol
+ public EncryptionZone getEZForPath(String src) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod method = new RemoteMethod("getEZForPath",
+ new Class<?>[] {String.class}, new RemoteParam());
+ return (EncryptionZone) rpcClient.invokeSequential(
+ locations, method, EncryptionZone.class, null);
+ }
+
+ @Override // ClientProtocol
+ public BatchedEntries<EncryptionZone> listEncryptionZones(long prevId)
+ throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override // ClientProtocol
+ public void reencryptEncryptionZone(String zone, ReencryptAction action)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // ClientProtocol
+ public BatchedEntries<ZoneReencryptionStatus> listReencryptionStatus(
+ long prevId) throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override // ClientProtocol
+ public void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("setXAttr",
+ new Class<?>[] {String.class, XAttr.class, EnumSet.class},
+ new RemoteParam(), xAttr, flag);
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override // ClientProtocol
+ public List<XAttr> getXAttrs(String src, List<XAttr> xAttrs)
+ throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod method = new RemoteMethod("getXAttrs",
+ new Class<?>[] {String.class, List.class}, new RemoteParam(), xAttrs);
+ return (List<XAttr>) rpcClient.invokeSequential(
+ locations, method, List.class, null);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override // ClientProtocol
+ public List<XAttr> listXAttrs(String src) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, false);
+ RemoteMethod method = new RemoteMethod("listXAttrs",
+ new Class<?>[] {String.class}, new RemoteParam());
+ return (List<XAttr>) rpcClient.invokeSequential(
+ locations, method, List.class, null);
+ }
+
+ @Override // ClientProtocol
+ public void removeXAttr(String src, XAttr xAttr) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(src, true);
+ RemoteMethod method = new RemoteMethod("removeXAttr",
+ new Class<?>[] {String.class, XAttr.class}, new RemoteParam(), xAttr);
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @Override // ClientProtocol
+ public void checkAccess(String path, FsAction mode) throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ // TODO handle virtual directories
+ final List<RemoteLocation> locations = getLocationsForPath(path, true);
+ RemoteMethod method = new RemoteMethod("checkAccess",
+ new Class<?>[] {String.class, FsAction.class},
+ new RemoteParam(), mode);
+ rpcClient.invokeSequential(locations, method);
+ }
+
+ @Override // ClientProtocol
+ public long getCurrentEditLogTxid() throws IOException {
+ checkOperation(OperationCategory.READ);
+
+ RemoteMethod method = new RemoteMethod(
+ "getCurrentEditLogTxid", new Class<?>[] {});
+ final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, Long> ret =
+ rpcClient.invokeConcurrent(nss, method, true, false, long.class);
+
+ // Return the maximum txid
+ long txid = 0;
+ for (long t : ret.values()) {
+ if (t > txid) {
+ txid = t;
+ }
+ }
+ return txid;
+ }
+
+ @Override // ClientProtocol
+ public EventBatchList getEditsFromTxid(long txid) throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override
+ public DataEncryptionKey getDataEncryptionKey() throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override
+ public String createSnapshot(String snapshotRoot, String snapshotName)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE);
+ return null;
+ }
+
+ @Override
+ public void deleteSnapshot(String snapshotRoot, String snapshotName)
+ throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override // ClientProtocol
+ public void setQuota(String path, long namespaceQuota, long storagespaceQuota,
+ StorageType type) throws IOException {
+ this.quotaCall.setQuota(path, namespaceQuota, storagespaceQuota, type);
+ }
+
+ @Override // ClientProtocol
+ public QuotaUsage getQuotaUsage(String path) throws IOException {
+ checkOperation(OperationCategory.READ);
+ return this.quotaCall.getQuotaUsage(path);
+ }
+
+ @Override
+ public void reportBadBlocks(LocatedBlock[] blocks) throws IOException {
+ checkOperation(OperationCategory.WRITE);
+
+ // Block pool id -> blocks
+ Map<String, List<LocatedBlock>> blockLocations = new HashMap<>();
+ for (LocatedBlock block : blocks) {
+ String bpId = block.getBlock().getBlockPoolId();
+ List<LocatedBlock> bpBlocks = blockLocations.get(bpId);
+ if (bpBlocks == null) {
+ bpBlocks = new LinkedList<>();
+ blockLocations.put(bpId, bpBlocks);
+ }
+ bpBlocks.add(block);
+ }
+
+ // Invoke each block pool
+ for (Entry<String, List<LocatedBlock>> entry : blockLocations.entrySet()) {
+ String bpId = entry.getKey();
+ List<LocatedBlock> bpBlocks = entry.getValue();
+
+ LocatedBlock[] bpBlocksArray =
+ bpBlocks.toArray(new LocatedBlock[bpBlocks.size()]);
+ RemoteMethod method = new RemoteMethod("reportBadBlocks",
+ new Class<?>[] {LocatedBlock[].class},
+ new Object[] {bpBlocksArray});
+ rpcClient.invokeSingleBlockPool(bpId, method);
+ }
+ }
+
+ @Override
+ public void unsetStoragePolicy(String src) throws IOException {
+ checkOperation(OperationCategory.WRITE, false);
+ }
+
+ @Override
+ public BlockStoragePolicy getStoragePolicy(String path) throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Override // ClientProtocol
+ public ErasureCodingPolicyInfo[] getErasureCodingPolicies()
+ throws IOException {
+ return erasureCoding.getErasureCodingPolicies();
+ }
+
+ @Override // ClientProtocol
+ public Map<String, String> getErasureCodingCodecs() throws IOException {
+ return erasureCoding.getErasureCodingCodecs();
+ }
+
+ @Override // ClientProtocol
+ public AddErasureCodingPolicyResponse[] addErasureCodingPolicies(
+ ErasureCodingPolicy[] policies) throws IOException {
+ return erasureCoding.addErasureCodingPolicies(policies);
+ }
+
+ @Override // ClientProtocol
+ public void removeErasureCodingPolicy(String ecPolicyName)
+ throws IOException {
+ erasureCoding.removeErasureCodingPolicy(ecPolicyName);
+ }
+
+ @Override // ClientProtocol
+ public void disableErasureCodingPolicy(String ecPolicyName)
+ throws IOException {
+ erasureCoding.disableErasureCodingPolicy(ecPolicyName);
+ }
+
+ @Override // ClientProtocol
+ public void enableErasureCodingPolicy(String ecPolicyName)
+ throws IOException {
+ erasureCoding.enableErasureCodingPolicy(ecPolicyName);
+ }
+
+ @Override // ClientProtocol
+ public ErasureCodingPolicy getErasureCodingPolicy(String src)
+ throws IOException {
+ return erasureCoding.getErasureCodingPolicy(src);
+ }
+
+ @Override // ClientProtocol
+ public void setErasureCodingPolicy(String src, String ecPolicyName)
+ throws IOException {
+ erasureCoding.setErasureCodingPolicy(src, ecPolicyName);
+ }
+
+ @Override // ClientProtocol
+ public void unsetErasureCodingPolicy(String src) throws IOException {
+ erasureCoding.unsetErasureCodingPolicy(src);
+ }
+
+ @Override
+ public ECBlockGroupStats getECBlockGroupStats() throws IOException {
+ return erasureCoding.getECBlockGroupStats();
+ }
+
+ @Override
+ public ReplicatedBlockStats getReplicatedBlockStats() throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ @Deprecated
+ @Override
+ public BatchedEntries<OpenFileEntry> listOpenFiles(long prevId)
+ throws IOException {
+ return listOpenFiles(prevId, EnumSet.of(OpenFilesType.ALL_OPEN_FILES),
+ OpenFilesIterator.FILTER_PATH_DEFAULT);
+ }
+
+ @Override
+ public BatchedEntries<OpenFileEntry> listOpenFiles(long prevId,
+ EnumSet<OpenFilesType> openFilesTypes, String path) throws IOException {
+ checkOperation(OperationCategory.READ, false);
+ return null;
+ }
+
+ /**
+ * Locate the location with the matching block pool id.
+ *
+ * @param path Path to check.
+ * @param failIfLocked Fail the request if locked (top mount point).
+ * @param blockPoolId The block pool ID of the namespace to search for.
+ * @return Prioritized list of locations in the federated cluster.
+ * @throws IOException if the location for this path cannot be determined.
+ */
+ private RemoteLocation getLocationForPath(
+ String path, boolean failIfLocked, String blockPoolId)
+ throws IOException {
+
+ final List<RemoteLocation> locations =
+ getLocationsForPath(path, failIfLocked);
+
+ String nameserviceId = null;
+ Set<FederationNamespaceInfo> namespaces =
+ this.namenodeResolver.getNamespaces();
+ for (FederationNamespaceInfo namespace : namespaces) {
+ if (namespace.getBlockPoolId().equals(blockPoolId)) {
+ nameserviceId = namespace.getNameserviceId();
+ break;
+ }
+ }
+ if (nameserviceId != null) {
+ for (RemoteLocation location : locations) {
+ if (location.getNameserviceId().equals(nameserviceId)) {
+ return location;
+ }
+ }
+ }
+ throw new IOException(
+ "Cannot locate a nameservice for block pool " + blockPoolId);
+ }
+
+ /**
+ * Get the possible locations of a path in the federated cluster.
+ *
+ * @param path Path to check.
+ * @param failIfLocked Fail the request if locked (top mount point).
+ * @return Prioritized list of locations in the federated cluster.
+ * @throws IOException If the location for this path cannot be determined.
+ */
+ protected List<RemoteLocation> getLocationsForPath(
+ String path, boolean failIfLocked) throws IOException {
+ try {
+ // Check the location for this path
+ final PathLocation location =
+ this.subclusterResolver.getDestinationForPath(path);
+ if (location == null) {
+ throw new IOException("Cannot find locations for " + path + " in " +
+ this.subclusterResolver);
+ }
+
+ // We may block some write operations
+ if (opCategory.get() == OperationCategory.WRITE) {
+ // Check if the path is in a read only mount point
+ if (isPathReadOnly(path)) {
+ if (this.rpcMonitor != null) {
+ this.rpcMonitor.routerFailureReadOnly();
+ }
+ throw new IOException(path + " is in a read only mount point");
+ }
+
+ // Check quota
+ if (this.router.isQuotaEnabled()) {
+ RouterQuotaUsage quotaUsage = this.router.getQuotaManager()
+ .getQuotaUsage(path);
+ if (quotaUsage != null) {
+ quotaUsage.verifyNamespaceQuota();
+ quotaUsage.verifyStoragespaceQuota();
+ }
+ }
+ }
+
+ return location.getDestinations();
+ } catch (IOException ioe) {
+ if (this.rpcMonitor != null) {
+ this.rpcMonitor.routerFailureStateStore();
+ }
+ throw ioe;
+ }
+ }
+
+ /**
+ * Check if a path is in a read only mount point.
+ *
+ * @param path Path to check.
+ * @return If the path is in a read only mount point.
+ */
+ private boolean isPathReadOnly(final String path) {
+ if (subclusterResolver instanceof MountTableResolver) {
+ try {
+ MountTableResolver mountTable = (MountTableResolver)subclusterResolver;
+ MountTable entry = mountTable.getMountPoint(path);
+ if (entry != null && entry.isReadOnly()) {
+ return true;
+ }
+ } catch (IOException e) {
+ LOG.error("Cannot get mount point: {}", e.getMessage());
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Get the modification dates for mount points.
+ *
+ * @param path Name of the path to start checking dates from.
+ * @return Map with the modification dates for all sub-entries.
+ */
+ private Map<String, Long> getMountPointDates(String path) {
+ Map<String, Long> ret = new TreeMap<>();
+ // TODO add when we have a Mount Table
+ return ret;
+ }
+
+ /**
+ * Create a new file status for a mount point.
+ *
+ * @param name Name of the mount point.
+ * @param childrenNum Number of children.
+ * @param date Map with the dates.
+ * @return New HDFS file status representing a mount point.
+ */
+ private HdfsFileStatus getMountPointStatus(
+ String name, int childrenNum, long date) {
+ long modTime = date;
+ long accessTime = date;
+ FsPermission permission = FsPermission.getDirDefault();
+ String owner = this.superUser;
+ String group = this.superGroup;
+ try {
+ // TODO support users, it should be the user for the pointed folder
+ UserGroupInformation ugi = getRemoteUser();
+ owner = ugi.getUserName();
+ group = ugi.getPrimaryGroupName();
+ } catch (IOException e) {
+ LOG.error("Cannot get the remote user: {}", e.getMessage());
+ }
+ long inodeId = 0;
+ return new HdfsFileStatus.Builder()
+ .isdir(true)
+ .mtime(modTime)
+ .atime(accessTime)
+ .perm(permission)
+ .owner(owner)
+ .group(group)
+ .symlink(new byte[0])
+ .path(DFSUtil.string2Bytes(name))
+ .fileId(inodeId)
+ .children(childrenNum)
+ .build();
+ }
+
+ /**
+ * Get the name of the method that is calling this function.
+ *
+ * @return Name of the method calling this function.
+ */
+ private static String getMethodName() {
+ final StackTraceElement[] stack = Thread.currentThread().getStackTrace();
+ String methodName = stack[3].getMethodName();
+ return methodName;
+ }
+
+ /**
+ * Get the user that is invoking this operation.
+ *
+ * @return Remote user group information.
+ * @throws IOException If we cannot get the user information.
+ */
+ static UserGroupInformation getRemoteUser() throws IOException {
+ UserGroupInformation ugi = Server.getRemoteUser();
+ return (ugi != null) ? ugi : UserGroupInformation.getCurrentUser();
+ }
+
+ /**
+ * Merge the outputs from multiple namespaces.
+ * @param map Namespace -> Output array.
+ * @param clazz Class of the values.
+ * @return Array with the outputs.
+ */
+ protected static <T> T[] merge(
+ Map<FederationNamespaceInfo, T[]> map, Class<T> clazz) {
+
+ // Put all results into a set to avoid repeats
+ Set<T> ret = new LinkedHashSet<>();
+ for (T[] values : map.values()) {
+ for (T val : values) {
+ ret.add(val);
+ }
+ }
+
+ return toArray(ret, clazz);
+ }
+
+ /**
+ * Convert a set of values into an array.
+ * @param set Input set.
+ * @param clazz Class of the values.
+ * @return Array with the values in set.
+ */
+ private static <T> T[] toArray(Set<T> set, Class<T> clazz) {
+ @SuppressWarnings("unchecked")
+ T[] combinedData = (T[]) Array.newInstance(clazz, set.size());
+ combinedData = set.toArray(combinedData);
+ return combinedData;
+ }
+
+ /**
+ * Get quota module implement.
+ */
+ public Quota getQuotaModule() {
+ return this.quotaCall;
+ }
+
+ /**
+ * Get RPC metrics info.
+ * @return The instance of FederationRPCMetrics.
+ */
+ public FederationRPCMetrics getRPCMetrics() {
+ return this.rpcMonitor.getRPCMetrics();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSafeModeException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSafeModeException.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSafeModeException.java
new file mode 100644
index 0000000..7a78b5b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSafeModeException.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
+import org.apache.hadoop.ipc.StandbyException;
+
+/**
+ * Exception that the Router throws when it is in safe mode. This extends
+ * {@link StandbyException} for the client to try another Router when it gets
+ * this exception.
+ */
+public class RouterSafeModeException extends StandbyException {
+
+ private static final long serialVersionUID = 453568188334993493L;
+
+ /** Identifier of the Router that generated this exception. */
+ private final String routerId;
+
+ /**
+ * Build a new Router safe mode exception.
+ * @param router Identifier of the Router.
+ * @param op Category of the operation (READ/WRITE).
+ */
+ public RouterSafeModeException(String router, OperationCategory op) {
+ super("Router " + router + " is in safe mode and cannot handle " + op
+ + " requests.");
+ this.routerId = router;
+ }
+
+ /**
+ * Get the id of the Router that generated this exception.
+ * @return Id of the Router that generated this exception.
+ */
+ public String getRouterId() {
+ return this.routerId;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMountTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMountTable.java
new file mode 100644
index 0000000..d5fb9ba
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMountTable.java
@@ -0,0 +1,259 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
+import org.apache.hadoop.hdfs.server.federation.router.RouterQuotaUsage;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Test;
+
+/**
+ * Test the Mount Table entry in the State Store.
+ */
+public class TestMountTable {
+
+ private static final String SRC = "/test";
+ private static final String DST_NS_0 = "ns0";
+ private static final String DST_NS_1 = "ns1";
+ private static final String DST_PATH_0 = "/path1";
+ private static final String DST_PATH_1 = "/path/path2";
+ private static final List<RemoteLocation> DST = new LinkedList<>();
+ static {
+ DST.add(new RemoteLocation(DST_NS_0, DST_PATH_0));
+ DST.add(new RemoteLocation(DST_NS_1, DST_PATH_1));
+ }
+ private static final Map<String, String> DST_MAP = new LinkedHashMap<>();
+ static {
+ DST_MAP.put(DST_NS_0, DST_PATH_0);
+ DST_MAP.put(DST_NS_1, DST_PATH_1);
+ }
+
+ private static final long DATE_CREATED = 100;
+ private static final long DATE_MOD = 200;
+
+ private static final long NS_COUNT = 1;
+ private static final long NS_QUOTA = 5;
+ private static final long SS_COUNT = 10;
+ private static final long SS_QUOTA = 100;
+
+ private static final RouterQuotaUsage QUOTA = new RouterQuotaUsage.Builder()
+ .fileAndDirectoryCount(NS_COUNT).quota(NS_QUOTA).spaceConsumed(SS_COUNT)
+ .spaceQuota(SS_QUOTA).build();
+
+ @Test
+ public void testGetterSetter() throws IOException {
+
+ MountTable record = MountTable.newInstance(SRC, DST_MAP);
+
+ validateDestinations(record);
+ assertEquals(SRC, record.getSourcePath());
+ assertEquals(DST, record.getDestinations());
+ assertTrue(DATE_CREATED > 0);
+ assertTrue(DATE_MOD > 0);
+
+ RouterQuotaUsage quota = record.getQuota();
+ assertEquals(0, quota.getFileAndDirectoryCount());
+ assertEquals(HdfsConstants.QUOTA_DONT_SET, quota.getQuota());
+ assertEquals(0, quota.getSpaceConsumed());
+ assertEquals(HdfsConstants.QUOTA_DONT_SET, quota.getSpaceQuota());
+
+ MountTable record2 =
+ MountTable.newInstance(SRC, DST_MAP, DATE_CREATED, DATE_MOD);
+
+ validateDestinations(record2);
+ assertEquals(SRC, record2.getSourcePath());
+ assertEquals(DST, record2.getDestinations());
+ assertEquals(DATE_CREATED, record2.getDateCreated());
+ assertEquals(DATE_MOD, record2.getDateModified());
+ assertFalse(record.isReadOnly());
+ assertEquals(DestinationOrder.HASH, record.getDestOrder());
+ }
+
+ @Test
+ public void testSerialization() throws IOException {
+ testSerialization(DestinationOrder.RANDOM);
+ testSerialization(DestinationOrder.HASH);
+ testSerialization(DestinationOrder.LOCAL);
+ }
+
+ private void testSerialization(final DestinationOrder order)
+ throws IOException {
+
+ MountTable record = MountTable.newInstance(
+ SRC, DST_MAP, DATE_CREATED, DATE_MOD);
+ record.setReadOnly(true);
+ record.setDestOrder(order);
+ record.setQuota(QUOTA);
+
+ StateStoreSerializer serializer = StateStoreSerializer.getSerializer();
+ String serializedString = serializer.serializeString(record);
+ MountTable record2 =
+ serializer.deserialize(serializedString, MountTable.class);
+
+ validateDestinations(record2);
+ assertEquals(SRC, record2.getSourcePath());
+ assertEquals(DST, record2.getDestinations());
+ assertEquals(DATE_CREATED, record2.getDateCreated());
+ assertEquals(DATE_MOD, record2.getDateModified());
+ assertTrue(record2.isReadOnly());
+ assertEquals(order, record2.getDestOrder());
+
+ RouterQuotaUsage quotaGet = record2.getQuota();
+ assertEquals(NS_COUNT, quotaGet.getFileAndDirectoryCount());
+ assertEquals(NS_QUOTA, quotaGet.getQuota());
+ assertEquals(SS_COUNT, quotaGet.getSpaceConsumed());
+ assertEquals(SS_QUOTA, quotaGet.getSpaceQuota());
+ }
+
+ @Test
+ public void testReadOnly() throws IOException {
+
+ Map<String, String> dest = new LinkedHashMap<>();
+ dest.put(DST_NS_0, DST_PATH_0);
+ dest.put(DST_NS_1, DST_PATH_1);
+ MountTable record1 = MountTable.newInstance(SRC, dest);
+ record1.setReadOnly(true);
+
+ validateDestinations(record1);
+ assertEquals(SRC, record1.getSourcePath());
+ assertEquals(DST, record1.getDestinations());
+ assertTrue(DATE_CREATED > 0);
+ assertTrue(DATE_MOD > 0);
+ assertTrue(record1.isReadOnly());
+
+ MountTable record2 = MountTable.newInstance(
+ SRC, DST_MAP, DATE_CREATED, DATE_MOD);
+ record2.setReadOnly(true);
+
+ validateDestinations(record2);
+ assertEquals(SRC, record2.getSourcePath());
+ assertEquals(DST, record2.getDestinations());
+ assertEquals(DATE_CREATED, record2.getDateCreated());
+ assertEquals(DATE_MOD, record2.getDateModified());
+ assertTrue(record2.isReadOnly());
+ }
+
+ @Test
+ public void testOrder() throws IOException {
+ testOrder(DestinationOrder.HASH);
+ testOrder(DestinationOrder.LOCAL);
+ testOrder(DestinationOrder.RANDOM);
+ }
+
+ private void testOrder(final DestinationOrder order)
+ throws IOException {
+
+ MountTable record = MountTable.newInstance(
+ SRC, DST_MAP, DATE_CREATED, DATE_MOD);
+ record.setDestOrder(order);
+
+ validateDestinations(record);
+ assertEquals(SRC, record.getSourcePath());
+ assertEquals(DST, record.getDestinations());
+ assertEquals(DATE_CREATED, record.getDateCreated());
+ assertEquals(DATE_MOD, record.getDateModified());
+ assertEquals(order, record.getDestOrder());
+ }
+
+ private void validateDestinations(MountTable record) {
+
+ assertEquals(SRC, record.getSourcePath());
+ assertEquals(2, record.getDestinations().size());
+
+ RemoteLocation location1 = record.getDestinations().get(0);
+ assertEquals(DST_NS_0, location1.getNameserviceId());
+ assertEquals(DST_PATH_0, location1.getDest());
+
+ RemoteLocation location2 = record.getDestinations().get(1);
+ assertEquals(DST_NS_1, location2.getNameserviceId());
+ assertEquals(DST_PATH_1, location2.getDest());
+ }
+
+ @Test
+ public void testQuota() throws IOException {
+ MountTable record = MountTable.newInstance(SRC, DST_MAP);
+ record.setQuota(QUOTA);
+
+ validateDestinations(record);
+ assertEquals(SRC, record.getSourcePath());
+ assertEquals(DST, record.getDestinations());
+ assertTrue(DATE_CREATED > 0);
+ assertTrue(DATE_MOD > 0);
+
+ RouterQuotaUsage quotaGet = record.getQuota();
+ assertEquals(NS_COUNT, quotaGet.getFileAndDirectoryCount());
+ assertEquals(NS_QUOTA, quotaGet.getQuota());
+ assertEquals(SS_COUNT, quotaGet.getSpaceConsumed());
+ assertEquals(SS_QUOTA, quotaGet.getSpaceQuota());
+ }
+
+ @Test
+ public void testValidation() throws IOException {
+ Map<String, String> destinations = new HashMap<>();
+ destinations.put("ns0", "/testValidate-dest");
+ try {
+ MountTable.newInstance("testValidate", destinations);
+ fail("Mount table entry should be created failed.");
+ } catch (Exception e) {
+ GenericTestUtils.assertExceptionContains(
+ MountTable.ERROR_MSG_MUST_START_WITH_BACK_SLASH, e);
+ }
+
+ destinations.clear();
+ destinations.put("ns0", "testValidate-dest");
+ try {
+ MountTable.newInstance("/testValidate", destinations);
+ fail("Mount table entry should be created failed.");
+ } catch (Exception e) {
+ GenericTestUtils.assertExceptionContains(
+ MountTable.ERROR_MSG_ALL_DEST_MUST_START_WITH_BACK_SLASH, e);
+ }
+
+ destinations.clear();
+ destinations.put("", "/testValidate-dest");
+ try {
+ MountTable.newInstance("/testValidate", destinations);
+ fail("Mount table entry should be created failed.");
+ } catch (Exception e) {
+ GenericTestUtils.assertExceptionContains(
+ MountTable.ERROR_MSG_INVAILD_DEST_NS, e);
+ }
+
+ destinations.clear();
+ destinations.put("ns0", "/testValidate-dest");
+ MountTable record = MountTable.newInstance("/testValidate", destinations);
+ assertNotNull(record);
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestRouterState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestRouterState.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestRouterState.java
new file mode 100644
index 0000000..dfe2bc9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestRouterState.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.server.federation.router.RouterServiceState;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.junit.Test;
+
+/**
+ * Test the Router State records.
+ */
+public class TestRouterState {
+
+ private static final String ADDRESS = "address";
+ private static final String VERSION = "version";
+ private static final String COMPILE_INFO = "compileInfo";
+ private static final long START_TIME = 100;
+ private static final long DATE_MODIFIED = 200;
+ private static final long DATE_CREATED = 300;
+ private static final long FILE_RESOLVER_VERSION = 500;
+ private static final RouterServiceState STATE = RouterServiceState.RUNNING;
+
+
+ private RouterState generateRecord() throws IOException {
+ RouterState record = RouterState.newInstance(ADDRESS, START_TIME, STATE);
+ record.setVersion(VERSION);
+ record.setCompileInfo(COMPILE_INFO);
+ record.setDateCreated(DATE_CREATED);
+ record.setDateModified(DATE_MODIFIED);
+
+ StateStoreVersion version = StateStoreVersion.newInstance();
+ version.setMountTableVersion(FILE_RESOLVER_VERSION);
+ record.setStateStoreVersion(version);
+ return record;
+ }
+
+ private void validateRecord(RouterState record) throws IOException {
+ assertEquals(ADDRESS, record.getAddress());
+ assertEquals(START_TIME, record.getDateStarted());
+ assertEquals(STATE, record.getStatus());
+ assertEquals(COMPILE_INFO, record.getCompileInfo());
+ assertEquals(VERSION, record.getVersion());
+
+ StateStoreVersion version = record.getStateStoreVersion();
+ assertEquals(FILE_RESOLVER_VERSION, version.getMountTableVersion());
+ }
+
+ @Test
+ public void testGetterSetter() throws IOException {
+ RouterState record = generateRecord();
+ validateRecord(record);
+ }
+
+ @Test
+ public void testSerialization() throws IOException {
+
+ RouterState record = generateRecord();
+
+ StateStoreSerializer serializer = StateStoreSerializer.getSerializer();
+ String serializedString = serializer.serializeString(record);
+ RouterState newRecord =
+ serializer.deserialize(serializedString, RouterState.class);
+
+ validateRecord(newRecord);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/resources/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/resources/hdfs-site.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/resources/hdfs-site.xml
new file mode 100644
index 0000000..e6dd2f1
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/resources/hdfs-site.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+ <!-- Disable min block size since most tests use tiny blocks -->
+ <property>
+ <name>dfs.namenode.fs-limits.min-block-size</name>
+ <value>0</value>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/resources/log4j.properties b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/resources/log4j.properties
new file mode 100644
index 0000000..c671ccc
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/resources/log4j.properties
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=info,stdout
+log4j.threshold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
index a99ea75..3b770f3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
@@ -15,9 +15,6 @@
<Package name="org.apache.hadoop.hdfs.qjournal.protocol" />
</Match>
<Match>
- <Package name="org.apache.hadoop.hdfs.federation.protocol.proto" />
- </Match>
- <Match>
<Bug pattern="EI_EXPOSE_REP" />
</Match>
<Match>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index 909e589..f8b1722 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -263,9 +263,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<copy file="${basedir}/src/main/webapps/proto-web.xml"
tofile="${project.build.directory}/webapps/nfs3/WEB-INF/web.xml"
filtering="true"/>
- <copy file="${basedir}/src/main/webapps/proto-web.xml"
- tofile="${project.build.directory}/webapps/router/WEB-INF/web.xml"
- filtering="true"/>
<copy toDir="${project.build.directory}/webapps">
<fileset dir="${basedir}/src/main/webapps">
<exclude name="**/proto-web.xml"/>
@@ -343,8 +340,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<include>QJournalProtocol.proto</include>
<include>editlog.proto</include>
<include>fsimage.proto</include>
- <include>FederationProtocol.proto</include>
- <include>RouterProtocol.proto</include>
<include>AliasMapProtocol.proto</include>
<include>InterQJournalProtocol.proto</include>
</includes>
@@ -395,7 +390,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<exclude>src/main/webapps/hdfs/robots.txt</exclude>
<exclude>src/main/webapps/journal/robots.txt</exclude>
<exclude>src/main/webapps/secondary/robots.txt</exclude>
- <exclude>src/main/webapps/router/robots.txt</exclude>
<exclude>src/contrib/**</exclude>
<exclude>src/site/resources/images/*</exclude>
<exclude>src/main/webapps/static/bootstrap-3.0.2/**</exclude>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 8ad9d65..b4b9d97 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.hdfs;
-import java.util.concurrent.TimeUnit;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
@@ -27,15 +25,6 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRackFaultTolerant;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.RamDiskReplicaLruTracker;
-import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
-import org.apache.hadoop.hdfs.server.federation.router.RouterRpcMonitor;
-import org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCPerformanceMonitor;
-import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
-import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreSerializerPBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl;
import org.apache.hadoop.http.HttpConfig;
/**
@@ -1188,199 +1177,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
"dfs.use.dfs.network.topology";
public static final boolean DFS_USE_DFS_NETWORK_TOPOLOGY_DEFAULT = true;
- // HDFS federation
- public static final String FEDERATION_PREFIX = "dfs.federation.";
-
- // HDFS Router-based federation
- public static final String FEDERATION_ROUTER_PREFIX =
- "dfs.federation.router.";
- public static final String DFS_ROUTER_DEFAULT_NAMESERVICE =
- FEDERATION_ROUTER_PREFIX + "default.nameserviceId";
- public static final String DFS_ROUTER_HANDLER_COUNT_KEY =
- FEDERATION_ROUTER_PREFIX + "handler.count";
- public static final int DFS_ROUTER_HANDLER_COUNT_DEFAULT = 10;
- public static final String DFS_ROUTER_READER_QUEUE_SIZE_KEY =
- FEDERATION_ROUTER_PREFIX + "reader.queue.size";
- public static final int DFS_ROUTER_READER_QUEUE_SIZE_DEFAULT = 100;
- public static final String DFS_ROUTER_READER_COUNT_KEY =
- FEDERATION_ROUTER_PREFIX + "reader.count";
- public static final int DFS_ROUTER_READER_COUNT_DEFAULT = 1;
- public static final String DFS_ROUTER_HANDLER_QUEUE_SIZE_KEY =
- FEDERATION_ROUTER_PREFIX + "handler.queue.size";
- public static final int DFS_ROUTER_HANDLER_QUEUE_SIZE_DEFAULT = 100;
- public static final String DFS_ROUTER_RPC_BIND_HOST_KEY =
- FEDERATION_ROUTER_PREFIX + "rpc-bind-host";
- public static final int DFS_ROUTER_RPC_PORT_DEFAULT = 8888;
- public static final String DFS_ROUTER_RPC_ADDRESS_KEY =
- FEDERATION_ROUTER_PREFIX + "rpc-address";
- public static final String DFS_ROUTER_RPC_ADDRESS_DEFAULT =
- "0.0.0.0:" + DFS_ROUTER_RPC_PORT_DEFAULT;
- public static final String DFS_ROUTER_RPC_ENABLE =
- FEDERATION_ROUTER_PREFIX + "rpc.enable";
- public static final boolean DFS_ROUTER_RPC_ENABLE_DEFAULT = true;
-
- public static final String DFS_ROUTER_METRICS_ENABLE =
- FEDERATION_ROUTER_PREFIX + "metrics.enable";
- public static final boolean DFS_ROUTER_METRICS_ENABLE_DEFAULT = true;
- public static final String DFS_ROUTER_METRICS_CLASS =
- FEDERATION_ROUTER_PREFIX + "metrics.class";
- public static final Class<? extends RouterRpcMonitor>
- DFS_ROUTER_METRICS_CLASS_DEFAULT =
- FederationRPCPerformanceMonitor.class;
-
- // HDFS Router heartbeat
- public static final String DFS_ROUTER_HEARTBEAT_ENABLE =
- FEDERATION_ROUTER_PREFIX + "heartbeat.enable";
- public static final boolean DFS_ROUTER_HEARTBEAT_ENABLE_DEFAULT = true;
- public static final String DFS_ROUTER_HEARTBEAT_INTERVAL_MS =
- FEDERATION_ROUTER_PREFIX + "heartbeat.interval";
- public static final long DFS_ROUTER_HEARTBEAT_INTERVAL_MS_DEFAULT =
- TimeUnit.SECONDS.toMillis(5);
- public static final String DFS_ROUTER_MONITOR_NAMENODE =
- FEDERATION_ROUTER_PREFIX + "monitor.namenode";
- public static final String DFS_ROUTER_MONITOR_LOCAL_NAMENODE =
- FEDERATION_ROUTER_PREFIX + "monitor.localnamenode.enable";
- public static final boolean DFS_ROUTER_MONITOR_LOCAL_NAMENODE_DEFAULT = true;
- public static final String DFS_ROUTER_HEARTBEAT_STATE_INTERVAL_MS =
- FEDERATION_ROUTER_PREFIX + "heartbeat-state.interval";
- public static final long DFS_ROUTER_HEARTBEAT_STATE_INTERVAL_MS_DEFAULT =
- TimeUnit.SECONDS.toMillis(5);
-
- // HDFS Router NN client
- public static final String DFS_ROUTER_NAMENODE_CONNECTION_POOL_SIZE =
- FEDERATION_ROUTER_PREFIX + "connection.pool-size";
- public static final int DFS_ROUTER_NAMENODE_CONNECTION_POOL_SIZE_DEFAULT =
- 64;
- public static final String DFS_ROUTER_NAMENODE_CONNECTION_POOL_CLEAN =
- FEDERATION_ROUTER_PREFIX + "connection.pool.clean.ms";
- public static final long DFS_ROUTER_NAMENODE_CONNECTION_POOL_CLEAN_DEFAULT =
- TimeUnit.MINUTES.toMillis(1);
- public static final String DFS_ROUTER_NAMENODE_CONNECTION_CLEAN_MS =
- FEDERATION_ROUTER_PREFIX + "connection.clean.ms";
- public static final long DFS_ROUTER_NAMENODE_CONNECTION_CLEAN_MS_DEFAULT =
- TimeUnit.SECONDS.toMillis(10);
-
- // HDFS Router RPC client
- public static final String DFS_ROUTER_CLIENT_THREADS_SIZE =
- FEDERATION_ROUTER_PREFIX + "client.thread-size";
- public static final int DFS_ROUTER_CLIENT_THREADS_SIZE_DEFAULT = 32;
- public static final String DFS_ROUTER_CLIENT_MAX_ATTEMPTS =
- FEDERATION_ROUTER_PREFIX + "client.retry.max.attempts";
- public static final int DFS_ROUTER_CLIENT_MAX_ATTEMPTS_DEFAULT = 3;
-
- // HDFS Router State Store connection
- public static final String FEDERATION_FILE_RESOLVER_CLIENT_CLASS =
- FEDERATION_ROUTER_PREFIX + "file.resolver.client.class";
- public static final Class<? extends FileSubclusterResolver>
- FEDERATION_FILE_RESOLVER_CLIENT_CLASS_DEFAULT =
- MountTableResolver.class;
- public static final String FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS =
- FEDERATION_ROUTER_PREFIX + "namenode.resolver.client.class";
- public static final Class<? extends ActiveNamenodeResolver>
- FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS_DEFAULT =
- MembershipNamenodeResolver.class;
-
- // HDFS Router-based federation State Store
- public static final String FEDERATION_STORE_PREFIX =
- FEDERATION_ROUTER_PREFIX + "store.";
-
- public static final String DFS_ROUTER_STORE_ENABLE =
- FEDERATION_STORE_PREFIX + "enable";
- public static final boolean DFS_ROUTER_STORE_ENABLE_DEFAULT = true;
-
- public static final String FEDERATION_STORE_SERIALIZER_CLASS =
- DFSConfigKeys.FEDERATION_STORE_PREFIX + "serializer";
- public static final Class<StateStoreSerializerPBImpl>
- FEDERATION_STORE_SERIALIZER_CLASS_DEFAULT =
- StateStoreSerializerPBImpl.class;
-
- public static final String FEDERATION_STORE_DRIVER_CLASS =
- FEDERATION_STORE_PREFIX + "driver.class";
- public static final Class<? extends StateStoreDriver>
- FEDERATION_STORE_DRIVER_CLASS_DEFAULT = StateStoreZooKeeperImpl.class;
-
- public static final String FEDERATION_STORE_CONNECTION_TEST_MS =
- FEDERATION_STORE_PREFIX + "connection.test";
- public static final long FEDERATION_STORE_CONNECTION_TEST_MS_DEFAULT =
- TimeUnit.MINUTES.toMillis(1);
-
- public static final String DFS_ROUTER_CACHE_TIME_TO_LIVE_MS =
- FEDERATION_ROUTER_PREFIX + "cache.ttl";
- public static final long DFS_ROUTER_CACHE_TIME_TO_LIVE_MS_DEFAULT =
- TimeUnit.MINUTES.toMillis(1);
-
- public static final String FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS =
- FEDERATION_STORE_PREFIX + "membership.expiration";
- public static final long FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS_DEFAULT =
- TimeUnit.MINUTES.toMillis(5);
- public static final String FEDERATION_STORE_ROUTER_EXPIRATION_MS =
- FEDERATION_STORE_PREFIX + "router.expiration";
- public static final long FEDERATION_STORE_ROUTER_EXPIRATION_MS_DEFAULT =
- TimeUnit.MINUTES.toMillis(5);
-
- // HDFS Router safe mode
- public static final String DFS_ROUTER_SAFEMODE_ENABLE =
- FEDERATION_ROUTER_PREFIX + "safemode.enable";
- public static final boolean DFS_ROUTER_SAFEMODE_ENABLE_DEFAULT = true;
- public static final String DFS_ROUTER_SAFEMODE_EXTENSION =
- FEDERATION_ROUTER_PREFIX + "safemode.extension";
- public static final long DFS_ROUTER_SAFEMODE_EXTENSION_DEFAULT =
- TimeUnit.SECONDS.toMillis(30);
- public static final String DFS_ROUTER_SAFEMODE_EXPIRATION =
- FEDERATION_ROUTER_PREFIX + "safemode.expiration";
- public static final long DFS_ROUTER_SAFEMODE_EXPIRATION_DEFAULT =
- 3 * DFS_ROUTER_CACHE_TIME_TO_LIVE_MS_DEFAULT;
-
- // HDFS Router-based federation mount table entries
- /** Maximum number of cache entries to have. */
- public static final String FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE =
- DFSConfigKeys.FEDERATION_ROUTER_PREFIX + "mount-table.max-cache-size";
- /** Remove cache entries if we have more than 10k. */
- public static final int FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE_DEFAULT = 10000;
-
- // HDFS Router-based federation admin
- public static final String DFS_ROUTER_ADMIN_HANDLER_COUNT_KEY =
- FEDERATION_ROUTER_PREFIX + "admin.handler.count";
- public static final int DFS_ROUTER_ADMIN_HANDLER_COUNT_DEFAULT = 1;
- public static final int DFS_ROUTER_ADMIN_PORT_DEFAULT = 8111;
- public static final String DFS_ROUTER_ADMIN_ADDRESS_KEY =
- FEDERATION_ROUTER_PREFIX + "admin-address";
- public static final String DFS_ROUTER_ADMIN_ADDRESS_DEFAULT =
- "0.0.0.0:" + DFS_ROUTER_ADMIN_PORT_DEFAULT;
- public static final String DFS_ROUTER_ADMIN_BIND_HOST_KEY =
- FEDERATION_ROUTER_PREFIX + "admin-bind-host";
- public static final String DFS_ROUTER_ADMIN_ENABLE =
- FEDERATION_ROUTER_PREFIX + "admin.enable";
- public static final boolean DFS_ROUTER_ADMIN_ENABLE_DEFAULT = true;
-
- // HDFS Router-based federation web
- public static final String DFS_ROUTER_HTTP_ENABLE =
- FEDERATION_ROUTER_PREFIX + "http.enable";
- public static final boolean DFS_ROUTER_HTTP_ENABLE_DEFAULT = true;
- public static final String DFS_ROUTER_HTTP_ADDRESS_KEY =
- FEDERATION_ROUTER_PREFIX + "http-address";
- public static final int DFS_ROUTER_HTTP_PORT_DEFAULT = 50071;
- public static final String DFS_ROUTER_HTTP_BIND_HOST_KEY =
- FEDERATION_ROUTER_PREFIX + "http-bind-host";
- public static final String DFS_ROUTER_HTTP_ADDRESS_DEFAULT =
- "0.0.0.0:" + DFS_ROUTER_HTTP_PORT_DEFAULT;
- public static final String DFS_ROUTER_HTTPS_ADDRESS_KEY =
- FEDERATION_ROUTER_PREFIX + "https-address";
- public static final int DFS_ROUTER_HTTPS_PORT_DEFAULT = 50072;
- public static final String DFS_ROUTER_HTTPS_BIND_HOST_KEY =
- FEDERATION_ROUTER_PREFIX + "https-bind-host";
- public static final String DFS_ROUTER_HTTPS_ADDRESS_DEFAULT =
- "0.0.0.0:" + DFS_ROUTER_HTTPS_PORT_DEFAULT;
-
- // HDFS Router-based federation quota
- public static final String DFS_ROUTER_QUOTA_ENABLE =
- FEDERATION_ROUTER_PREFIX + "quota.enable";
- public static final boolean DFS_ROUTER_QUOTA_ENABLED_DEFAULT = false;
- public static final String DFS_ROUTER_QUOTA_CACHE_UPATE_INTERVAL =
- FEDERATION_ROUTER_PREFIX + "quota-cache.update.interval";
- public static final long DFS_ROUTER_QUOTA_CACHE_UPATE_INTERVAL_DEFAULT =
- 60000;
-
// dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry
@Deprecated
public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolPB.java
deleted file mode 100644
index 96fa794..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolPB.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.protocolPB;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.protocol.proto.RouterProtocolProtos.RouterAdminProtocolService;
-import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector;
-import org.apache.hadoop.ipc.ProtocolInfo;
-import org.apache.hadoop.security.KerberosInfo;
-import org.apache.hadoop.security.token.TokenInfo;
-
-/**
- * Protocol that a clients use to communicate with the NameNode.
- * Note: This extends the protocolbuffer service based interface to
- * add annotations required for security.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Stable
-@KerberosInfo(
- serverPrincipal = DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY)
-@TokenInfo(DelegationTokenSelector.class)
-@ProtocolInfo(protocolName = HdfsConstants.CLIENT_NAMENODE_PROTOCOL_NAME,
- protocolVersion = 1)
-public interface RouterAdminProtocolPB extends
- RouterAdminProtocolService.BlockingInterface {
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java
deleted file mode 100644
index 159d5c2..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.protocolPB;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryResponseProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.EnterSafeModeRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.EnterSafeModeResponseProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesResponseProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetSafeModeRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetSafeModeResponseProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.LeaveSafeModeRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.LeaveSafeModeResponseProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryResponseProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryResponseProto;
-import org.apache.hadoop.hdfs.server.federation.router.RouterAdminServer;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.AddMountTableEntryRequestPBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.AddMountTableEntryResponsePBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.EnterSafeModeRequestPBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.EnterSafeModeResponsePBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.GetMountTableEntriesRequestPBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.GetMountTableEntriesResponsePBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.GetSafeModeRequestPBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.GetSafeModeResponsePBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.LeaveSafeModeRequestPBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.LeaveSafeModeResponsePBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryRequestPBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryResponsePBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.UpdateMountTableEntryRequestPBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.UpdateMountTableEntryResponsePBImpl;
-
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-
-/**
- * This class is used on the server side. Calls come across the wire for the for
- * protocol {@link RouterAdminProtocolPB}. This class translates the PB data
- * types to the native data types used inside the HDFS Router as specified in
- * the generic RouterAdminProtocol.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Stable
-public class RouterAdminProtocolServerSideTranslatorPB implements
- RouterAdminProtocolPB {
-
- private final RouterAdminServer server;
-
- /**
- * Constructor.
- * @param server The NN server.
- * @throws IOException
- */
- public RouterAdminProtocolServerSideTranslatorPB(RouterAdminServer server)
- throws IOException {
- this.server = server;
- }
-
- @Override
- public AddMountTableEntryResponseProto addMountTableEntry(
- RpcController controller, AddMountTableEntryRequestProto request)
- throws ServiceException {
-
- try {
- AddMountTableEntryRequest req =
- new AddMountTableEntryRequestPBImpl(request);
- AddMountTableEntryResponse response = server.addMountTableEntry(req);
- AddMountTableEntryResponsePBImpl responsePB =
- (AddMountTableEntryResponsePBImpl)response;
- return responsePB.getProto();
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-
- /**
- * Remove an entry from the mount table.
- */
- @Override
- public RemoveMountTableEntryResponseProto removeMountTableEntry(
- RpcController controller, RemoveMountTableEntryRequestProto request)
- throws ServiceException {
- try {
- RemoveMountTableEntryRequest req =
- new RemoveMountTableEntryRequestPBImpl(request);
- RemoveMountTableEntryResponse response =
- server.removeMountTableEntry(req);
- RemoveMountTableEntryResponsePBImpl responsePB =
- (RemoveMountTableEntryResponsePBImpl)response;
- return responsePB.getProto();
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-
- /**
- * Get matching mount table entries.
- */
- @Override
- public GetMountTableEntriesResponseProto getMountTableEntries(
- RpcController controller, GetMountTableEntriesRequestProto request)
- throws ServiceException {
- try {
- GetMountTableEntriesRequest req =
- new GetMountTableEntriesRequestPBImpl(request);
- GetMountTableEntriesResponse response = server.getMountTableEntries(req);
- GetMountTableEntriesResponsePBImpl responsePB =
- (GetMountTableEntriesResponsePBImpl)response;
- return responsePB.getProto();
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-
- /**
- * Update a single mount table entry.
- */
- @Override
- public UpdateMountTableEntryResponseProto updateMountTableEntry(
- RpcController controller, UpdateMountTableEntryRequestProto request)
- throws ServiceException {
- try {
- UpdateMountTableEntryRequest req =
- new UpdateMountTableEntryRequestPBImpl(request);
- UpdateMountTableEntryResponse response =
- server.updateMountTableEntry(req);
- UpdateMountTableEntryResponsePBImpl responsePB =
- (UpdateMountTableEntryResponsePBImpl)response;
- return responsePB.getProto();
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-
- @Override
- public EnterSafeModeResponseProto enterSafeMode(RpcController controller,
- EnterSafeModeRequestProto request) throws ServiceException {
- try {
- EnterSafeModeRequest req = new EnterSafeModeRequestPBImpl(request);
- EnterSafeModeResponse response = server.enterSafeMode(req);
- EnterSafeModeResponsePBImpl responsePB =
- (EnterSafeModeResponsePBImpl) response;
- return responsePB.getProto();
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-
- @Override
- public LeaveSafeModeResponseProto leaveSafeMode(RpcController controller,
- LeaveSafeModeRequestProto request) throws ServiceException {
- try {
- LeaveSafeModeRequest req = new LeaveSafeModeRequestPBImpl(request);
- LeaveSafeModeResponse response = server.leaveSafeMode(req);
- LeaveSafeModeResponsePBImpl responsePB =
- (LeaveSafeModeResponsePBImpl) response;
- return responsePB.getProto();
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-
- @Override
- public GetSafeModeResponseProto getSafeMode(RpcController controller,
- GetSafeModeRequestProto request) throws ServiceException {
- try {
- GetSafeModeRequest req = new GetSafeModeRequestPBImpl(request);
- GetSafeModeResponse response = server.getSafeMode(req);
- GetSafeModeResponsePBImpl responsePB =
- (GetSafeModeResponsePBImpl) response;
- return responsePB.getProto();
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java
deleted file mode 100644
index d6210ce..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.protocolPB;
-
-import java.io.Closeable;
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryResponseProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.EnterSafeModeRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.EnterSafeModeResponseProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesResponseProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetSafeModeRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetSafeModeResponseProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.LeaveSafeModeRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.LeaveSafeModeResponseProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryResponseProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryResponseProto;
-import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
-import org.apache.hadoop.hdfs.server.federation.router.RouterStateManager;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.AddMountTableEntryRequestPBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.AddMountTableEntryResponsePBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.EnterSafeModeResponsePBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.GetMountTableEntriesRequestPBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.GetMountTableEntriesResponsePBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.GetSafeModeResponsePBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.LeaveSafeModeResponsePBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryRequestPBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryResponsePBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.UpdateMountTableEntryRequestPBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.UpdateMountTableEntryResponsePBImpl;
-import org.apache.hadoop.ipc.ProtobufHelper;
-import org.apache.hadoop.ipc.ProtocolMetaInterface;
-import org.apache.hadoop.ipc.ProtocolTranslator;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.RpcClientUtil;
-
-import com.google.protobuf.ServiceException;
-
-/**
- * This class forwards NN's ClientProtocol calls as RPC calls to the NN server
- * while translating from the parameter types used in ClientProtocol to the
- * new PB types.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Stable
-public class RouterAdminProtocolTranslatorPB
- implements ProtocolMetaInterface, MountTableManager,
- Closeable, ProtocolTranslator, RouterStateManager {
- final private RouterAdminProtocolPB rpcProxy;
-
- public RouterAdminProtocolTranslatorPB(RouterAdminProtocolPB proxy) {
- rpcProxy = proxy;
- }
-
- @Override
- public void close() {
- RPC.stopProxy(rpcProxy);
- }
-
- @Override
- public Object getUnderlyingProxyObject() {
- return rpcProxy;
- }
-
- @Override
- public boolean isMethodSupported(String methodName) throws IOException {
- return RpcClientUtil.isMethodSupported(rpcProxy,
- RouterAdminProtocolPB.class, RPC.RpcKind.RPC_PROTOCOL_BUFFER,
- RPC.getProtocolVersion(RouterAdminProtocolPB.class), methodName);
- }
-
- @Override
- public AddMountTableEntryResponse addMountTableEntry(
- AddMountTableEntryRequest request) throws IOException {
- AddMountTableEntryRequestPBImpl requestPB =
- (AddMountTableEntryRequestPBImpl)request;
- AddMountTableEntryRequestProto proto = requestPB.getProto();
- try {
- AddMountTableEntryResponseProto response =
- rpcProxy.addMountTableEntry(null, proto);
- return new AddMountTableEntryResponsePBImpl(response);
- } catch (ServiceException e) {
- throw new IOException(ProtobufHelper.getRemoteException(e).getMessage());
- }
- }
-
- @Override
- public UpdateMountTableEntryResponse updateMountTableEntry(
- UpdateMountTableEntryRequest request) throws IOException {
- UpdateMountTableEntryRequestPBImpl requestPB =
- (UpdateMountTableEntryRequestPBImpl)request;
- UpdateMountTableEntryRequestProto proto = requestPB.getProto();
- try {
- UpdateMountTableEntryResponseProto response =
- rpcProxy.updateMountTableEntry(null, proto);
- return new UpdateMountTableEntryResponsePBImpl(response);
- } catch (ServiceException e) {
- throw new IOException(ProtobufHelper.getRemoteException(e).getMessage());
- }
- }
-
- @Override
- public RemoveMountTableEntryResponse removeMountTableEntry(
- RemoveMountTableEntryRequest request) throws IOException {
- RemoveMountTableEntryRequestPBImpl requestPB =
- (RemoveMountTableEntryRequestPBImpl)request;
- RemoveMountTableEntryRequestProto proto = requestPB.getProto();
- try {
- RemoveMountTableEntryResponseProto responseProto =
- rpcProxy.removeMountTableEntry(null, proto);
- return new RemoveMountTableEntryResponsePBImpl(responseProto);
- } catch (ServiceException e) {
- throw new IOException(ProtobufHelper.getRemoteException(e).getMessage());
- }
- }
-
- @Override
- public GetMountTableEntriesResponse getMountTableEntries(
- GetMountTableEntriesRequest request) throws IOException {
- GetMountTableEntriesRequestPBImpl requestPB =
- (GetMountTableEntriesRequestPBImpl)request;
- GetMountTableEntriesRequestProto proto = requestPB.getProto();
- try {
- GetMountTableEntriesResponseProto response =
- rpcProxy.getMountTableEntries(null, proto);
- return new GetMountTableEntriesResponsePBImpl(response);
- } catch (ServiceException e) {
- throw new IOException(ProtobufHelper.getRemoteException(e).getMessage());
- }
- }
-
- @Override
- public EnterSafeModeResponse enterSafeMode(EnterSafeModeRequest request)
- throws IOException {
- EnterSafeModeRequestProto proto =
- EnterSafeModeRequestProto.newBuilder().build();
- try {
- EnterSafeModeResponseProto response =
- rpcProxy.enterSafeMode(null, proto);
- return new EnterSafeModeResponsePBImpl(response);
- } catch (ServiceException e) {
- throw new IOException(ProtobufHelper.getRemoteException(e).getMessage());
- }
- }
-
- @Override
- public LeaveSafeModeResponse leaveSafeMode(LeaveSafeModeRequest request)
- throws IOException {
- LeaveSafeModeRequestProto proto =
- LeaveSafeModeRequestProto.newBuilder().build();
- try {
- LeaveSafeModeResponseProto response =
- rpcProxy.leaveSafeMode(null, proto);
- return new LeaveSafeModeResponsePBImpl(response);
- } catch (ServiceException e) {
- throw new IOException(ProtobufHelper.getRemoteException(e).getMessage());
- }
- }
-
- @Override
- public GetSafeModeResponse getSafeMode(GetSafeModeRequest request)
- throws IOException {
- GetSafeModeRequestProto proto =
- GetSafeModeRequestProto.newBuilder().build();
- try {
- GetSafeModeResponseProto response =
- rpcProxy.getSafeMode(null, proto);
- return new GetSafeModeResponsePBImpl(response);
- } catch (ServiceException e) {
- throw new IOException(ProtobufHelper.getRemoteException(e).getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java
deleted file mode 100644
index 79fb3e4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java
+++ /dev/null
@@ -1,223 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.metrics;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * JMX interface for the federation statistics.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public interface FederationMBean {
-
- /**
- * Get information about all the namenodes in the federation or null if
- * failure.
- * @return JSON with all the Namenodes.
- */
- String getNamenodes();
-
- /**
- * Get the latest info for each registered nameservice.
- * @return JSON with all the nameservices.
- */
- String getNameservices();
-
- /**
- * Get the mount table for the federated filesystem or null if failure.
- * @return JSON with the mount table.
- */
- String getMountTable();
-
- /**
- * Get the latest state of all routers.
- * @return JSON with all of the known routers or null if failure.
- */
- String getRouters();
-
- /**
- * Get the total capacity of the federated cluster.
- * @return Total capacity of the federated cluster.
- */
- long getTotalCapacity();
-
- /**
- * Get the used capacity of the federated cluster.
- * @return Used capacity of the federated cluster.
- */
- long getUsedCapacity();
-
- /**
- * Get the remaining capacity of the federated cluster.
- * @return Remaining capacity of the federated cluster.
- */
- long getRemainingCapacity();
-
- /**
- * Get the total remote storage capacity mounted in the federated cluster.
- * @return Remote capacity of the federated cluster.
- */
- long getProvidedSpace();
-
- /**
- * Get the number of nameservices in the federation.
- * @return Number of nameservices in the federation.
- */
- int getNumNameservices();
-
- /**
- * Get the number of namenodes.
- * @return Number of namenodes.
- */
- int getNumNamenodes();
-
- /**
- * Get the number of expired namenodes.
- * @return Number of expired namenodes.
- */
- int getNumExpiredNamenodes();
-
- /**
- * Get the number of live datanodes.
- * @return Number of live datanodes.
- */
- int getNumLiveNodes();
-
- /**
- * Get the number of dead datanodes.
- * @return Number of dead datanodes.
- */
- int getNumDeadNodes();
-
- /**
- * Get the number of decommissioning datanodes.
- * @return Number of decommissioning datanodes.
- */
- int getNumDecommissioningNodes();
-
- /**
- * Get the number of live decommissioned datanodes.
- * @return Number of live decommissioned datanodes.
- */
- int getNumDecomLiveNodes();
-
- /**
- * Get the number of dead decommissioned datanodes.
- * @return Number of dead decommissioned datanodes.
- */
- int getNumDecomDeadNodes();
-
- /**
- * Get Max, Median, Min and Standard Deviation of DataNodes usage.
- * @return the DataNode usage information, as a JSON string.
- */
- String getNodeUsage();
-
- /**
- * Get the number of blocks in the federation.
- * @return Number of blocks in the federation.
- */
- long getNumBlocks();
-
- /**
- * Get the number of missing blocks in the federation.
- * @return Number of missing blocks in the federation.
- */
- long getNumOfMissingBlocks();
-
- /**
- * Get the number of pending replication blocks in the federation.
- * @return Number of pending replication blocks in the federation.
- */
- long getNumOfBlocksPendingReplication();
-
- /**
- * Get the number of under replicated blocks in the federation.
- * @return Number of under replicated blocks in the federation.
- */
- long getNumOfBlocksUnderReplicated();
-
- /**
- * Get the number of pending deletion blocks in the federation.
- * @return Number of pending deletion blocks in the federation.
- */
- long getNumOfBlocksPendingDeletion();
-
- /**
- * Get the number of files in the federation.
- * @return Number of files in the federation.
- */
- long getNumFiles();
-
- /**
- * When the router started.
- * @return Date as a string the router started.
- */
- String getRouterStarted();
-
- /**
- * Get the version of the router.
- * @return Version of the router.
- */
- String getVersion();
-
- /**
- * Get the compilation date of the router.
- * @return Compilation date of the router.
- */
- String getCompiledDate();
-
- /**
- * Get the compilation info of the router.
- * @return Compilation info of the router.
- */
- String getCompileInfo();
-
- /**
- * Get the host and port of the router.
- * @return Host and port of the router.
- */
- String getHostAndPort();
-
- /**
- * Get the identifier of the router.
- * @return Identifier of the router.
- */
- String getRouterId();
-
- /**
- * Get the host and port of the router.
- * @return Host and port of the router.
- */
- String getClusterId();
-
- /**
- * Get the host and port of the router.
- * @return Host and port of the router.
- */
- String getBlockPoolId();
-
- /**
- * Get the current state of the router.
- *
- * @return String label for the current router state.
- */
- String getRouterStatus();
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterHeartbeatService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterHeartbeatService.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterHeartbeatService.java
deleted file mode 100644
index 427d514..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterHeartbeatService.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.retry.RetryNTimes;
-import org.apache.curator.test.TestingServer;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
-import org.apache.hadoop.hdfs.server.federation.store.RouterStore;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
-import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
-import org.apache.hadoop.hdfs.server.federation.store.records.StateStoreVersion;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.waitStateStore;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test cases for router heartbeat service.
- */
-public class TestRouterHeartbeatService {
- private Router router;
- private final String routerId = "router1";
- private TestingServer testingServer;
- private CuratorFramework curatorFramework;
-
- @Before
- public void setup() throws Exception {
- router = new Router();
- router.setRouterId(routerId);
- Configuration conf = new Configuration();
- conf.setInt(DFSConfigKeys.DFS_ROUTER_CACHE_TIME_TO_LIVE_MS, 1);
- Configuration routerConfig =
- new RouterConfigBuilder(conf).stateStore().build();
- routerConfig.setLong(DFSConfigKeys.FEDERATION_STORE_CONNECTION_TEST_MS,
- TimeUnit.HOURS.toMillis(1));
- routerConfig.setClass(DFSConfigKeys.FEDERATION_STORE_DRIVER_CLASS,
- StateStoreZooKeeperImpl.class, StateStoreDriver.class);
-
- testingServer = new TestingServer();
- String connectStr = testingServer.getConnectString();
- curatorFramework = CuratorFrameworkFactory.builder()
- .connectString(connectStr)
- .retryPolicy(new RetryNTimes(100, 100))
- .build();
- curatorFramework.start();
- routerConfig.set(CommonConfigurationKeys.ZK_ADDRESS, connectStr);
- router.init(routerConfig);
- router.start();
-
-
- waitStateStore(router.getStateStore(), TimeUnit.SECONDS.toMicros(10));
- }
-
- @Test
- public void testStateStoreUnavailable() throws IOException {
- curatorFramework.close();
- testingServer.stop();
- router.getStateStore().stop();
- // The driver is not ready
- assertFalse(router.getStateStore().isDriverReady());
-
- // Do a heartbeat, and no exception thrown out
- RouterHeartbeatService heartbeatService =
- new RouterHeartbeatService(router);
- heartbeatService.updateStateStore();
- }
-
- @Test
- public void testStateStoreAvailable() throws Exception {
- // The driver is ready
- StateStoreService stateStore = router.getStateStore();
- assertTrue(router.getStateStore().isDriverReady());
- RouterStore routerStore = router.getRouterStateManager();
-
- // No record about this router
- stateStore.refreshCaches(true);
- GetRouterRegistrationRequest request =
- GetRouterRegistrationRequest.newInstance(routerId);
- GetRouterRegistrationResponse response =
- router.getRouterStateManager().getRouterRegistration(request);
- RouterState routerState = response.getRouter();
- String id = routerState.getRouterId();
- StateStoreVersion version = routerState.getStateStoreVersion();
- assertNull(id);
- assertNull(version);
-
- // Do a heartbeat
- RouterHeartbeatService heartbeatService =
- new RouterHeartbeatService(router);
- heartbeatService.updateStateStore();
-
- // We should have a record
- stateStore.refreshCaches(true);
- request = GetRouterRegistrationRequest.newInstance(routerId);
- response = routerStore.getRouterRegistration(request);
- routerState = response.getRouter();
- id = routerState.getRouterId();
- version = routerState.getStateStoreVersion();
- assertNotNull(id);
- assertNotNull(version);
- }
-
- @After
- public void tearDown() throws IOException {
- if (curatorFramework != null) {
- curatorFramework.close();
- }
- if (testingServer != null) {
- testingServer.stop();
- }
- if (router != null) {
- router.shutDown();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
deleted file mode 100644
index 8702b3c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-import java.util.Collections;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
-import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
-import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
-import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
-import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
-import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- * Test a router end-to-end including the MountTable.
- */
-public class TestRouterMountTable {
-
- private static StateStoreDFSCluster cluster;
- private static NamenodeContext nnContext;
- private static RouterContext routerContext;
- private static MountTableResolver mountTable;
-
- @BeforeClass
- public static void globalSetUp() throws Exception {
-
- // Build and start a federated cluster
- cluster = new StateStoreDFSCluster(false, 1);
- Configuration conf = new RouterConfigBuilder()
- .stateStore()
- .admin()
- .rpc()
- .build();
- cluster.addRouterOverrides(conf);
- cluster.startCluster();
- cluster.startRouters();
- cluster.waitClusterUp();
-
- // Get the end points
- nnContext = cluster.getRandomNamenode();
- routerContext = cluster.getRandomRouter();
- Router router = routerContext.getRouter();
- mountTable = (MountTableResolver) router.getSubclusterResolver();
- }
-
- @AfterClass
- public static void tearDown() {
- if (cluster != null) {
- cluster.stopRouter(routerContext);
- cluster.shutdown();
- cluster = null;
- }
- }
-
- @Test
- public void testReadOnly() throws Exception {
-
- // Add a read only entry
- MountTable readOnlyEntry = MountTable.newInstance(
- "/readonly", Collections.singletonMap("ns0", "/testdir"));
- readOnlyEntry.setReadOnly(true);
- assertTrue(addMountTable(readOnlyEntry));
-
- // Add a regular entry
- MountTable regularEntry = MountTable.newInstance(
- "/regular", Collections.singletonMap("ns0", "/testdir"));
- assertTrue(addMountTable(regularEntry));
-
- // Create a folder which should show in all locations
- final FileSystem nnFs = nnContext.getFileSystem();
- final FileSystem routerFs = routerContext.getFileSystem();
- assertTrue(routerFs.mkdirs(new Path("/regular/newdir")));
-
- FileStatus dirStatusNn =
- nnFs.getFileStatus(new Path("/testdir/newdir"));
- assertTrue(dirStatusNn.isDirectory());
- FileStatus dirStatusRegular =
- routerFs.getFileStatus(new Path("/regular/newdir"));
- assertTrue(dirStatusRegular.isDirectory());
- FileStatus dirStatusReadOnly =
- routerFs.getFileStatus(new Path("/readonly/newdir"));
- assertTrue(dirStatusReadOnly.isDirectory());
-
- // It should fail writing into a read only path
- try {
- routerFs.mkdirs(new Path("/readonly/newdirfail"));
- fail("We should not be able to write into a read only mount point");
- } catch (IOException ioe) {
- String msg = ioe.getMessage();
- assertTrue(msg.startsWith(
- "/readonly/newdirfail is in a read only mount point"));
- }
- }
-
- /**
- * Add a mount table entry to the mount table through the admin API.
- * @param entry Mount table entry to add.
- * @return If it was succesfully added.
- * @throws IOException Problems adding entries.
- */
- private boolean addMountTable(final MountTable entry) throws IOException {
- RouterClient client = routerContext.getAdminClient();
- MountTableManager mountTableManager = client.getMountTableManager();
- AddMountTableEntryRequest addRequest =
- AddMountTableEntryRequest.newInstance(entry);
- AddMountTableEntryResponse addResponse =
- mountTableManager.addMountTableEntry(addRequest);
-
- // Reload the Router cache
- mountTable.loadCache(true);
-
- return addResponse.getStatus();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeMonitoring.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeMonitoring.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeMonitoring.java
deleted file mode 100644
index 3d58146..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeMonitoring.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_MONITOR_NAMENODE;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
-import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
-import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
-import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
-import org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver;
-import org.apache.hadoop.util.Time;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Test namenodes monitor behavior in the Router.
- */
-public class TestRouterNamenodeMonitoring {
-
- private static StateStoreDFSCluster cluster;
- private static RouterContext routerContext;
- private static MembershipNamenodeResolver resolver;
-
- private String ns0;
- private String ns1;
- private long initializedTime;
-
- @Before
- public void setUp() throws Exception {
- // Build and start a federated cluster with HA enabled
- cluster = new StateStoreDFSCluster(true, 2);
- // Enable heartbeat service and local heartbeat
- Configuration routerConf = new RouterConfigBuilder()
- .stateStore()
- .admin()
- .rpc()
- .enableLocalHeartbeat(true)
- .heartbeat()
- .build();
-
- // Specify local node (ns0.nn1) to monitor
- StringBuilder sb = new StringBuilder();
- ns0 = cluster.getNameservices().get(0);
- NamenodeContext context = cluster.getNamenodes(ns0).get(1);
- routerConf.set(DFS_NAMESERVICE_ID, ns0);
- routerConf.set(DFS_HA_NAMENODE_ID_KEY, context.getNamenodeId());
-
- // Specify namenodes (ns1.nn0,ns1.nn1) to monitor
- sb = new StringBuilder();
- ns1 = cluster.getNameservices().get(1);
- for (NamenodeContext ctx : cluster.getNamenodes(ns1)) {
- String suffix = ctx.getConfSuffix();
- if (sb.length() != 0) {
- sb.append(",");
- }
- sb.append(suffix);
- }
- // override with the namenodes: ns1.nn0,ns1.nn1
- routerConf.set(DFS_ROUTER_MONITOR_NAMENODE, sb.toString());
-
- cluster.addRouterOverrides(routerConf);
- cluster.startCluster();
- cluster.startRouters();
- cluster.waitClusterUp();
-
- routerContext = cluster.getRandomRouter();
- resolver = (MembershipNamenodeResolver) routerContext.getRouter()
- .getNamenodeResolver();
- initializedTime = Time.now();
- }
-
- @After
- public void tearDown() {
- if (cluster != null) {
- cluster.stopRouter(routerContext);
- cluster.shutdown();
- cluster = null;
- }
- }
-
- @Test
- public void testNamenodeMonitoring() throws Exception {
- // Set nn0 to active for all nameservices
- for (String ns : cluster.getNameservices()) {
- cluster.switchToActive(ns, "nn0");
- cluster.switchToStandby(ns, "nn1");
- }
-
- Collection<NamenodeHeartbeatService> heartbeatServices = routerContext
- .getRouter().getNamenodeHearbeatServices();
- // manually trigger the heartbeat
- for (NamenodeHeartbeatService service : heartbeatServices) {
- service.periodicInvoke();
- }
-
- resolver.loadCache(true);
- List<? extends FederationNamenodeContext> namespaceInfo0 =
- resolver.getNamenodesForNameserviceId(ns0);
- List<? extends FederationNamenodeContext> namespaceInfo1 =
- resolver.getNamenodesForNameserviceId(ns1);
-
- // The modified date won't be updated in ns0.nn0 since it isn't
- // monitored by the Router.
- assertEquals("nn0", namespaceInfo0.get(1).getNamenodeId());
- assertTrue(namespaceInfo0.get(1).getDateModified() < initializedTime);
-
- // other namnodes should be updated as expected
- assertEquals("nn1", namespaceInfo0.get(0).getNamenodeId());
- assertTrue(namespaceInfo0.get(0).getDateModified() > initializedTime);
-
- assertEquals("nn0", namespaceInfo1.get(0).getNamenodeId());
- assertTrue(namespaceInfo1.get(0).getDateModified() > initializedTime);
-
- assertEquals("nn1", namespaceInfo1.get(1).getNamenodeId());
- assertTrue(namespaceInfo1.get(1).getDateModified() > initializedTime);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java
deleted file mode 100644
index f125f8b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java
+++ /dev/null
@@ -1,452 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.UUID;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CreateFlag;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.QuotaUsage;
-import org.apache.hadoop.hdfs.DFSClient;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
-import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
-import org.apache.hadoop.hdfs.protocol.ClientProtocol;
-import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
-import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
-import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
-import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
-import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
-import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
-import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
-import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.google.common.base.Supplier;
-
-/**
- * Tests quota behaviors in Router-based Federation.
- */
-public class TestRouterQuota {
- private static StateStoreDFSCluster cluster;
- private static NamenodeContext nnContext1;
- private static NamenodeContext nnContext2;
- private static RouterContext routerContext;
- private static MountTableResolver resolver;
-
- private static final int BLOCK_SIZE = 512;
-
- @Before
- public void setUp() throws Exception {
-
- // Build and start a federated cluster
- cluster = new StateStoreDFSCluster(false, 2);
- Configuration routerConf = new RouterConfigBuilder()
- .stateStore()
- .admin()
- .quota()
- .rpc()
- .build();
- routerConf.set(DFSConfigKeys.DFS_ROUTER_QUOTA_CACHE_UPATE_INTERVAL, "2s");
-
- // override some hdfs settings that used in testing space quota
- Configuration hdfsConf = new Configuration(false);
- hdfsConf.setInt(HdfsClientConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
- hdfsConf.setInt(HdfsClientConfigKeys.DFS_REPLICATION_KEY, 1);
-
- cluster.addRouterOverrides(routerConf);
- cluster.addNamenodeOverrides(hdfsConf);
- cluster.startCluster();
- cluster.startRouters();
- cluster.waitClusterUp();
-
- nnContext1 = cluster.getNamenode(cluster.getNameservices().get(0), null);
- nnContext2 = cluster.getNamenode(cluster.getNameservices().get(1), null);
- routerContext = cluster.getRandomRouter();
- Router router = routerContext.getRouter();
- resolver = (MountTableResolver) router.getSubclusterResolver();
- }
-
- @After
- public void tearDown() {
- if (cluster != null) {
- cluster.stopRouter(routerContext);
- cluster.shutdown();
- cluster = null;
- }
- }
-
- @Test
- public void testNamespaceQuotaExceed() throws Exception {
- long nsQuota = 3;
- final FileSystem nnFs1 = nnContext1.getFileSystem();
- final FileSystem nnFs2 = nnContext2.getFileSystem();
-
- // Add two mount tables:
- // /nsquota --> ns0---testdir1
- // /nsquota/subdir --> ns1---testdir2
- nnFs1.mkdirs(new Path("/testdir1"));
- nnFs2.mkdirs(new Path("/testdir2"));
- MountTable mountTable1 = MountTable.newInstance("/nsquota",
- Collections.singletonMap("ns0", "/testdir1"));
-
- mountTable1.setQuota(new RouterQuotaUsage.Builder().quota(nsQuota).build());
- addMountTable(mountTable1);
-
- MountTable mountTable2 = MountTable.newInstance("/nsquota/subdir",
- Collections.singletonMap("ns1", "/testdir2"));
- mountTable2.setQuota(new RouterQuotaUsage.Builder().quota(nsQuota).build());
- addMountTable(mountTable2);
-
- final FileSystem routerFs = routerContext.getFileSystem();
- GenericTestUtils.waitFor(new Supplier<Boolean>() {
-
- @Override
- public Boolean get() {
- boolean isNsQuotaViolated = false;
- try {
- // create new directory to trigger NSQuotaExceededException
- routerFs.mkdirs(new Path("/nsquota/" + UUID.randomUUID()));
- routerFs.mkdirs(new Path("/nsquota/subdir/" + UUID.randomUUID()));
- } catch (NSQuotaExceededException e) {
- isNsQuotaViolated = true;
- } catch (IOException ignored) {
- }
- return isNsQuotaViolated;
- }
- }, 5000, 60000);
- // mkdir in real FileSystem should be okay
- nnFs1.mkdirs(new Path("/testdir1/" + UUID.randomUUID()));
- nnFs2.mkdirs(new Path("/testdir2/" + UUID.randomUUID()));
- }
-
- @Test
- public void testStorageSpaceQuotaaExceed() throws Exception {
- long ssQuota = 3071;
- final FileSystem nnFs1 = nnContext1.getFileSystem();
- final FileSystem nnFs2 = nnContext2.getFileSystem();
-
- // Add two mount tables:
- // /ssquota --> ns0---testdir3
- // /ssquota/subdir --> ns1---testdir4
- nnFs1.mkdirs(new Path("/testdir3"));
- nnFs2.mkdirs(new Path("/testdir4"));
- MountTable mountTable1 = MountTable.newInstance("/ssquota",
- Collections.singletonMap("ns0", "/testdir3"));
-
- mountTable1
- .setQuota(new RouterQuotaUsage.Builder().spaceQuota(ssQuota).build());
- addMountTable(mountTable1);
-
- MountTable mountTable2 = MountTable.newInstance("/ssquota/subdir",
- Collections.singletonMap("ns1", "/testdir4"));
- mountTable2
- .setQuota(new RouterQuotaUsage.Builder().spaceQuota(ssQuota).build());
- addMountTable(mountTable2);
-
- DFSClient routerClient = routerContext.getClient();
- routerClient.create("/ssquota/file", true).close();
- routerClient.create("/ssquota/subdir/file", true).close();
-
- GenericTestUtils.waitFor(new Supplier<Boolean>() {
-
- @Override
- public Boolean get() {
- boolean isDsQuotaViolated = false;
- try {
- // append data to trigger NSQuotaExceededException
- appendData("/ssquota/file", routerClient, BLOCK_SIZE);
- appendData("/ssquota/subdir/file", routerClient, BLOCK_SIZE);
- } catch (DSQuotaExceededException e) {
- isDsQuotaViolated = true;
- } catch (IOException ignored) {
- }
- return isDsQuotaViolated;
- }
- }, 5000, 60000);
-
- // append data to destination path in real FileSystem should be okay
- appendData("/testdir3/file", nnContext1.getClient(), BLOCK_SIZE);
- appendData("/testdir4/file", nnContext2.getClient(), BLOCK_SIZE);
- }
-
- /**
- * Add a mount table entry to the mount table through the admin API.
- * @param entry Mount table entry to add.
- * @return If it was successfully added.
- * @throws IOException Problems adding entries.
- */
- private boolean addMountTable(final MountTable entry) throws IOException {
- RouterClient client = routerContext.getAdminClient();
- MountTableManager mountTableManager = client.getMountTableManager();
- AddMountTableEntryRequest addRequest =
- AddMountTableEntryRequest.newInstance(entry);
- AddMountTableEntryResponse addResponse =
- mountTableManager.addMountTableEntry(addRequest);
-
- // Reload the Router cache
- resolver.loadCache(true);
-
- return addResponse.getStatus();
- }
-
- /**
- * Append data in specified file.
- * @param path Path of file.
- * @param client DFS Client.
- * @param dataLen The length of write data.
- * @throws IOException
- */
- private void appendData(String path, DFSClient client, int dataLen)
- throws IOException {
- EnumSet<CreateFlag> createFlag = EnumSet.of(CreateFlag.APPEND);
- HdfsDataOutputStream stream = client.append(path, 1024, createFlag, null,
- null);
- byte[] data = new byte[dataLen];
- stream.write(data);
- stream.close();
- }
-
- @Test
- public void testSetQuota() throws Exception {
- long nsQuota = 5;
- long ssQuota = 100;
- final FileSystem nnFs1 = nnContext1.getFileSystem();
- final FileSystem nnFs2 = nnContext2.getFileSystem();
-
- // Add two mount tables:
- // /setquota --> ns0---testdir5
- // /setquota/subdir --> ns1---testdir6
- nnFs1.mkdirs(new Path("/testdir5"));
- nnFs2.mkdirs(new Path("/testdir6"));
- MountTable mountTable1 = MountTable.newInstance("/setquota",
- Collections.singletonMap("ns0", "/testdir5"));
- mountTable1
- .setQuota(new RouterQuotaUsage.Builder().quota(nsQuota)
- .spaceQuota(ssQuota).build());
- addMountTable(mountTable1);
-
- // don't set quota for subpath of mount table
- MountTable mountTable2 = MountTable.newInstance("/setquota/subdir",
- Collections.singletonMap("ns1", "/testdir6"));
- addMountTable(mountTable2);
-
- RouterQuotaUpdateService updateService = routerContext.getRouter()
- .getQuotaCacheUpdateService();
- // ensure setQuota RPC call was invoked
- updateService.periodicInvoke();
-
- ClientProtocol client1 = nnContext1.getClient().getNamenode();
- ClientProtocol client2 = nnContext2.getClient().getNamenode();
- final QuotaUsage quota1 = client1.getQuotaUsage("/testdir5");
- final QuotaUsage quota2 = client2.getQuotaUsage("/testdir6");
-
- assertEquals(nsQuota, quota1.getQuota());
- assertEquals(ssQuota, quota1.getSpaceQuota());
- assertEquals(nsQuota, quota2.getQuota());
- assertEquals(ssQuota, quota2.getSpaceQuota());
- }
-
- @Test
- public void testGetQuota() throws Exception {
- long nsQuota = 10;
- long ssQuota = 100;
- final FileSystem nnFs1 = nnContext1.getFileSystem();
- final FileSystem nnFs2 = nnContext2.getFileSystem();
-
- // Add two mount tables:
- // /getquota --> ns0---/testdir7
- // /getquota/subdir1 --> ns0---/testdir7/subdir
- // /getquota/subdir2 --> ns1---/testdir8
- nnFs1.mkdirs(new Path("/testdir7"));
- nnFs1.mkdirs(new Path("/testdir7/subdir"));
- nnFs2.mkdirs(new Path("/testdir8"));
- MountTable mountTable1 = MountTable.newInstance("/getquota",
- Collections.singletonMap("ns0", "/testdir7"));
- mountTable1
- .setQuota(new RouterQuotaUsage.Builder().quota(nsQuota)
- .spaceQuota(ssQuota).build());
- addMountTable(mountTable1);
-
- MountTable mountTable2 = MountTable.newInstance("/getquota/subdir1",
- Collections.singletonMap("ns0", "/testdir7/subdir"));
- addMountTable(mountTable2);
-
- MountTable mountTable3 = MountTable.newInstance("/getquota/subdir2",
- Collections.singletonMap("ns1", "/testdir8"));
- addMountTable(mountTable3);
-
- // use router client to create new files
- DFSClient routerClient = routerContext.getClient();
- routerClient.create("/getquota/file", true).close();
- routerClient.create("/getquota/subdir1/file", true).close();
- routerClient.create("/getquota/subdir2/file", true).close();
-
- ClientProtocol clientProtocol = routerContext.getClient().getNamenode();
- RouterQuotaUpdateService updateService = routerContext.getRouter()
- .getQuotaCacheUpdateService();
- updateService.periodicInvoke();
- final QuotaUsage quota = clientProtocol.getQuotaUsage("/getquota");
- // the quota should be aggregated
- assertEquals(6, quota.getFileAndDirectoryCount());
- }
-
- @Test
- public void testStaleQuotaRemoving() throws Exception {
- long nsQuota = 20;
- long ssQuota = 200;
- String stalePath = "/stalequota";
- final FileSystem nnFs1 = nnContext1.getFileSystem();
-
- // Add one mount tables:
- // /stalequota --> ns0---/testdir9
- nnFs1.mkdirs(new Path("/testdir9"));
- MountTable mountTable = MountTable.newInstance(stalePath,
- Collections.singletonMap("ns0", "/testdir9"));
- mountTable.setQuota(new RouterQuotaUsage.Builder().quota(nsQuota)
- .spaceQuota(ssQuota).build());
- addMountTable(mountTable);
-
- // Call periodicInvoke to ensure quota for stalePath was
- // loaded into quota manager.
- RouterQuotaUpdateService updateService = routerContext.getRouter()
- .getQuotaCacheUpdateService();
- updateService.periodicInvoke();
-
- // use quota manager to get its quota usage and do verification
- RouterQuotaManager quotaManager = routerContext.getRouter()
- .getQuotaManager();
- RouterQuotaUsage quota = quotaManager.getQuotaUsage(stalePath);
- assertEquals(nsQuota, quota.getQuota());
- assertEquals(ssQuota, quota.getSpaceQuota());
-
- // remove stale path entry
- removeMountTable(stalePath);
- updateService.periodicInvoke();
- // the stale entry should be removed and we will get null
- quota = quotaManager.getQuotaUsage(stalePath);
- assertNull(quota);
- }
-
- /**
- * Remove a mount table entry to the mount table through the admin API.
- * @param entry Mount table entry to remove.
- * @return If it was successfully removed.
- * @throws IOException Problems removing entries.
- */
- private boolean removeMountTable(String path) throws IOException {
- RouterClient client = routerContext.getAdminClient();
- MountTableManager mountTableManager = client.getMountTableManager();
- RemoveMountTableEntryRequest removeRequest = RemoveMountTableEntryRequest
- .newInstance(path);
- RemoveMountTableEntryResponse removeResponse = mountTableManager
- .removeMountTableEntry(removeRequest);
-
- // Reload the Router cache
- resolver.loadCache(true);
- return removeResponse.getStatus();
- }
-
- @Test
- public void testQuotaUpdating() throws Exception {
- long nsQuota = 30;
- long ssQuota = 1024;
- String path = "/updatequota";
- final FileSystem nnFs1 = nnContext1.getFileSystem();
-
- // Add one mount table:
- // /updatequota --> ns0---/testdir10
- nnFs1.mkdirs(new Path("/testdir10"));
- MountTable mountTable = MountTable.newInstance(path,
- Collections.singletonMap("ns0", "/testdir10"));
- mountTable.setQuota(new RouterQuotaUsage.Builder().quota(nsQuota)
- .spaceQuota(ssQuota).build());
- addMountTable(mountTable);
-
- // Call periodicInvoke to ensure quota updated in quota manager
- // and state store.
- RouterQuotaUpdateService updateService = routerContext.getRouter()
- .getQuotaCacheUpdateService();
- updateService.periodicInvoke();
-
- // verify initial quota value
- List<MountTable> results = getMountTable(path);
- MountTable updatedMountTable = !results.isEmpty() ? results.get(0) : null;
- RouterQuotaUsage quota = updatedMountTable.getQuota();
- assertEquals(nsQuota, quota.getQuota());
- assertEquals(ssQuota, quota.getSpaceQuota());
- assertEquals(1, quota.getFileAndDirectoryCount());
- assertEquals(0, quota.getSpaceConsumed());
-
- // mkdir and write a new file
- final FileSystem routerFs = routerContext.getFileSystem();
- routerFs.mkdirs(new Path(path + "/" + UUID.randomUUID()));
- DFSClient routerClient = routerContext.getClient();
- routerClient.create(path + "/file", true).close();
- appendData(path + "/file", routerClient, BLOCK_SIZE);
-
- updateService.periodicInvoke();
- results = getMountTable(path);
- updatedMountTable = !results.isEmpty() ? results.get(0) : null;
- quota = updatedMountTable.getQuota();
-
- // verify if quota has been updated in state store
- assertEquals(nsQuota, quota.getQuota());
- assertEquals(ssQuota, quota.getSpaceQuota());
- assertEquals(3, quota.getFileAndDirectoryCount());
- assertEquals(BLOCK_SIZE, quota.getSpaceConsumed());
- }
-
- /**
- * Get the mount table entries of specified path through the admin API.
- * @param path Mount table entry to get.
- * @return If it was successfully got.
- * @throws IOException Problems getting entries.
- */
- private List<MountTable> getMountTable(String path) throws IOException {
- // Reload the Router cache
- resolver.loadCache(true);
- RouterClient client = routerContext.getAdminClient();
- MountTableManager mountTableManager = client.getMountTableManager();
- GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest
- .newInstance(path);
- GetMountTableEntriesResponse removeResponse = mountTableManager
- .getMountTableEntries(getRequest);
-
- return removeResponse.getEntries();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuotaManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuotaManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuotaManager.java
deleted file mode 100644
index ce3ee17..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuotaManager.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-import java.util.Set;
-
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Tests for class {@link RouterQuotaManager}.
- */
-public class TestRouterQuotaManager {
- private static RouterQuotaManager manager;
-
- @Before
- public void setup() {
- manager = new RouterQuotaManager();
- }
-
- @After
- public void cleanup() {
- manager.clear();
- }
-
- @Test
- public void testGetChildrenPaths() {
- RouterQuotaUsage quotaUsage = new RouterQuotaUsage.Builder().build();
- manager.put("/path1", quotaUsage);
- manager.put("/path2", quotaUsage);
- manager.put("/path1/subdir", quotaUsage);
- manager.put("/path1/subdir/subdir", quotaUsage);
-
- Set<String> childrenPaths = manager.getPaths("/path1");
- assertEquals(3, childrenPaths.size());
- assertTrue(childrenPaths.contains("/path1/subdir")
- && childrenPaths.contains("/path1/subdir/subdir")
- && childrenPaths.contains("/path1"));
-
- // test for corner case
- manager.put("/path3", quotaUsage);
- manager.put("/path3/subdir", quotaUsage);
- manager.put("/path3-subdir", quotaUsage);
-
- childrenPaths = manager.getPaths("/path3");
- assertEquals(2, childrenPaths.size());
- // path /path3-subdir should not be returned
- assertTrue(childrenPaths.contains("/path3")
- && childrenPaths.contains("/path3/subdir")
- && !childrenPaths.contains("/path3-subdir"));
- }
-
- @Test
- public void testGetQuotaUsage() {
- RouterQuotaUsage quotaGet;
-
- // test case1: get quota with an non-exist path
- quotaGet = manager.getQuotaUsage("/non-exist-path");
- assertNull(quotaGet);
-
- // test case2: get quota from an no-quota set path
- RouterQuotaUsage.Builder quota = new RouterQuotaUsage.Builder()
- .quota(HdfsConstants.QUOTA_DONT_SET)
- .spaceQuota(HdfsConstants.QUOTA_DONT_SET);
- manager.put("/noQuotaSet", quota.build());
- quotaGet = manager.getQuotaUsage("/noQuotaSet");
- // it should return null
- assertNull(quotaGet);
-
- // test case3: get quota from an quota-set path
- quota.quota(1);
- quota.spaceQuota(HdfsConstants.QUOTA_DONT_SET);
- manager.put("/hasQuotaSet", quota.build());
- quotaGet = manager.getQuotaUsage("/hasQuotaSet");
- assertEquals(1, quotaGet.getQuota());
- assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaGet.getSpaceQuota());
-
- // test case4: get quota with an non-exist child path
- quotaGet = manager.getQuotaUsage("/hasQuotaSet/file");
- // it will return the nearest ancestor which quota was set
- assertEquals(1, quotaGet.getQuota());
- assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaGet.getSpaceQuota());
-
- // test case5: get quota with an child path which its parent
- // wasn't quota set
- quota.quota(HdfsConstants.QUOTA_DONT_SET);
- quota.spaceQuota(HdfsConstants.QUOTA_DONT_SET);
- manager.put("/hasQuotaSet/noQuotaSet", quota.build());
- // here should returns the quota of path /hasQuotaSet
- // (the nearest ancestor which quota was set)
- quotaGet = manager.getQuotaUsage("/hasQuotaSet/noQuotaSet/file");
- assertEquals(1, quotaGet.getQuota());
- assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaGet.getSpaceQuota());
-
- // test case6: get quota with an child path which its parent was quota set
- quota.quota(2);
- quota.spaceQuota(HdfsConstants.QUOTA_DONT_SET);
- manager.put("/hasQuotaSet/hasQuotaSet", quota.build());
- // here should return the quota of path /hasQuotaSet/hasQuotaSet
- quotaGet = manager.getQuotaUsage("/hasQuotaSet/hasQuotaSet/file");
- assertEquals(2, quotaGet.getQuota());
- assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaGet.getSpaceQuota());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java
deleted file mode 100644
index 61e7657..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.DFSClient;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.protocol.ClientProtocol;
-import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
-import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
-import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
-import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
-import org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCMetrics;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
-import org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Test retry behavior of the Router RPC Client.
- */
-public class TestRouterRPCClientRetries {
-
- private static StateStoreDFSCluster cluster;
- private static NamenodeContext nnContext1;
- private static RouterContext routerContext;
- private static MembershipNamenodeResolver resolver;
- private static ClientProtocol routerProtocol;
-
- @Before
- public void setUp() throws Exception {
- // Build and start a federated cluster
- cluster = new StateStoreDFSCluster(false, 2);
- Configuration routerConf = new RouterConfigBuilder()
- .stateStore()
- .admin()
- .rpc()
- .build();
-
- // reduce IPC client connection retry times and interval time
- Configuration clientConf = new Configuration(false);
- clientConf.setInt(
- CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 1);
- clientConf.setInt(
- CommonConfigurationKeys.IPC_CLIENT_CONNECT_RETRY_INTERVAL_KEY, 100);
-
- cluster.addRouterOverrides(routerConf);
- // override some settings for the client
- cluster.startCluster(clientConf);
- cluster.startRouters();
- cluster.waitClusterUp();
-
- nnContext1 = cluster.getNamenode(cluster.getNameservices().get(0), null);
- routerContext = cluster.getRandomRouter();
- resolver = (MembershipNamenodeResolver) routerContext.getRouter()
- .getNamenodeResolver();
- routerProtocol = routerContext.getClient().getNamenode();
- }
-
- @After
- public void tearDown() {
- if (cluster != null) {
- cluster.stopRouter(routerContext);
- cluster.shutdown();
- cluster = null;
- }
- }
-
- @Test
- public void testRetryWhenAllNameServiceDown() throws Exception {
- // shutdown the dfs cluster
- MiniDFSCluster dfsCluster = cluster.getCluster();
- dfsCluster.shutdown();
-
- // register an invalid namenode report
- registerInvalidNameReport();
-
- // Create a directory via the router
- String dirPath = "/testRetryWhenClusterisDown";
- FsPermission permission = new FsPermission("705");
- try {
- routerProtocol.mkdirs(dirPath, permission, false);
- fail("Should have thrown RemoteException error.");
- } catch (RemoteException e) {
- String ns0 = cluster.getNameservices().get(0);
- GenericTestUtils.assertExceptionContains(
- "No namenode available under nameservice " + ns0, e);
- }
-
- // Verify the retry times, it should only retry one time.
- FederationRPCMetrics rpcMetrics = routerContext.getRouter()
- .getRpcServer().getRPCMetrics();
- assertEquals(1, rpcMetrics.getProxyOpRetries());
- }
-
- @Test
- public void testRetryWhenOneNameServiceDown() throws Exception {
- // shutdown the dfs cluster
- MiniDFSCluster dfsCluster = cluster.getCluster();
- dfsCluster.shutdownNameNode(0);
-
- // register an invalid namenode report
- registerInvalidNameReport();
-
- DFSClient client = nnContext1.getClient();
- // Renew lease for the DFS client, it will succeed.
- routerProtocol.renewLease(client.getClientName());
-
- // Verify the retry times, it will retry one time for ns0.
- FederationRPCMetrics rpcMetrics = routerContext.getRouter()
- .getRpcServer().getRPCMetrics();
- assertEquals(1, rpcMetrics.getProxyOpRetries());
- }
-
- /**
- * Register an invalid namenode report.
- * @throws IOException
- */
- private void registerInvalidNameReport() throws IOException {
- String ns0 = cluster.getNameservices().get(0);
- List<? extends FederationNamenodeContext> origin = resolver
- .getNamenodesForNameserviceId(ns0);
- FederationNamenodeContext nnInfo = origin.get(0);
- NamenodeStatusReport report = new NamenodeStatusReport(ns0,
- nnInfo.getNamenodeId(), nnInfo.getRpcAddress(),
- nnInfo.getServiceAddress(), nnInfo.getLifelineAddress(),
- nnInfo.getWebAddress());
- report.setRegistrationValid(false);
- assertTrue(resolver.registerNamenode(report));
- resolver.loadCache(true);
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
new file mode 100644
index 0000000..6111c6b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
@@ -0,0 +1,477 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayOutputStream;
+
+import java.io.PrintStream;
+import java.net.InetSocketAddress;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
+import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.impl.MountTableStoreImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.hdfs.tools.federation.RouterAdmin;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.google.common.base.Supplier;
+
+/**
+ * Tests Router admin commands.
+ */
+public class TestRouterAdminCLI {
+ private static StateStoreDFSCluster cluster;
+ private static RouterContext routerContext;
+ private static StateStoreService stateStore;
+
+ private static RouterAdmin admin;
+ private static RouterClient client;
+
+ private static final String TEST_USER = "test-user";
+
+ private final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ private static final PrintStream OLD_OUT = System.out;
+
+ @BeforeClass
+ public static void globalSetUp() throws Exception {
+ cluster = new StateStoreDFSCluster(false, 1);
+ // Build and start a router with State Store + admin + RPC
+ Configuration conf = new RouterConfigBuilder()
+ .stateStore()
+ .admin()
+ .rpc()
+ .build();
+ cluster.addRouterOverrides(conf);
+
+ // Start routers
+ cluster.startRouters();
+
+ routerContext = cluster.getRandomRouter();
+ Router router = routerContext.getRouter();
+ stateStore = router.getStateStore();
+
+ Configuration routerConf = new Configuration();
+ InetSocketAddress routerSocket = router.getAdminServerAddress();
+ routerConf.setSocketAddr(RBFConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_KEY,
+ routerSocket);
+ admin = new RouterAdmin(routerConf);
+ client = routerContext.getAdminClient();
+ }
+
+ @AfterClass
+ public static void tearDownCluster() {
+ cluster.stopRouter(routerContext);
+ cluster.shutdown();
+ cluster = null;
+ }
+
+ @After
+ public void tearDown() {
+ // set back system out
+ System.setOut(OLD_OUT);
+ }
+
+ @Test
+ public void testAddMountTable() throws Exception {
+ String nsId = "ns0";
+ String src = "/test-addmounttable";
+ String dest = "/addmounttable";
+ String[] argv = new String[] {"-add", src, nsId, dest};
+ assertEquals(0, ToolRunner.run(admin, argv));
+
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest
+ .newInstance(src);
+ GetMountTableEntriesResponse getResponse = client.getMountTableManager()
+ .getMountTableEntries(getRequest);
+ MountTable mountTable = getResponse.getEntries().get(0);
+
+ List<RemoteLocation> destinations = mountTable.getDestinations();
+ assertEquals(1, destinations.size());
+
+ assertEquals(src, mountTable.getSourcePath());
+ assertEquals(nsId, destinations.get(0).getNameserviceId());
+ assertEquals(dest, destinations.get(0).getDest());
+ assertFalse(mountTable.isReadOnly());
+
+ // test mount table update behavior
+ dest = dest + "-new";
+ argv = new String[] {"-add", src, nsId, dest, "-readonly"};
+ assertEquals(0, ToolRunner.run(admin, argv));
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+
+ getResponse = client.getMountTableManager()
+ .getMountTableEntries(getRequest);
+ mountTable = getResponse.getEntries().get(0);
+ assertEquals(2, mountTable.getDestinations().size());
+ assertEquals(nsId, mountTable.getDestinations().get(1).getNameserviceId());
+ assertEquals(dest, mountTable.getDestinations().get(1).getDest());
+ assertTrue(mountTable.isReadOnly());
+ }
+
+ @Test
+ public void testAddOrderMountTable() throws Exception {
+ testAddOrderMountTable(DestinationOrder.HASH);
+ testAddOrderMountTable(DestinationOrder.LOCAL);
+ testAddOrderMountTable(DestinationOrder.RANDOM);
+ testAddOrderMountTable(DestinationOrder.HASH_ALL);
+ }
+
+ private void testAddOrderMountTable(DestinationOrder order)
+ throws Exception {
+ final String mnt = "/" + order;
+ final String nsId = "ns0,ns1";
+ final String dest = "/";
+ String[] argv = new String[] {
+ "-add", mnt, nsId, dest, "-order", order.toString()};
+ assertEquals(0, ToolRunner.run(admin, argv));
+
+ // Check the state in the State Store
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ MountTableManager mountTable = client.getMountTableManager();
+ GetMountTableEntriesRequest request =
+ GetMountTableEntriesRequest.newInstance(mnt);
+ GetMountTableEntriesResponse response =
+ mountTable.getMountTableEntries(request);
+ List<MountTable> entries = response.getEntries();
+ assertEquals(1, entries.size());
+ assertEquals(2, entries.get(0).getDestinations().size());
+ assertEquals(order, response.getEntries().get(0).getDestOrder());
+ }
+
+ @Test
+ public void testListMountTable() throws Exception {
+ String nsId = "ns0";
+ String src = "/test-lsmounttable";
+ String dest = "/lsmounttable";
+ String[] argv = new String[] {"-add", src, nsId, dest};
+ assertEquals(0, ToolRunner.run(admin, argv));
+
+ // re-set system out for testing
+ System.setOut(new PrintStream(out));
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ argv = new String[] {"-ls", src};
+ assertEquals(0, ToolRunner.run(admin, argv));
+ assertTrue(out.toString().contains(src));
+
+ out.reset();
+ GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest
+ .newInstance("/");
+ GetMountTableEntriesResponse getResponse = client.getMountTableManager()
+ .getMountTableEntries(getRequest);
+
+ // Test ls command without input path, it will list
+ // mount table under root path.
+ argv = new String[] {"-ls"};
+ assertEquals(0, ToolRunner.run(admin, argv));
+ assertTrue(out.toString().contains(src));
+ String outStr = out.toString();
+ // verify if all the mount table are listed
+ for(MountTable entry: getResponse.getEntries()) {
+ assertTrue(outStr.contains(entry.getSourcePath()));
+ }
+ }
+
+ @Test
+ public void testRemoveMountTable() throws Exception {
+ String nsId = "ns0";
+ String src = "/test-rmmounttable";
+ String dest = "/rmmounttable";
+ String[] argv = new String[] {"-add", src, nsId, dest};
+ assertEquals(0, ToolRunner.run(admin, argv));
+
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest
+ .newInstance(src);
+ GetMountTableEntriesResponse getResponse = client.getMountTableManager()
+ .getMountTableEntries(getRequest);
+ // ensure mount table added successfully
+ MountTable mountTable = getResponse.getEntries().get(0);
+ assertEquals(src, mountTable.getSourcePath());
+
+ argv = new String[] {"-rm", src};
+ assertEquals(0, ToolRunner.run(admin, argv));
+
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ getResponse = client.getMountTableManager()
+ .getMountTableEntries(getRequest);
+ assertEquals(0, getResponse.getEntries().size());
+
+ // remove an invalid mount table
+ String invalidPath = "/invalid";
+ System.setOut(new PrintStream(out));
+ argv = new String[] {"-rm", invalidPath};
+ assertEquals(0, ToolRunner.run(admin, argv));
+ assertTrue(out.toString().contains(
+ "Cannot remove mount point " + invalidPath));
+ }
+
+ @Test
+ public void testMountTableDefaultACL() throws Exception {
+ String[] argv = new String[] {"-add", "/testpath0", "ns0", "/testdir0"};
+ assertEquals(0, ToolRunner.run(admin, argv));
+
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest
+ .newInstance("/testpath0");
+ GetMountTableEntriesResponse getResponse = client.getMountTableManager()
+ .getMountTableEntries(getRequest);
+ MountTable mountTable = getResponse.getEntries().get(0);
+
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ String group = ugi.getGroups().isEmpty() ? ugi.getShortUserName()
+ : ugi.getPrimaryGroupName();
+ assertEquals(ugi.getShortUserName(), mountTable.getOwnerName());
+ assertEquals(group, mountTable.getGroupName());
+ assertEquals((short) 0755, mountTable.getMode().toShort());
+ }
+
+ @Test
+ public void testMountTablePermissions() throws Exception {
+ // re-set system out for testing
+ System.setOut(new PrintStream(out));
+ // use superuser to add new mount table with only read permission
+ String[] argv = new String[] {"-add", "/testpath2-1", "ns0", "/testdir2-1",
+ "-owner", TEST_USER, "-group", TEST_USER, "-mode", "0455"};
+ assertEquals(0, ToolRunner.run(admin, argv));
+
+ String superUser = UserGroupInformation.
+ getCurrentUser().getShortUserName();
+ // use normal user as current user to test
+ UserGroupInformation remoteUser = UserGroupInformation
+ .createRemoteUser(TEST_USER);
+ UserGroupInformation.setLoginUser(remoteUser);
+
+ // verify read permission by executing other commands
+ verifyExecutionResult("/testpath2-1", true, -1, -1);
+
+ // add new mount table with only write permission
+ argv = new String[] {"-add", "/testpath2-2", "ns0", "/testdir2-2",
+ "-owner", TEST_USER, "-group", TEST_USER, "-mode", "0255"};
+ assertEquals(0, ToolRunner.run(admin, argv));
+ verifyExecutionResult("/testpath2-2", false, 0, 0);
+
+ // set mount table entry with read and write permission
+ argv = new String[] {"-add", "/testpath2-3", "ns0", "/testdir2-3",
+ "-owner", TEST_USER, "-group", TEST_USER, "-mode", "0755"};
+ assertEquals(0, ToolRunner.run(admin, argv));
+ verifyExecutionResult("/testpath2-3", true, 0, 0);
+
+ // set back login user
+ remoteUser = UserGroupInformation.createRemoteUser(superUser);
+ UserGroupInformation.setLoginUser(remoteUser);
+ }
+
+ /**
+ * Verify router admin commands execution result.
+ *
+ * @param mount
+ * target mount table
+ * @param canRead
+ * whether can list mount tables under specified mount
+ * @param addCommandCode
+ * expected return code of add command executed for specified mount
+ * @param rmCommandCode
+ * expected return code of rm command executed for specified mount
+ * @throws Exception
+ */
+ private void verifyExecutionResult(String mount, boolean canRead,
+ int addCommandCode, int rmCommandCode) throws Exception {
+ String[] argv = null;
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+
+ out.reset();
+ // execute ls command
+ argv = new String[] {"-ls", mount};
+ assertEquals(0, ToolRunner.run(admin, argv));
+ assertEquals(canRead, out.toString().contains(mount));
+
+ // execute add/update command
+ argv = new String[] {"-add", mount, "ns0", mount + "newdir"};
+ assertEquals(addCommandCode, ToolRunner.run(admin, argv));
+
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ // execute remove command
+ argv = new String[] {"-rm", mount};
+ assertEquals(rmCommandCode, ToolRunner.run(admin, argv));
+ }
+
+ @Test
+ public void testSetAndClearQuota() throws Exception {
+ String nsId = "ns0";
+ String src = "/test-QuotaMounttable";
+ String dest = "/QuotaMounttable";
+ String[] argv = new String[] {"-add", src, nsId, dest};
+ assertEquals(0, ToolRunner.run(admin, argv));
+
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest
+ .newInstance(src);
+ GetMountTableEntriesResponse getResponse = client.getMountTableManager()
+ .getMountTableEntries(getRequest);
+ MountTable mountTable = getResponse.getEntries().get(0);
+ RouterQuotaUsage quotaUsage = mountTable.getQuota();
+
+ // verify the default quota set
+ assertEquals(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT,
+ quotaUsage.getFileAndDirectoryCount());
+ assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaUsage.getQuota());
+ assertEquals(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT,
+ quotaUsage.getSpaceConsumed());
+ assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaUsage.getSpaceQuota());
+
+ long nsQuota = 50;
+ long ssQuota = 100;
+ argv = new String[] {"-setQuota", src, "-nsQuota", String.valueOf(nsQuota),
+ "-ssQuota", String.valueOf(ssQuota)};
+ assertEquals(0, ToolRunner.run(admin, argv));
+
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ getResponse = client.getMountTableManager()
+ .getMountTableEntries(getRequest);
+ mountTable = getResponse.getEntries().get(0);
+ quotaUsage = mountTable.getQuota();
+
+ // verify if the quota is set
+ assertEquals(nsQuota, quotaUsage.getQuota());
+ assertEquals(ssQuota, quotaUsage.getSpaceQuota());
+
+ // use quota string for setting ss quota
+ String newSsQuota = "2m";
+ argv = new String[] {"-setQuota", src, "-ssQuota", newSsQuota};
+ assertEquals(0, ToolRunner.run(admin, argv));
+
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ getResponse = client.getMountTableManager()
+ .getMountTableEntries(getRequest);
+ mountTable = getResponse.getEntries().get(0);
+ quotaUsage = mountTable.getQuota();
+ // verify if ss quota is correctly set
+ assertEquals(2 * 1024 * 1024, quotaUsage.getSpaceQuota());
+
+ // test clrQuota command
+ argv = new String[] {"-clrQuota", src};
+ assertEquals(0, ToolRunner.run(admin, argv));
+
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ getResponse = client.getMountTableManager()
+ .getMountTableEntries(getRequest);
+ mountTable = getResponse.getEntries().get(0);
+ quotaUsage = mountTable.getQuota();
+
+ // verify if quota unset successfully
+ assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaUsage.getQuota());
+ assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaUsage.getSpaceQuota());
+ }
+
+ @Test
+ public void testManageSafeMode() throws Exception {
+ // ensure the Router become RUNNING state
+ waitState(RouterServiceState.RUNNING);
+ assertFalse(routerContext.getRouter().getRpcServer().isInSafeMode());
+ assertEquals(0, ToolRunner.run(admin,
+ new String[] {"-safemode", "enter"}));
+ // verify state
+ assertEquals(RouterServiceState.SAFEMODE,
+ routerContext.getRouter().getRouterState());
+ assertTrue(routerContext.getRouter().getRpcServer().isInSafeMode());
+
+ System.setOut(new PrintStream(out));
+ assertEquals(0, ToolRunner.run(admin,
+ new String[] {"-safemode", "get"}));
+ assertTrue(out.toString().contains("true"));
+
+ assertEquals(0, ToolRunner.run(admin,
+ new String[] {"-safemode", "leave"}));
+ // verify state
+ assertEquals(RouterServiceState.RUNNING,
+ routerContext.getRouter().getRouterState());
+ assertFalse(routerContext.getRouter().getRpcServer().isInSafeMode());
+
+ out.reset();
+ assertEquals(0, ToolRunner.run(admin,
+ new String[] {"-safemode", "get"}));
+ assertTrue(out.toString().contains("false"));
+ }
+
+ @Test
+ public void testCreateInvalidEntry() throws Exception {
+ String[] argv = new String[] {
+ "-add", "test-createInvalidEntry", "ns0", "/createInvalidEntry"};
+ assertEquals(-1, ToolRunner.run(admin, argv));
+
+ argv = new String[] {
+ "-add", "/test-createInvalidEntry", "ns0", "createInvalidEntry"};
+ assertEquals(-1, ToolRunner.run(admin, argv));
+
+ argv = new String[] {
+ "-add", null, "ns0", "/createInvalidEntry"};
+ assertEquals(-1, ToolRunner.run(admin, argv));
+
+ argv = new String[] {
+ "-add", "/test-createInvalidEntry", "ns0", null};
+ assertEquals(-1, ToolRunner.run(admin, argv));
+
+ argv = new String[] {
+ "-add", "", "ns0", "/createInvalidEntry"};
+ assertEquals(-1, ToolRunner.run(admin, argv));
+
+ argv = new String[] {
+ "-add", "/test-createInvalidEntry", null, "/createInvalidEntry"};
+ assertEquals(-1, ToolRunner.run(admin, argv));
+
+ argv = new String[] {
+ "-add", "/test-createInvalidEntry", "", "/createInvalidEntry"};
+ assertEquals(-1, ToolRunner.run(admin, argv));
+ }
+
+ /**
+ * Wait for the Router transforming to expected state.
+ * @param expectedState Expected Router state.
+ * @throws Exception
+ */
+ private void waitState(final RouterServiceState expectedState)
+ throws Exception {
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return expectedState == routerContext.getRouter().getRouterState();
+ }
+ }, 1000, 30000);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterHeartbeatService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterHeartbeatService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterHeartbeatService.java
new file mode 100644
index 0000000..80f2327
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterHeartbeatService.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.RetryNTimes;
+import org.apache.curator.test.TestingServer;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.RouterStore;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+import org.apache.hadoop.hdfs.server.federation.store.records.StateStoreVersion;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.waitStateStore;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test cases for router heartbeat service.
+ */
+public class TestRouterHeartbeatService {
+ private Router router;
+ private final String routerId = "router1";
+ private TestingServer testingServer;
+ private CuratorFramework curatorFramework;
+
+ @Before
+ public void setup() throws Exception {
+ router = new Router();
+ router.setRouterId(routerId);
+ Configuration conf = new Configuration();
+ conf.setInt(RBFConfigKeys.DFS_ROUTER_CACHE_TIME_TO_LIVE_MS, 1);
+ Configuration routerConfig =
+ new RouterConfigBuilder(conf).stateStore().build();
+ routerConfig.setLong(RBFConfigKeys.FEDERATION_STORE_CONNECTION_TEST_MS,
+ TimeUnit.HOURS.toMillis(1));
+ routerConfig.setClass(RBFConfigKeys.FEDERATION_STORE_DRIVER_CLASS,
+ StateStoreZooKeeperImpl.class, StateStoreDriver.class);
+
+ testingServer = new TestingServer();
+ String connectStr = testingServer.getConnectString();
+ curatorFramework = CuratorFrameworkFactory.builder()
+ .connectString(connectStr)
+ .retryPolicy(new RetryNTimes(100, 100))
+ .build();
+ curatorFramework.start();
+ routerConfig.set(CommonConfigurationKeys.ZK_ADDRESS, connectStr);
+ router.init(routerConfig);
+ router.start();
+
+
+ waitStateStore(router.getStateStore(), TimeUnit.SECONDS.toMicros(10));
+ }
+
+ @Test
+ public void testStateStoreUnavailable() throws IOException {
+ curatorFramework.close();
+ testingServer.stop();
+ router.getStateStore().stop();
+ // The driver is not ready
+ assertFalse(router.getStateStore().isDriverReady());
+
+ // Do a heartbeat, and no exception thrown out
+ RouterHeartbeatService heartbeatService =
+ new RouterHeartbeatService(router);
+ heartbeatService.updateStateStore();
+ }
+
+ @Test
+ public void testStateStoreAvailable() throws Exception {
+ // The driver is ready
+ StateStoreService stateStore = router.getStateStore();
+ assertTrue(router.getStateStore().isDriverReady());
+ RouterStore routerStore = router.getRouterStateManager();
+
+ // No record about this router
+ stateStore.refreshCaches(true);
+ GetRouterRegistrationRequest request =
+ GetRouterRegistrationRequest.newInstance(routerId);
+ GetRouterRegistrationResponse response =
+ router.getRouterStateManager().getRouterRegistration(request);
+ RouterState routerState = response.getRouter();
+ String id = routerState.getRouterId();
+ StateStoreVersion version = routerState.getStateStoreVersion();
+ assertNull(id);
+ assertNull(version);
+
+ // Do a heartbeat
+ RouterHeartbeatService heartbeatService =
+ new RouterHeartbeatService(router);
+ heartbeatService.updateStateStore();
+
+ // We should have a record
+ stateStore.refreshCaches(true);
+ request = GetRouterRegistrationRequest.newInstance(routerId);
+ response = routerStore.getRouterRegistration(request);
+ routerState = response.getRouter();
+ id = routerState.getRouterId();
+ version = routerState.getStateStoreVersion();
+ assertNotNull(id);
+ assertNotNull(version);
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ if (curatorFramework != null) {
+ curatorFramework.close();
+ }
+ if (testingServer != null) {
+ testingServer.stop();
+ }
+ if (router != null) {
+ router.shutDown();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
new file mode 100644
index 0000000..8702b3c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.Collections;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
+import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test a router end-to-end including the MountTable.
+ */
+public class TestRouterMountTable {
+
+ private static StateStoreDFSCluster cluster;
+ private static NamenodeContext nnContext;
+ private static RouterContext routerContext;
+ private static MountTableResolver mountTable;
+
+ @BeforeClass
+ public static void globalSetUp() throws Exception {
+
+ // Build and start a federated cluster
+ cluster = new StateStoreDFSCluster(false, 1);
+ Configuration conf = new RouterConfigBuilder()
+ .stateStore()
+ .admin()
+ .rpc()
+ .build();
+ cluster.addRouterOverrides(conf);
+ cluster.startCluster();
+ cluster.startRouters();
+ cluster.waitClusterUp();
+
+ // Get the end points
+ nnContext = cluster.getRandomNamenode();
+ routerContext = cluster.getRandomRouter();
+ Router router = routerContext.getRouter();
+ mountTable = (MountTableResolver) router.getSubclusterResolver();
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ if (cluster != null) {
+ cluster.stopRouter(routerContext);
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+
+ @Test
+ public void testReadOnly() throws Exception {
+
+ // Add a read only entry
+ MountTable readOnlyEntry = MountTable.newInstance(
+ "/readonly", Collections.singletonMap("ns0", "/testdir"));
+ readOnlyEntry.setReadOnly(true);
+ assertTrue(addMountTable(readOnlyEntry));
+
+ // Add a regular entry
+ MountTable regularEntry = MountTable.newInstance(
+ "/regular", Collections.singletonMap("ns0", "/testdir"));
+ assertTrue(addMountTable(regularEntry));
+
+ // Create a folder which should show in all locations
+ final FileSystem nnFs = nnContext.getFileSystem();
+ final FileSystem routerFs = routerContext.getFileSystem();
+ assertTrue(routerFs.mkdirs(new Path("/regular/newdir")));
+
+ FileStatus dirStatusNn =
+ nnFs.getFileStatus(new Path("/testdir/newdir"));
+ assertTrue(dirStatusNn.isDirectory());
+ FileStatus dirStatusRegular =
+ routerFs.getFileStatus(new Path("/regular/newdir"));
+ assertTrue(dirStatusRegular.isDirectory());
+ FileStatus dirStatusReadOnly =
+ routerFs.getFileStatus(new Path("/readonly/newdir"));
+ assertTrue(dirStatusReadOnly.isDirectory());
+
+ // It should fail writing into a read only path
+ try {
+ routerFs.mkdirs(new Path("/readonly/newdirfail"));
+ fail("We should not be able to write into a read only mount point");
+ } catch (IOException ioe) {
+ String msg = ioe.getMessage();
+ assertTrue(msg.startsWith(
+ "/readonly/newdirfail is in a read only mount point"));
+ }
+ }
+
+ /**
+ * Add a mount table entry to the mount table through the admin API.
+ * @param entry Mount table entry to add.
+ * @return If it was succesfully added.
+ * @throws IOException Problems adding entries.
+ */
+ private boolean addMountTable(final MountTable entry) throws IOException {
+ RouterClient client = routerContext.getAdminClient();
+ MountTableManager mountTableManager = client.getMountTableManager();
+ AddMountTableEntryRequest addRequest =
+ AddMountTableEntryRequest.newInstance(entry);
+ AddMountTableEntryResponse addResponse =
+ mountTableManager.addMountTableEntry(addRequest);
+
+ // Reload the Router cache
+ mountTable.loadCache(true);
+
+ return addResponse.getStatus();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeMonitoring.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeMonitoring.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeMonitoring.java
new file mode 100644
index 0000000..eabc0fe
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeMonitoring.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_MONITOR_NAMENODE;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
+import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver;
+import org.apache.hadoop.util.Time;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test namenodes monitor behavior in the Router.
+ */
+public class TestRouterNamenodeMonitoring {
+
+ private static StateStoreDFSCluster cluster;
+ private static RouterContext routerContext;
+ private static MembershipNamenodeResolver resolver;
+
+ private String ns0;
+ private String ns1;
+ private long initializedTime;
+
+ @Before
+ public void setUp() throws Exception {
+ // Build and start a federated cluster with HA enabled
+ cluster = new StateStoreDFSCluster(true, 2);
+ // Enable heartbeat service and local heartbeat
+ Configuration routerConf = new RouterConfigBuilder()
+ .stateStore()
+ .admin()
+ .rpc()
+ .enableLocalHeartbeat(true)
+ .heartbeat()
+ .build();
+
+ // Specify local node (ns0.nn1) to monitor
+ StringBuilder sb = new StringBuilder();
+ ns0 = cluster.getNameservices().get(0);
+ NamenodeContext context = cluster.getNamenodes(ns0).get(1);
+ routerConf.set(DFS_NAMESERVICE_ID, ns0);
+ routerConf.set(DFS_HA_NAMENODE_ID_KEY, context.getNamenodeId());
+
+ // Specify namenodes (ns1.nn0,ns1.nn1) to monitor
+ sb = new StringBuilder();
+ ns1 = cluster.getNameservices().get(1);
+ for (NamenodeContext ctx : cluster.getNamenodes(ns1)) {
+ String suffix = ctx.getConfSuffix();
+ if (sb.length() != 0) {
+ sb.append(",");
+ }
+ sb.append(suffix);
+ }
+ // override with the namenodes: ns1.nn0,ns1.nn1
+ routerConf.set(DFS_ROUTER_MONITOR_NAMENODE, sb.toString());
+
+ cluster.addRouterOverrides(routerConf);
+ cluster.startCluster();
+ cluster.startRouters();
+ cluster.waitClusterUp();
+
+ routerContext = cluster.getRandomRouter();
+ resolver = (MembershipNamenodeResolver) routerContext.getRouter()
+ .getNamenodeResolver();
+ initializedTime = Time.now();
+ }
+
+ @After
+ public void tearDown() {
+ if (cluster != null) {
+ cluster.stopRouter(routerContext);
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+
+ @Test
+ public void testNamenodeMonitoring() throws Exception {
+ // Set nn0 to active for all nameservices
+ for (String ns : cluster.getNameservices()) {
+ cluster.switchToActive(ns, "nn0");
+ cluster.switchToStandby(ns, "nn1");
+ }
+
+ Collection<NamenodeHeartbeatService> heartbeatServices = routerContext
+ .getRouter().getNamenodeHearbeatServices();
+ // manually trigger the heartbeat
+ for (NamenodeHeartbeatService service : heartbeatServices) {
+ service.periodicInvoke();
+ }
+
+ resolver.loadCache(true);
+ List<? extends FederationNamenodeContext> namespaceInfo0 =
+ resolver.getNamenodesForNameserviceId(ns0);
+ List<? extends FederationNamenodeContext> namespaceInfo1 =
+ resolver.getNamenodesForNameserviceId(ns1);
+
+ // The modified date won't be updated in ns0.nn0 since it isn't
+ // monitored by the Router.
+ assertEquals("nn0", namespaceInfo0.get(1).getNamenodeId());
+ assertTrue(namespaceInfo0.get(1).getDateModified() < initializedTime);
+
+ // other namnodes should be updated as expected
+ assertEquals("nn1", namespaceInfo0.get(0).getNamenodeId());
+ assertTrue(namespaceInfo0.get(0).getDateModified() > initializedTime);
+
+ assertEquals("nn0", namespaceInfo1.get(0).getNamenodeId());
+ assertTrue(namespaceInfo1.get(0).getDateModified() > initializedTime);
+
+ assertEquals("nn1", namespaceInfo1.get(1).getNamenodeId());
+ assertTrue(namespaceInfo1.get(1).getDateModified() > initializedTime);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java
new file mode 100644
index 0000000..66a955e
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java
@@ -0,0 +1,451 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.QuotaUsage;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
+import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.base.Supplier;
+
+/**
+ * Tests quota behaviors in Router-based Federation.
+ */
+public class TestRouterQuota {
+ private static StateStoreDFSCluster cluster;
+ private static NamenodeContext nnContext1;
+ private static NamenodeContext nnContext2;
+ private static RouterContext routerContext;
+ private static MountTableResolver resolver;
+
+ private static final int BLOCK_SIZE = 512;
+
+ @Before
+ public void setUp() throws Exception {
+
+ // Build and start a federated cluster
+ cluster = new StateStoreDFSCluster(false, 2);
+ Configuration routerConf = new RouterConfigBuilder()
+ .stateStore()
+ .admin()
+ .quota()
+ .rpc()
+ .build();
+ routerConf.set(RBFConfigKeys.DFS_ROUTER_QUOTA_CACHE_UPATE_INTERVAL, "2s");
+
+ // override some hdfs settings that used in testing space quota
+ Configuration hdfsConf = new Configuration(false);
+ hdfsConf.setInt(HdfsClientConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
+ hdfsConf.setInt(HdfsClientConfigKeys.DFS_REPLICATION_KEY, 1);
+
+ cluster.addRouterOverrides(routerConf);
+ cluster.addNamenodeOverrides(hdfsConf);
+ cluster.startCluster();
+ cluster.startRouters();
+ cluster.waitClusterUp();
+
+ nnContext1 = cluster.getNamenode(cluster.getNameservices().get(0), null);
+ nnContext2 = cluster.getNamenode(cluster.getNameservices().get(1), null);
+ routerContext = cluster.getRandomRouter();
+ Router router = routerContext.getRouter();
+ resolver = (MountTableResolver) router.getSubclusterResolver();
+ }
+
+ @After
+ public void tearDown() {
+ if (cluster != null) {
+ cluster.stopRouter(routerContext);
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+
+ @Test
+ public void testNamespaceQuotaExceed() throws Exception {
+ long nsQuota = 3;
+ final FileSystem nnFs1 = nnContext1.getFileSystem();
+ final FileSystem nnFs2 = nnContext2.getFileSystem();
+
+ // Add two mount tables:
+ // /nsquota --> ns0---testdir1
+ // /nsquota/subdir --> ns1---testdir2
+ nnFs1.mkdirs(new Path("/testdir1"));
+ nnFs2.mkdirs(new Path("/testdir2"));
+ MountTable mountTable1 = MountTable.newInstance("/nsquota",
+ Collections.singletonMap("ns0", "/testdir1"));
+
+ mountTable1.setQuota(new RouterQuotaUsage.Builder().quota(nsQuota).build());
+ addMountTable(mountTable1);
+
+ MountTable mountTable2 = MountTable.newInstance("/nsquota/subdir",
+ Collections.singletonMap("ns1", "/testdir2"));
+ mountTable2.setQuota(new RouterQuotaUsage.Builder().quota(nsQuota).build());
+ addMountTable(mountTable2);
+
+ final FileSystem routerFs = routerContext.getFileSystem();
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+
+ @Override
+ public Boolean get() {
+ boolean isNsQuotaViolated = false;
+ try {
+ // create new directory to trigger NSQuotaExceededException
+ routerFs.mkdirs(new Path("/nsquota/" + UUID.randomUUID()));
+ routerFs.mkdirs(new Path("/nsquota/subdir/" + UUID.randomUUID()));
+ } catch (NSQuotaExceededException e) {
+ isNsQuotaViolated = true;
+ } catch (IOException ignored) {
+ }
+ return isNsQuotaViolated;
+ }
+ }, 5000, 60000);
+ // mkdir in real FileSystem should be okay
+ nnFs1.mkdirs(new Path("/testdir1/" + UUID.randomUUID()));
+ nnFs2.mkdirs(new Path("/testdir2/" + UUID.randomUUID()));
+ }
+
+ @Test
+ public void testStorageSpaceQuotaaExceed() throws Exception {
+ long ssQuota = 3071;
+ final FileSystem nnFs1 = nnContext1.getFileSystem();
+ final FileSystem nnFs2 = nnContext2.getFileSystem();
+
+ // Add two mount tables:
+ // /ssquota --> ns0---testdir3
+ // /ssquota/subdir --> ns1---testdir4
+ nnFs1.mkdirs(new Path("/testdir3"));
+ nnFs2.mkdirs(new Path("/testdir4"));
+ MountTable mountTable1 = MountTable.newInstance("/ssquota",
+ Collections.singletonMap("ns0", "/testdir3"));
+
+ mountTable1
+ .setQuota(new RouterQuotaUsage.Builder().spaceQuota(ssQuota).build());
+ addMountTable(mountTable1);
+
+ MountTable mountTable2 = MountTable.newInstance("/ssquota/subdir",
+ Collections.singletonMap("ns1", "/testdir4"));
+ mountTable2
+ .setQuota(new RouterQuotaUsage.Builder().spaceQuota(ssQuota).build());
+ addMountTable(mountTable2);
+
+ DFSClient routerClient = routerContext.getClient();
+ routerClient.create("/ssquota/file", true).close();
+ routerClient.create("/ssquota/subdir/file", true).close();
+
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+
+ @Override
+ public Boolean get() {
+ boolean isDsQuotaViolated = false;
+ try {
+ // append data to trigger NSQuotaExceededException
+ appendData("/ssquota/file", routerClient, BLOCK_SIZE);
+ appendData("/ssquota/subdir/file", routerClient, BLOCK_SIZE);
+ } catch (DSQuotaExceededException e) {
+ isDsQuotaViolated = true;
+ } catch (IOException ignored) {
+ }
+ return isDsQuotaViolated;
+ }
+ }, 5000, 60000);
+
+ // append data to destination path in real FileSystem should be okay
+ appendData("/testdir3/file", nnContext1.getClient(), BLOCK_SIZE);
+ appendData("/testdir4/file", nnContext2.getClient(), BLOCK_SIZE);
+ }
+
+ /**
+ * Add a mount table entry to the mount table through the admin API.
+ * @param entry Mount table entry to add.
+ * @return If it was successfully added.
+ * @throws IOException Problems adding entries.
+ */
+ private boolean addMountTable(final MountTable entry) throws IOException {
+ RouterClient client = routerContext.getAdminClient();
+ MountTableManager mountTableManager = client.getMountTableManager();
+ AddMountTableEntryRequest addRequest =
+ AddMountTableEntryRequest.newInstance(entry);
+ AddMountTableEntryResponse addResponse =
+ mountTableManager.addMountTableEntry(addRequest);
+
+ // Reload the Router cache
+ resolver.loadCache(true);
+
+ return addResponse.getStatus();
+ }
+
+ /**
+ * Append data in specified file.
+ * @param path Path of file.
+ * @param client DFS Client.
+ * @param dataLen The length of write data.
+ * @throws IOException
+ */
+ private void appendData(String path, DFSClient client, int dataLen)
+ throws IOException {
+ EnumSet<CreateFlag> createFlag = EnumSet.of(CreateFlag.APPEND);
+ HdfsDataOutputStream stream = client.append(path, 1024, createFlag, null,
+ null);
+ byte[] data = new byte[dataLen];
+ stream.write(data);
+ stream.close();
+ }
+
+ @Test
+ public void testSetQuota() throws Exception {
+ long nsQuota = 5;
+ long ssQuota = 100;
+ final FileSystem nnFs1 = nnContext1.getFileSystem();
+ final FileSystem nnFs2 = nnContext2.getFileSystem();
+
+ // Add two mount tables:
+ // /setquota --> ns0---testdir5
+ // /setquota/subdir --> ns1---testdir6
+ nnFs1.mkdirs(new Path("/testdir5"));
+ nnFs2.mkdirs(new Path("/testdir6"));
+ MountTable mountTable1 = MountTable.newInstance("/setquota",
+ Collections.singletonMap("ns0", "/testdir5"));
+ mountTable1
+ .setQuota(new RouterQuotaUsage.Builder().quota(nsQuota)
+ .spaceQuota(ssQuota).build());
+ addMountTable(mountTable1);
+
+ // don't set quota for subpath of mount table
+ MountTable mountTable2 = MountTable.newInstance("/setquota/subdir",
+ Collections.singletonMap("ns1", "/testdir6"));
+ addMountTable(mountTable2);
+
+ RouterQuotaUpdateService updateService = routerContext.getRouter()
+ .getQuotaCacheUpdateService();
+ // ensure setQuota RPC call was invoked
+ updateService.periodicInvoke();
+
+ ClientProtocol client1 = nnContext1.getClient().getNamenode();
+ ClientProtocol client2 = nnContext2.getClient().getNamenode();
+ final QuotaUsage quota1 = client1.getQuotaUsage("/testdir5");
+ final QuotaUsage quota2 = client2.getQuotaUsage("/testdir6");
+
+ assertEquals(nsQuota, quota1.getQuota());
+ assertEquals(ssQuota, quota1.getSpaceQuota());
+ assertEquals(nsQuota, quota2.getQuota());
+ assertEquals(ssQuota, quota2.getSpaceQuota());
+ }
+
+ @Test
+ public void testGetQuota() throws Exception {
+ long nsQuota = 10;
+ long ssQuota = 100;
+ final FileSystem nnFs1 = nnContext1.getFileSystem();
+ final FileSystem nnFs2 = nnContext2.getFileSystem();
+
+ // Add two mount tables:
+ // /getquota --> ns0---/testdir7
+ // /getquota/subdir1 --> ns0---/testdir7/subdir
+ // /getquota/subdir2 --> ns1---/testdir8
+ nnFs1.mkdirs(new Path("/testdir7"));
+ nnFs1.mkdirs(new Path("/testdir7/subdir"));
+ nnFs2.mkdirs(new Path("/testdir8"));
+ MountTable mountTable1 = MountTable.newInstance("/getquota",
+ Collections.singletonMap("ns0", "/testdir7"));
+ mountTable1
+ .setQuota(new RouterQuotaUsage.Builder().quota(nsQuota)
+ .spaceQuota(ssQuota).build());
+ addMountTable(mountTable1);
+
+ MountTable mountTable2 = MountTable.newInstance("/getquota/subdir1",
+ Collections.singletonMap("ns0", "/testdir7/subdir"));
+ addMountTable(mountTable2);
+
+ MountTable mountTable3 = MountTable.newInstance("/getquota/subdir2",
+ Collections.singletonMap("ns1", "/testdir8"));
+ addMountTable(mountTable3);
+
+ // use router client to create new files
+ DFSClient routerClient = routerContext.getClient();
+ routerClient.create("/getquota/file", true).close();
+ routerClient.create("/getquota/subdir1/file", true).close();
+ routerClient.create("/getquota/subdir2/file", true).close();
+
+ ClientProtocol clientProtocol = routerContext.getClient().getNamenode();
+ RouterQuotaUpdateService updateService = routerContext.getRouter()
+ .getQuotaCacheUpdateService();
+ updateService.periodicInvoke();
+ final QuotaUsage quota = clientProtocol.getQuotaUsage("/getquota");
+ // the quota should be aggregated
+ assertEquals(6, quota.getFileAndDirectoryCount());
+ }
+
+ @Test
+ public void testStaleQuotaRemoving() throws Exception {
+ long nsQuota = 20;
+ long ssQuota = 200;
+ String stalePath = "/stalequota";
+ final FileSystem nnFs1 = nnContext1.getFileSystem();
+
+ // Add one mount tables:
+ // /stalequota --> ns0---/testdir9
+ nnFs1.mkdirs(new Path("/testdir9"));
+ MountTable mountTable = MountTable.newInstance(stalePath,
+ Collections.singletonMap("ns0", "/testdir9"));
+ mountTable.setQuota(new RouterQuotaUsage.Builder().quota(nsQuota)
+ .spaceQuota(ssQuota).build());
+ addMountTable(mountTable);
+
+ // Call periodicInvoke to ensure quota for stalePath was
+ // loaded into quota manager.
+ RouterQuotaUpdateService updateService = routerContext.getRouter()
+ .getQuotaCacheUpdateService();
+ updateService.periodicInvoke();
+
+ // use quota manager to get its quota usage and do verification
+ RouterQuotaManager quotaManager = routerContext.getRouter()
+ .getQuotaManager();
+ RouterQuotaUsage quota = quotaManager.getQuotaUsage(stalePath);
+ assertEquals(nsQuota, quota.getQuota());
+ assertEquals(ssQuota, quota.getSpaceQuota());
+
+ // remove stale path entry
+ removeMountTable(stalePath);
+ updateService.periodicInvoke();
+ // the stale entry should be removed and we will get null
+ quota = quotaManager.getQuotaUsage(stalePath);
+ assertNull(quota);
+ }
+
+ /**
+ * Remove a mount table entry to the mount table through the admin API.
+ * @param entry Mount table entry to remove.
+ * @return If it was successfully removed.
+ * @throws IOException Problems removing entries.
+ */
+ private boolean removeMountTable(String path) throws IOException {
+ RouterClient client = routerContext.getAdminClient();
+ MountTableManager mountTableManager = client.getMountTableManager();
+ RemoveMountTableEntryRequest removeRequest = RemoveMountTableEntryRequest
+ .newInstance(path);
+ RemoveMountTableEntryResponse removeResponse = mountTableManager
+ .removeMountTableEntry(removeRequest);
+
+ // Reload the Router cache
+ resolver.loadCache(true);
+ return removeResponse.getStatus();
+ }
+
+ @Test
+ public void testQuotaUpdating() throws Exception {
+ long nsQuota = 30;
+ long ssQuota = 1024;
+ String path = "/updatequota";
+ final FileSystem nnFs1 = nnContext1.getFileSystem();
+
+ // Add one mount table:
+ // /updatequota --> ns0---/testdir10
+ nnFs1.mkdirs(new Path("/testdir10"));
+ MountTable mountTable = MountTable.newInstance(path,
+ Collections.singletonMap("ns0", "/testdir10"));
+ mountTable.setQuota(new RouterQuotaUsage.Builder().quota(nsQuota)
+ .spaceQuota(ssQuota).build());
+ addMountTable(mountTable);
+
+ // Call periodicInvoke to ensure quota updated in quota manager
+ // and state store.
+ RouterQuotaUpdateService updateService = routerContext.getRouter()
+ .getQuotaCacheUpdateService();
+ updateService.periodicInvoke();
+
+ // verify initial quota value
+ List<MountTable> results = getMountTable(path);
+ MountTable updatedMountTable = !results.isEmpty() ? results.get(0) : null;
+ RouterQuotaUsage quota = updatedMountTable.getQuota();
+ assertEquals(nsQuota, quota.getQuota());
+ assertEquals(ssQuota, quota.getSpaceQuota());
+ assertEquals(1, quota.getFileAndDirectoryCount());
+ assertEquals(0, quota.getSpaceConsumed());
+
+ // mkdir and write a new file
+ final FileSystem routerFs = routerContext.getFileSystem();
+ routerFs.mkdirs(new Path(path + "/" + UUID.randomUUID()));
+ DFSClient routerClient = routerContext.getClient();
+ routerClient.create(path + "/file", true).close();
+ appendData(path + "/file", routerClient, BLOCK_SIZE);
+
+ updateService.periodicInvoke();
+ results = getMountTable(path);
+ updatedMountTable = !results.isEmpty() ? results.get(0) : null;
+ quota = updatedMountTable.getQuota();
+
+ // verify if quota has been updated in state store
+ assertEquals(nsQuota, quota.getQuota());
+ assertEquals(ssQuota, quota.getSpaceQuota());
+ assertEquals(3, quota.getFileAndDirectoryCount());
+ assertEquals(BLOCK_SIZE, quota.getSpaceConsumed());
+ }
+
+ /**
+ * Get the mount table entries of specified path through the admin API.
+ * @param path Mount table entry to get.
+ * @return If it was successfully got.
+ * @throws IOException Problems getting entries.
+ */
+ private List<MountTable> getMountTable(String path) throws IOException {
+ // Reload the Router cache
+ resolver.loadCache(true);
+ RouterClient client = routerContext.getAdminClient();
+ MountTableManager mountTableManager = client.getMountTableManager();
+ GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest
+ .newInstance(path);
+ GetMountTableEntriesResponse removeResponse = mountTableManager
+ .getMountTableEntries(getRequest);
+
+ return removeResponse.getEntries();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuotaManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuotaManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuotaManager.java
new file mode 100644
index 0000000..ce3ee17
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuotaManager.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Set;
+
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Tests for class {@link RouterQuotaManager}.
+ */
+public class TestRouterQuotaManager {
+ private static RouterQuotaManager manager;
+
+ @Before
+ public void setup() {
+ manager = new RouterQuotaManager();
+ }
+
+ @After
+ public void cleanup() {
+ manager.clear();
+ }
+
+ @Test
+ public void testGetChildrenPaths() {
+ RouterQuotaUsage quotaUsage = new RouterQuotaUsage.Builder().build();
+ manager.put("/path1", quotaUsage);
+ manager.put("/path2", quotaUsage);
+ manager.put("/path1/subdir", quotaUsage);
+ manager.put("/path1/subdir/subdir", quotaUsage);
+
+ Set<String> childrenPaths = manager.getPaths("/path1");
+ assertEquals(3, childrenPaths.size());
+ assertTrue(childrenPaths.contains("/path1/subdir")
+ && childrenPaths.contains("/path1/subdir/subdir")
+ && childrenPaths.contains("/path1"));
+
+ // test for corner case
+ manager.put("/path3", quotaUsage);
+ manager.put("/path3/subdir", quotaUsage);
+ manager.put("/path3-subdir", quotaUsage);
+
+ childrenPaths = manager.getPaths("/path3");
+ assertEquals(2, childrenPaths.size());
+ // path /path3-subdir should not be returned
+ assertTrue(childrenPaths.contains("/path3")
+ && childrenPaths.contains("/path3/subdir")
+ && !childrenPaths.contains("/path3-subdir"));
+ }
+
+ @Test
+ public void testGetQuotaUsage() {
+ RouterQuotaUsage quotaGet;
+
+ // test case1: get quota with an non-exist path
+ quotaGet = manager.getQuotaUsage("/non-exist-path");
+ assertNull(quotaGet);
+
+ // test case2: get quota from an no-quota set path
+ RouterQuotaUsage.Builder quota = new RouterQuotaUsage.Builder()
+ .quota(HdfsConstants.QUOTA_DONT_SET)
+ .spaceQuota(HdfsConstants.QUOTA_DONT_SET);
+ manager.put("/noQuotaSet", quota.build());
+ quotaGet = manager.getQuotaUsage("/noQuotaSet");
+ // it should return null
+ assertNull(quotaGet);
+
+ // test case3: get quota from an quota-set path
+ quota.quota(1);
+ quota.spaceQuota(HdfsConstants.QUOTA_DONT_SET);
+ manager.put("/hasQuotaSet", quota.build());
+ quotaGet = manager.getQuotaUsage("/hasQuotaSet");
+ assertEquals(1, quotaGet.getQuota());
+ assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaGet.getSpaceQuota());
+
+ // test case4: get quota with an non-exist child path
+ quotaGet = manager.getQuotaUsage("/hasQuotaSet/file");
+ // it will return the nearest ancestor which quota was set
+ assertEquals(1, quotaGet.getQuota());
+ assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaGet.getSpaceQuota());
+
+ // test case5: get quota with an child path which its parent
+ // wasn't quota set
+ quota.quota(HdfsConstants.QUOTA_DONT_SET);
+ quota.spaceQuota(HdfsConstants.QUOTA_DONT_SET);
+ manager.put("/hasQuotaSet/noQuotaSet", quota.build());
+ // here should returns the quota of path /hasQuotaSet
+ // (the nearest ancestor which quota was set)
+ quotaGet = manager.getQuotaUsage("/hasQuotaSet/noQuotaSet/file");
+ assertEquals(1, quotaGet.getQuota());
+ assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaGet.getSpaceQuota());
+
+ // test case6: get quota with an child path which its parent was quota set
+ quota.quota(2);
+ quota.spaceQuota(HdfsConstants.QUOTA_DONT_SET);
+ manager.put("/hasQuotaSet/hasQuotaSet", quota.build());
+ // here should return the quota of path /hasQuotaSet/hasQuotaSet
+ quotaGet = manager.getQuotaUsage("/hasQuotaSet/hasQuotaSet/file");
+ assertEquals(2, quotaGet.getQuota());
+ assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaGet.getSpaceQuota());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java
new file mode 100644
index 0000000..61e7657
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java
@@ -0,0 +1,160 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
+import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCMetrics;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test retry behavior of the Router RPC Client.
+ */
+public class TestRouterRPCClientRetries {
+
+ private static StateStoreDFSCluster cluster;
+ private static NamenodeContext nnContext1;
+ private static RouterContext routerContext;
+ private static MembershipNamenodeResolver resolver;
+ private static ClientProtocol routerProtocol;
+
+ @Before
+ public void setUp() throws Exception {
+ // Build and start a federated cluster
+ cluster = new StateStoreDFSCluster(false, 2);
+ Configuration routerConf = new RouterConfigBuilder()
+ .stateStore()
+ .admin()
+ .rpc()
+ .build();
+
+ // reduce IPC client connection retry times and interval time
+ Configuration clientConf = new Configuration(false);
+ clientConf.setInt(
+ CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 1);
+ clientConf.setInt(
+ CommonConfigurationKeys.IPC_CLIENT_CONNECT_RETRY_INTERVAL_KEY, 100);
+
+ cluster.addRouterOverrides(routerConf);
+ // override some settings for the client
+ cluster.startCluster(clientConf);
+ cluster.startRouters();
+ cluster.waitClusterUp();
+
+ nnContext1 = cluster.getNamenode(cluster.getNameservices().get(0), null);
+ routerContext = cluster.getRandomRouter();
+ resolver = (MembershipNamenodeResolver) routerContext.getRouter()
+ .getNamenodeResolver();
+ routerProtocol = routerContext.getClient().getNamenode();
+ }
+
+ @After
+ public void tearDown() {
+ if (cluster != null) {
+ cluster.stopRouter(routerContext);
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+
+ @Test
+ public void testRetryWhenAllNameServiceDown() throws Exception {
+ // shutdown the dfs cluster
+ MiniDFSCluster dfsCluster = cluster.getCluster();
+ dfsCluster.shutdown();
+
+ // register an invalid namenode report
+ registerInvalidNameReport();
+
+ // Create a directory via the router
+ String dirPath = "/testRetryWhenClusterisDown";
+ FsPermission permission = new FsPermission("705");
+ try {
+ routerProtocol.mkdirs(dirPath, permission, false);
+ fail("Should have thrown RemoteException error.");
+ } catch (RemoteException e) {
+ String ns0 = cluster.getNameservices().get(0);
+ GenericTestUtils.assertExceptionContains(
+ "No namenode available under nameservice " + ns0, e);
+ }
+
+ // Verify the retry times, it should only retry one time.
+ FederationRPCMetrics rpcMetrics = routerContext.getRouter()
+ .getRpcServer().getRPCMetrics();
+ assertEquals(1, rpcMetrics.getProxyOpRetries());
+ }
+
+ @Test
+ public void testRetryWhenOneNameServiceDown() throws Exception {
+ // shutdown the dfs cluster
+ MiniDFSCluster dfsCluster = cluster.getCluster();
+ dfsCluster.shutdownNameNode(0);
+
+ // register an invalid namenode report
+ registerInvalidNameReport();
+
+ DFSClient client = nnContext1.getClient();
+ // Renew lease for the DFS client, it will succeed.
+ routerProtocol.renewLease(client.getClientName());
+
+ // Verify the retry times, it will retry one time for ns0.
+ FederationRPCMetrics rpcMetrics = routerContext.getRouter()
+ .getRpcServer().getRPCMetrics();
+ assertEquals(1, rpcMetrics.getProxyOpRetries());
+ }
+
+ /**
+ * Register an invalid namenode report.
+ * @throws IOException
+ */
+ private void registerInvalidNameReport() throws IOException {
+ String ns0 = cluster.getNameservices().get(0);
+ List<? extends FederationNamenodeContext> origin = resolver
+ .getNamenodesForNameserviceId(ns0);
+ FederationNamenodeContext nnInfo = origin.get(0);
+ NamenodeStatusReport report = new NamenodeStatusReport(ns0,
+ nnInfo.getNamenodeId(), nnInfo.getRpcAddress(),
+ nnInfo.getServiceAddress(), nnInfo.getLifelineAddress(),
+ nnInfo.getWebAddress());
+ report.setRegistrationValid(false);
+ assertTrue(resolver.registerNamenode(report));
+ resolver.loadCache(true);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreBaseImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreBaseImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreBaseImpl.java
deleted file mode 100644
index 1bd35f2..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreBaseImpl.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
-
-import static org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils.filterMultiple;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
-import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
-import org.apache.hadoop.hdfs.server.federation.store.records.Query;
-import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
-
-/**
- * Base implementation of a State Store driver. It contains default
- * implementations for the optional functions. These implementations use an
- * uncached read/write all algorithm for all changes. In most cases it is
- * recommended to override the optional functions.
- * <p>
- * Drivers may optionally override additional routines for performance
- * optimization, such as custom get/put/remove queries, depending on the
- * capabilities of the data store.
- * <p>
- */
-public abstract class StateStoreBaseImpl extends StateStoreDriver {
-
- @Override
- public <T extends BaseRecord> T get(
- Class<T> clazz, Query<T> query) throws IOException {
- List<T> records = getMultiple(clazz, query);
- if (records.size() > 1) {
- throw new IOException("Found more than one object in collection");
- } else if (records.size() == 1) {
- return records.get(0);
- } else {
- return null;
- }
- }
-
- @Override
- public <T extends BaseRecord> List<T> getMultiple(
- Class<T> clazz, Query<T> query) throws IOException {
- QueryResult<T> result = get(clazz);
- List<T> records = result.getRecords();
- List<T> ret = filterMultiple(query, records);
- if (ret == null) {
- throw new IOException("Cannot fetch records from the store");
- }
- return ret;
- }
-
- @Override
- public <T extends BaseRecord> boolean put(
- T record, boolean allowUpdate, boolean errorIfExists) throws IOException {
- List<T> singletonList = new ArrayList<>();
- singletonList.add(record);
- return putAll(singletonList, allowUpdate, errorIfExists);
- }
-
- @Override
- public <T extends BaseRecord> boolean remove(T record) throws IOException {
- final Query<T> query = new Query<T>(record);
- Class<? extends BaseRecord> clazz = record.getClass();
- @SuppressWarnings("unchecked")
- Class<T> recordClass = (Class<T>)StateStoreUtils.getRecordClass(clazz);
- return remove(recordClass, query) == 1;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
deleted file mode 100644
index 6638d1c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
+++ /dev/null
@@ -1,462 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
-
-import static org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils.filterMultiple;
-import static org.apache.hadoop.util.Time.monotonicNow;
-import static org.apache.hadoop.util.Time.now;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.concurrent.TimeUnit;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.hadoop.hdfs.server.federation.metrics.StateStoreMetrics;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreUnavailableException;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
-import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
-import org.apache.hadoop.hdfs.server.federation.store.records.Query;
-import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.annotations.VisibleForTesting;
-
-/**
- * {@link StateStoreDriver} implementation based on files. In this approach, we
- * use temporary files for the writes and renaming "atomically" to the final
- * value. Instead of writing to the final location, it will go to a temporary
- * one and then rename to the final destination.
- */
-public abstract class StateStoreFileBaseImpl
- extends StateStoreSerializableImpl {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(StateStoreFileBaseImpl.class);
-
- /** File extension for temporary files. */
- private static final String TMP_MARK = ".tmp";
- /** We remove temporary files older than 10 seconds. */
- private static final long OLD_TMP_RECORD_MS = TimeUnit.SECONDS.toMillis(10);
- /** File pattern for temporary records: file.XYZ.tmp. */
- private static final Pattern OLD_TMP_RECORD_PATTERN =
- Pattern.compile(".+\\.(\\d+)\\.tmp");
-
- /** If it is initialized. */
- private boolean initialized = false;
-
-
- /**
- * Get the reader of a record for the file system.
- *
- * @param path Path of the record to read.
- * @return Reader for the record.
- */
- protected abstract <T extends BaseRecord> BufferedReader getReader(
- String path);
-
- /**
- * Get the writer of a record for the file system.
- *
- * @param path Path of the record to write.
- * @return Writer for the record.
- */
- protected abstract <T extends BaseRecord> BufferedWriter getWriter(
- String path);
-
- /**
- * Check if a path exists.
- *
- * @param path Path to check.
- * @return If the path exists.
- */
- protected abstract boolean exists(String path);
-
- /**
- * Make a directory.
- *
- * @param path Path of the directory to create.
- * @return If the directory was created.
- */
- protected abstract boolean mkdir(String path);
-
- /**
- * Rename a file. This should be atomic.
- *
- * @param src Source name.
- * @param dst Destination name.
- * @return If the rename was successful.
- */
- protected abstract boolean rename(String src, String dst);
-
- /**
- * Remove a file.
- *
- * @param path Path for the file to remove
- * @return If the file was removed.
- */
- protected abstract boolean remove(String path);
-
- /**
- * Get the children for a path.
- *
- * @param path Path to check.
- * @return List of children.
- */
- protected abstract List<String> getChildren(String path);
-
- /**
- * Get root directory.
- *
- * @return Root directory.
- */
- protected abstract String getRootDir();
-
- /**
- * Set the driver as initialized.
- *
- * @param ini If the driver is initialized.
- */
- public void setInitialized(boolean ini) {
- this.initialized = ini;
- }
-
- @Override
- public boolean initDriver() {
- String rootDir = getRootDir();
- try {
- if (rootDir == null) {
- LOG.error("Invalid root directory, unable to initialize driver.");
- return false;
- }
-
- // Check root path
- if (!exists(rootDir)) {
- if (!mkdir(rootDir)) {
- LOG.error("Cannot create State Store root directory {}", rootDir);
- return false;
- }
- }
- } catch (Exception ex) {
- LOG.error(
- "Cannot initialize filesystem using root directory {}", rootDir, ex);
- return false;
- }
- setInitialized(true);
- return true;
- }
-
- @Override
- public <T extends BaseRecord> boolean initRecordStorage(
- String className, Class<T> recordClass) {
-
- String dataDirPath = getRootDir() + "/" + className;
- try {
- // Create data directories for files
- if (!exists(dataDirPath)) {
- LOG.info("{} data directory doesn't exist, creating it", dataDirPath);
- if (!mkdir(dataDirPath)) {
- LOG.error("Cannot create data directory {}", dataDirPath);
- return false;
- }
- }
- } catch (Exception ex) {
- LOG.error("Cannot create data directory {}", dataDirPath, ex);
- return false;
- }
- return true;
- }
-
- @Override
- public <T extends BaseRecord> QueryResult<T> get(Class<T> clazz)
- throws IOException {
- verifyDriverReady();
- long start = monotonicNow();
- StateStoreMetrics metrics = getMetrics();
- List<T> ret = new ArrayList<>();
- try {
- String path = getPathForClass(clazz);
- List<String> children = getChildren(path);
- for (String child : children) {
- String pathRecord = path + "/" + child;
- if (child.endsWith(TMP_MARK)) {
- LOG.debug("There is a temporary file {} in {}", child, path);
- if (isOldTempRecord(child)) {
- LOG.warn("Removing {} as it's an old temporary record", child);
- remove(pathRecord);
- }
- } else {
- T record = getRecord(pathRecord, clazz);
- ret.add(record);
- }
- }
- } catch (Exception e) {
- if (metrics != null) {
- metrics.addFailure(monotonicNow() - start);
- }
- String msg = "Cannot fetch records for " + clazz.getSimpleName();
- LOG.error(msg, e);
- throw new IOException(msg, e);
- }
-
- if (metrics != null) {
- metrics.addRead(monotonicNow() - start);
- }
- return new QueryResult<T>(ret, getTime());
- }
-
- /**
- * Check if a record is temporary and old.
- *
- * @param pathRecord Path for the record to check.
- * @return If the record is temporary and old.
- */
- @VisibleForTesting
- public static boolean isOldTempRecord(final String pathRecord) {
- if (!pathRecord.endsWith(TMP_MARK)) {
- return false;
- }
- // Extract temporary record creation time
- Matcher m = OLD_TMP_RECORD_PATTERN.matcher(pathRecord);
- if (m.find()) {
- long time = Long.parseLong(m.group(1));
- return now() - time > OLD_TMP_RECORD_MS;
- }
- return false;
- }
-
- /**
- * Read a record from a file.
- *
- * @param path Path to the file containing the record.
- * @param clazz Class of the record.
- * @return Record read from the file.
- * @throws IOException If the file cannot be read.
- */
- private <T extends BaseRecord> T getRecord(
- final String path, final Class<T> clazz) throws IOException {
- BufferedReader reader = getReader(path);
- try {
- String line;
- while ((line = reader.readLine()) != null) {
- if (!line.startsWith("#") && line.length() > 0) {
- try {
- T record = newRecord(line, clazz, false);
- return record;
- } catch (Exception ex) {
- LOG.error("Cannot parse line {} in file {}", line, path, ex);
- }
- }
- }
- } finally {
- if (reader != null) {
- reader.close();
- }
- }
- throw new IOException("Cannot read " + path + " for record " +
- clazz.getSimpleName());
- }
-
- /**
- * Get the path for a record class.
- * @param clazz Class of the record.
- * @return Path for this record class.
- */
- private <T extends BaseRecord> String getPathForClass(final Class<T> clazz) {
- String className = StateStoreUtils.getRecordName(clazz);
- StringBuilder sb = new StringBuilder();
- sb.append(getRootDir());
- if (sb.charAt(sb.length() - 1) != '/') {
- sb.append("/");
- }
- sb.append(className);
- return sb.toString();
- }
-
- @Override
- public boolean isDriverReady() {
- return this.initialized;
- }
-
- @Override
- public <T extends BaseRecord> boolean putAll(
- List<T> records, boolean allowUpdate, boolean errorIfExists)
- throws StateStoreUnavailableException {
- verifyDriverReady();
- if (records.isEmpty()) {
- return true;
- }
-
- long start = monotonicNow();
- StateStoreMetrics metrics = getMetrics();
-
- // Check if any record exists
- Map<String, T> toWrite = new HashMap<>();
- for (T record : records) {
- Class<? extends BaseRecord> recordClass = record.getClass();
- String path = getPathForClass(recordClass);
- String primaryKey = getPrimaryKey(record);
- String recordPath = path + "/" + primaryKey;
-
- if (exists(recordPath)) {
- if (allowUpdate) {
- // Update the mod time stamp. Many backends will use their
- // own timestamp for the mod time.
- record.setDateModified(this.getTime());
- toWrite.put(recordPath, record);
- } else if (errorIfExists) {
- LOG.error("Attempt to insert record {} that already exists",
- recordPath);
- if (metrics != null) {
- metrics.addFailure(monotonicNow() - start);
- }
- return false;
- } else {
- LOG.debug("Not updating {}", record);
- }
- } else {
- toWrite.put(recordPath, record);
- }
- }
-
- // Write the records
- boolean success = true;
- for (Entry<String, T> entry : toWrite.entrySet()) {
- String recordPath = entry.getKey();
- String recordPathTemp = recordPath + "." + now() + TMP_MARK;
- BufferedWriter writer = getWriter(recordPathTemp);
- try {
- T record = entry.getValue();
- String line = serializeString(record);
- writer.write(line);
- } catch (IOException e) {
- LOG.error("Cannot write {}", recordPathTemp, e);
- success = false;
- } finally {
- if (writer != null) {
- try {
- writer.close();
- } catch (IOException e) {
- LOG.error("Cannot close the writer for {}", recordPathTemp);
- }
- }
- }
- // Commit
- if (!rename(recordPathTemp, recordPath)) {
- LOG.error("Failed committing record into {}", recordPath);
- success = false;
- }
- }
-
- long end = monotonicNow();
- if (metrics != null) {
- if (success) {
- metrics.addWrite(end - start);
- } else {
- metrics.addFailure(end - start);
- }
- }
- return success;
- }
-
- @Override
- public <T extends BaseRecord> int remove(Class<T> clazz, Query<T> query)
- throws StateStoreUnavailableException {
- verifyDriverReady();
-
- if (query == null) {
- return 0;
- }
-
- long start = Time.monotonicNow();
- StateStoreMetrics metrics = getMetrics();
- int removed = 0;
- // Get the current records
- try {
- final QueryResult<T> result = get(clazz);
- final List<T> existingRecords = result.getRecords();
- // Write all of the existing records except those to be removed
- final List<T> recordsToRemove = filterMultiple(query, existingRecords);
- boolean success = true;
- for (T recordToRemove : recordsToRemove) {
- String path = getPathForClass(clazz);
- String primaryKey = getPrimaryKey(recordToRemove);
- String recordToRemovePath = path + "/" + primaryKey;
- if (remove(recordToRemovePath)) {
- removed++;
- } else {
- LOG.error("Cannot remove record {}", recordToRemovePath);
- success = false;
- }
- }
- if (!success) {
- LOG.error("Cannot remove records {} query {}", clazz, query);
- if (metrics != null) {
- metrics.addFailure(monotonicNow() - start);
- }
- }
- } catch (IOException e) {
- LOG.error("Cannot remove records {} query {}", clazz, query, e);
- if (metrics != null) {
- metrics.addFailure(monotonicNow() - start);
- }
- }
-
- if (removed > 0 && metrics != null) {
- metrics.addRemove(monotonicNow() - start);
- }
- return removed;
- }
-
- @Override
- public <T extends BaseRecord> boolean removeAll(Class<T> clazz)
- throws StateStoreUnavailableException {
- verifyDriverReady();
- long start = Time.monotonicNow();
- StateStoreMetrics metrics = getMetrics();
-
- boolean success = true;
- String path = getPathForClass(clazz);
- List<String> children = getChildren(path);
- for (String child : children) {
- String pathRecord = path + "/" + child;
- if (!remove(pathRecord)) {
- success = false;
- }
- }
-
- if (metrics != null) {
- long time = Time.monotonicNow() - start;
- if (success) {
- metrics.addRemove(time);
- } else {
- metrics.addFailure(time);
- }
- }
- return success;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java
deleted file mode 100644
index c585a23..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.nio.charset.StandardCharsets;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.io.Files;
-
-/**
- * StateStoreDriver implementation based on a local file.
- */
-public class StateStoreFileImpl extends StateStoreFileBaseImpl {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(StateStoreFileImpl.class);
-
- /** Configuration keys. */
- public static final String FEDERATION_STORE_FILE_DIRECTORY =
- DFSConfigKeys.FEDERATION_STORE_PREFIX + "driver.file.directory";
-
- /** Root directory for the state store. */
- private String rootDirectory;
-
-
- @Override
- protected boolean exists(String path) {
- File test = new File(path);
- return test.exists();
- }
-
- @Override
- protected boolean mkdir(String path) {
- File dir = new File(path);
- return dir.mkdirs();
- }
-
- @Override
- protected boolean rename(String src, String dst) {
- try {
- Files.move(new File(src), new File(dst));
- return true;
- } catch (IOException e) {
- LOG.error("Cannot rename {} to {}", src, dst, e);
- return false;
- }
- }
-
- @Override
- protected boolean remove(String path) {
- File file = new File(path);
- return file.delete();
- }
-
- @Override
- protected String getRootDir() {
- if (this.rootDirectory == null) {
- String dir = getConf().get(FEDERATION_STORE_FILE_DIRECTORY);
- if (dir == null) {
- File tempDir = Files.createTempDir();
- dir = tempDir.getAbsolutePath();
- LOG.warn("The root directory is not available, using {}", dir);
- }
- this.rootDirectory = dir;
- }
- return this.rootDirectory;
- }
-
- @Override
- protected <T extends BaseRecord> BufferedReader getReader(String filename) {
- BufferedReader reader = null;
- try {
- LOG.debug("Loading file: {}", filename);
- File file = new File(filename);
- FileInputStream fis = new FileInputStream(file);
- InputStreamReader isr =
- new InputStreamReader(fis, StandardCharsets.UTF_8);
- reader = new BufferedReader(isr);
- } catch (Exception ex) {
- LOG.error("Cannot open read stream for record {}", filename, ex);
- }
- return reader;
- }
-
- @Override
- protected <T extends BaseRecord> BufferedWriter getWriter(String filename) {
- BufferedWriter writer = null;
- try {
- LOG.debug("Writing file: {}", filename);
- File file = new File(filename);
- FileOutputStream fos = new FileOutputStream(file, false);
- OutputStreamWriter osw =
- new OutputStreamWriter(fos, StandardCharsets.UTF_8);
- writer = new BufferedWriter(osw);
- } catch (IOException e) {
- LOG.error("Cannot open write stream for record {}", filename, e);
- }
- return writer;
- }
-
- @Override
- public void close() throws Exception {
- setInitialized(false);
- }
-
- @Override
- protected List<String> getChildren(String path) {
- List<String> ret = new LinkedList<>();
- File dir = new File(path);
- File[] files = dir.listFiles();
- if (files != null) {
- for (File file : files) {
- String filename = file.getName();
- ret.add(filename);
- }
- }
- return ret;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileSystemImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileSystemImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileSystemImpl.java
deleted file mode 100644
index 8d6c626..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileSystemImpl.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.net.URI;
-import java.nio.charset.StandardCharsets;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Options;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * StateStoreDriver} implementation based on a filesystem. The most common uses
- * HDFS as a backend.
- */
-public class StateStoreFileSystemImpl extends StateStoreFileBaseImpl {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(StateStoreFileSystemImpl.class);
-
-
- /** Configuration keys. */
- public static final String FEDERATION_STORE_FS_PATH =
- DFSConfigKeys.FEDERATION_STORE_PREFIX + "driver.fs.path";
-
- /** File system to back the State Store. */
- private FileSystem fs;
- /** Working path in the filesystem. */
- private String workPath;
-
- @Override
- protected boolean exists(String path) {
- try {
- return fs.exists(new Path(path));
- } catch (IOException e) {
- return false;
- }
- }
-
- @Override
- protected boolean mkdir(String path) {
- try {
- return fs.mkdirs(new Path(path));
- } catch (IOException e) {
- return false;
- }
- }
-
- @Override
- protected boolean rename(String src, String dst) {
- try {
- if (fs instanceof DistributedFileSystem) {
- DistributedFileSystem dfs = (DistributedFileSystem)fs;
- dfs.rename(new Path(src), new Path(dst), Options.Rename.OVERWRITE);
- return true;
- } else {
- // Replace should be atomic but not available
- if (fs.exists(new Path(dst))) {
- fs.delete(new Path(dst), true);
- }
- return fs.rename(new Path(src), new Path(dst));
- }
- } catch (Exception e) {
- LOG.error("Cannot rename {} to {}", src, dst, e);
- return false;
- }
- }
-
- @Override
- protected boolean remove(String path) {
- try {
- return fs.delete(new Path(path), true);
- } catch (Exception e) {
- LOG.error("Cannot remove {}", path, e);
- return false;
- }
- }
-
- @Override
- protected String getRootDir() {
- if (this.workPath == null) {
- String rootPath = getConf().get(FEDERATION_STORE_FS_PATH);
- URI workUri;
- try {
- workUri = new URI(rootPath);
- fs = FileSystem.get(workUri, getConf());
- } catch (Exception ex) {
- return null;
- }
- this.workPath = rootPath;
- }
- return this.workPath;
- }
-
- @Override
- public void close() throws Exception {
- if (fs != null) {
- fs.close();
- }
- }
-
- @Override
- protected <T extends BaseRecord> BufferedReader getReader(String pathName) {
- BufferedReader reader = null;
- Path path = new Path(pathName);
- try {
- FSDataInputStream fdis = fs.open(path);
- InputStreamReader isr =
- new InputStreamReader(fdis, StandardCharsets.UTF_8);
- reader = new BufferedReader(isr);
- } catch (IOException ex) {
- LOG.error("Cannot open read stream for {}", path);
- }
- return reader;
- }
-
- @Override
- protected <T extends BaseRecord> BufferedWriter getWriter(String pathName) {
- BufferedWriter writer = null;
- Path path = new Path(pathName);
- try {
- FSDataOutputStream fdos = fs.create(path, true);
- OutputStreamWriter osw =
- new OutputStreamWriter(fdos, StandardCharsets.UTF_8);
- writer = new BufferedWriter(osw);
- } catch (IOException ex) {
- LOG.error("Cannot open write stream for {}", path);
- }
- return writer;
- }
-
- @Override
- protected List<String> getChildren(String pathName) {
- List<String> ret = new LinkedList<>();
- Path path = new Path(workPath, pathName);
- try {
- FileStatus[] files = fs.listStatus(path);
- for (FileStatus file : files) {
- Path filePath = file.getPath();
- String fileName = filePath.getName();
- ret.add(fileName);
- }
- } catch (Exception e) {
- LOG.error("Cannot get children for {}", pathName, e);
- }
- return ret;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializableImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializableImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializableImpl.java
deleted file mode 100644
index 7bc93de..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializableImpl.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
-
-import java.io.IOException;
-import java.util.Collection;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.server.federation.metrics.StateStoreMetrics;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
-
-/**
- * State Store driver that stores a serialization of the records. The serializer
- * is pluggable.
- */
-public abstract class StateStoreSerializableImpl extends StateStoreBaseImpl {
-
- /** Mark for slashes in path names. */
- protected static final String SLASH_MARK = "0SLASH0";
- /** Mark for colon in path names. */
- protected static final String COLON_MARK = "_";
-
- /** Default serializer for this driver. */
- private StateStoreSerializer serializer;
-
-
- @Override
- public boolean init(final Configuration config, final String id,
- final Collection<Class<? extends BaseRecord>> records,
- final StateStoreMetrics metrics) {
- boolean ret = super.init(config, id, records, metrics);
-
- this.serializer = StateStoreSerializer.getSerializer(config);
-
- return ret;
- }
-
- /**
- * Serialize a record using the serializer.
- * @param record Record to serialize.
- * @return Byte array with the serialization of the record.
- */
- protected <T extends BaseRecord> byte[] serialize(T record) {
- return serializer.serialize(record);
- }
-
- /**
- * Serialize a record using the serializer.
- * @param record Record to serialize.
- * @return String with the serialization of the record.
- */
- protected <T extends BaseRecord> String serializeString(T record) {
- return serializer.serializeString(record);
- }
-
- /**
- * Creates a record from an input data string.
- * @param data Serialized text of the record.
- * @param clazz Record class.
- * @param includeDates If dateModified and dateCreated are serialized.
- * @return The created record.
- * @throws IOException
- */
- protected <T extends BaseRecord> T newRecord(
- String data, Class<T> clazz, boolean includeDates) throws IOException {
- return serializer.deserialize(data, clazz);
- }
-
- /**
- * Get the primary key for a record. If we don't want to store in folders, we
- * need to remove / from the name.
- *
- * @param record Record to get the primary key for.
- * @return Primary key for the record.
- */
- protected static String getPrimaryKey(BaseRecord record) {
- String primaryKey = record.getPrimaryKey();
- primaryKey = primaryKey.replaceAll("/", SLASH_MARK);
- primaryKey = primaryKey.replaceAll(":", COLON_MARK);
- return primaryKey;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializerPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializerPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializerPBImpl.java
deleted file mode 100644
index 45c5dd6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializerPBImpl.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
-
-import java.io.IOException;
-
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.codec.binary.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-import org.apache.hadoop.util.ReflectionUtils;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the State Store serializer.
- */
-public final class StateStoreSerializerPBImpl extends StateStoreSerializer {
-
- private static final String PB_IMPL_PACKAGE_SUFFIX = "impl.pb";
- private static final String PB_IMPL_CLASS_SUFFIX = "PBImpl";
-
- private Configuration localConf = new Configuration();
-
-
- private StateStoreSerializerPBImpl() {
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public <T> T newRecordInstance(Class<T> clazz) {
- try {
- String clazzPBImpl = getPBImplClassName(clazz);
- Class<?> pbClazz = localConf.getClassByName(clazzPBImpl);
- Object retObject = ReflectionUtils.newInstance(pbClazz, localConf);
- return (T)retObject;
- } catch (ClassNotFoundException e) {
- throw new RuntimeException(e);
- }
- }
-
- private String getPBImplClassName(Class<?> clazz) {
- String srcPackagePart = getPackageName(clazz);
- String srcClassName = getClassName(clazz);
- String destPackagePart = srcPackagePart + "." + PB_IMPL_PACKAGE_SUFFIX;
- String destClassPart = srcClassName + PB_IMPL_CLASS_SUFFIX;
- return destPackagePart + "." + destClassPart;
- }
-
- private String getClassName(Class<?> clazz) {
- String fqName = clazz.getName();
- return (fqName.substring(fqName.lastIndexOf(".") + 1, fqName.length()));
- }
-
- private String getPackageName(Class<?> clazz) {
- return clazz.getPackage().getName();
- }
-
- @Override
- public byte[] serialize(BaseRecord record) {
- byte[] byteArray64 = null;
- if (record instanceof PBRecord) {
- PBRecord recordPB = (PBRecord) record;
- Message msg = recordPB.getProto();
- byte[] byteArray = msg.toByteArray();
- byteArray64 = Base64.encodeBase64(byteArray, false);
- }
- return byteArray64;
- }
-
- @Override
- public String serializeString(BaseRecord record) {
- byte[] byteArray64 = serialize(record);
- String base64Encoded = StringUtils.newStringUtf8(byteArray64);
- return base64Encoded;
- }
-
- @Override
- public <T extends BaseRecord> T deserialize(
- byte[] byteArray, Class<T> clazz) throws IOException {
-
- T record = newRecord(clazz);
- if (record instanceof PBRecord) {
- PBRecord pbRecord = (PBRecord) record;
- byte[] byteArray64 = Base64.encodeBase64(byteArray, false);
- String base64Encoded = StringUtils.newStringUtf8(byteArray64);
- pbRecord.readInstance(base64Encoded);
- }
- return record;
- }
-
- @Override
- public <T extends BaseRecord> T deserialize(String data, Class<T> clazz)
- throws IOException {
- byte[] byteArray64 = Base64.decodeBase64(data);
- return deserialize(byteArray64, clazz);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreZooKeeperImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreZooKeeperImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreZooKeeperImpl.java
deleted file mode 100644
index 7f98171..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreZooKeeperImpl.java
+++ /dev/null
@@ -1,328 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
-
-import static org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils.filterMultiple;
-import static org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils.getRecordName;
-import static org.apache.hadoop.util.curator.ZKCuratorManager.getNodePath;
-import static org.apache.hadoop.util.Time.monotonicNow;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.imps.CuratorFrameworkState;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
-import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
-import org.apache.hadoop.hdfs.server.federation.store.records.Query;
-import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
-import org.apache.hadoop.util.curator.ZKCuratorManager;
-import org.apache.zookeeper.data.Stat;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * {@link StateStoreDriver} driver implementation that uses ZooKeeper as a
- * backend.
- * <p>
- * The structure of the znodes in the ensemble is:
- * PARENT_PATH
- * |--- MOUNT
- * |--- MEMBERSHIP
- * |--- REBALANCER
- * |--- ROUTERS
- */
-public class StateStoreZooKeeperImpl extends StateStoreSerializableImpl {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(StateStoreZooKeeperImpl.class);
-
-
- /** Configuration keys. */
- public static final String FEDERATION_STORE_ZK_DRIVER_PREFIX =
- DFSConfigKeys.FEDERATION_STORE_PREFIX + "driver.zk.";
- public static final String FEDERATION_STORE_ZK_PARENT_PATH =
- FEDERATION_STORE_ZK_DRIVER_PREFIX + "parent-path";
- public static final String FEDERATION_STORE_ZK_PARENT_PATH_DEFAULT =
- "/hdfs-federation";
-
-
- /** Directory to store the state store data. */
- private String baseZNode;
-
- /** Interface to ZooKeeper. */
- private ZKCuratorManager zkManager;
-
-
- @Override
- public boolean initDriver() {
- LOG.info("Initializing ZooKeeper connection");
-
- Configuration conf = getConf();
- baseZNode = conf.get(
- FEDERATION_STORE_ZK_PARENT_PATH,
- FEDERATION_STORE_ZK_PARENT_PATH_DEFAULT);
- try {
- this.zkManager = new ZKCuratorManager(conf);
- this.zkManager.start();
- } catch (IOException e) {
- LOG.error("Cannot initialize the ZK connection", e);
- return false;
- }
- return true;
- }
-
- @Override
- public <T extends BaseRecord> boolean initRecordStorage(
- String className, Class<T> clazz) {
- try {
- String checkPath = getNodePath(baseZNode, className);
- zkManager.createRootDirRecursively(checkPath);
- return true;
- } catch (Exception e) {
- LOG.error("Cannot initialize ZK node for {}: {}",
- className, e.getMessage());
- return false;
- }
- }
-
- @Override
- public void close() throws Exception {
- if (zkManager != null) {
- zkManager.close();
- }
- }
-
- @Override
- public boolean isDriverReady() {
- if (zkManager == null) {
- return false;
- }
- CuratorFramework curator = zkManager.getCurator();
- if (curator == null) {
- return false;
- }
- return curator.getState() == CuratorFrameworkState.STARTED;
- }
-
- @Override
- public <T extends BaseRecord> QueryResult<T> get(Class<T> clazz)
- throws IOException {
- verifyDriverReady();
- long start = monotonicNow();
- List<T> ret = new ArrayList<>();
- String znode = getZNodeForClass(clazz);
- try {
- List<String> children = zkManager.getChildren(znode);
- for (String child : children) {
- try {
- String path = getNodePath(znode, child);
- Stat stat = new Stat();
- String data = zkManager.getStringData(path, stat);
- boolean corrupted = false;
- if (data == null || data.equals("")) {
- // All records should have data, otherwise this is corrupted
- corrupted = true;
- } else {
- try {
- T record = createRecord(data, stat, clazz);
- ret.add(record);
- } catch (IOException e) {
- LOG.error("Cannot create record type \"{}\" from \"{}\": {}",
- clazz.getSimpleName(), data, e.getMessage());
- corrupted = true;
- }
- }
-
- if (corrupted) {
- LOG.error("Cannot get data for {} at {}, cleaning corrupted data",
- child, path);
- zkManager.delete(path);
- }
- } catch (Exception e) {
- LOG.error("Cannot get data for {}: {}", child, e.getMessage());
- }
- }
- } catch (Exception e) {
- getMetrics().addFailure(monotonicNow() - start);
- String msg = "Cannot get children for \"" + znode + "\": " +
- e.getMessage();
- LOG.error(msg);
- throw new IOException(msg);
- }
- long end = monotonicNow();
- getMetrics().addRead(end - start);
- return new QueryResult<T>(ret, getTime());
- }
-
- @Override
- public <T extends BaseRecord> boolean putAll(
- List<T> records, boolean update, boolean error) throws IOException {
- verifyDriverReady();
- if (records.isEmpty()) {
- return true;
- }
-
- // All records should be the same
- T record0 = records.get(0);
- Class<? extends BaseRecord> recordClass = record0.getClass();
- String znode = getZNodeForClass(recordClass);
-
- long start = monotonicNow();
- boolean status = true;
- for (T record : records) {
- String primaryKey = getPrimaryKey(record);
- String recordZNode = getNodePath(znode, primaryKey);
- byte[] data = serialize(record);
- if (!writeNode(recordZNode, data, update, error)){
- status = false;
- }
- }
- long end = monotonicNow();
- if (status) {
- getMetrics().addWrite(end - start);
- } else {
- getMetrics().addFailure(end - start);
- }
- return status;
- }
-
- @Override
- public <T extends BaseRecord> int remove(
- Class<T> clazz, Query<T> query) throws IOException {
- verifyDriverReady();
- if (query == null) {
- return 0;
- }
-
- // Read the current data
- long start = monotonicNow();
- List<T> records = null;
- try {
- QueryResult<T> result = get(clazz);
- records = result.getRecords();
- } catch (IOException ex) {
- LOG.error("Cannot get existing records", ex);
- getMetrics().addFailure(monotonicNow() - start);
- return 0;
- }
-
- // Check the records to remove
- String znode = getZNodeForClass(clazz);
- List<T> recordsToRemove = filterMultiple(query, records);
-
- // Remove the records
- int removed = 0;
- for (T existingRecord : recordsToRemove) {
- LOG.info("Removing \"{}\"", existingRecord);
- try {
- String primaryKey = getPrimaryKey(existingRecord);
- String path = getNodePath(znode, primaryKey);
- if (zkManager.delete(path)) {
- removed++;
- } else {
- LOG.error("Did not remove \"{}\"", existingRecord);
- }
- } catch (Exception e) {
- LOG.error("Cannot remove \"{}\"", existingRecord, e);
- getMetrics().addFailure(monotonicNow() - start);
- }
- }
- long end = monotonicNow();
- if (removed > 0) {
- getMetrics().addRemove(end - start);
- }
- return removed;
- }
-
- @Override
- public <T extends BaseRecord> boolean removeAll(Class<T> clazz)
- throws IOException {
- long start = monotonicNow();
- boolean status = true;
- String znode = getZNodeForClass(clazz);
- LOG.info("Deleting all children under {}", znode);
- try {
- List<String> children = zkManager.getChildren(znode);
- for (String child : children) {
- String path = getNodePath(znode, child);
- LOG.info("Deleting {}", path);
- zkManager.delete(path);
- }
- } catch (Exception e) {
- LOG.error("Cannot remove {}: {}", znode, e.getMessage());
- status = false;
- }
- long time = monotonicNow() - start;
- if (status) {
- getMetrics().addRemove(time);
- } else {
- getMetrics().addFailure(time);
- }
- return status;
- }
-
- private boolean writeNode(
- String znode, byte[] bytes, boolean update, boolean error) {
- try {
- boolean created = zkManager.create(znode);
- if (!update && !created && error) {
- LOG.info("Cannot write record \"{}\", it already exists", znode);
- return false;
- }
-
- // Write data
- zkManager.setData(znode, bytes, -1);
- return true;
- } catch (Exception e) {
- LOG.error("Cannot write record \"{}\": {}", znode, e.getMessage());
- }
- return false;
- }
-
- /**
- * Get the ZNode for a class.
- *
- * @param clazz Record class to evaluate.
- * @return The ZNode for the class.
- */
- private <T extends BaseRecord> String getZNodeForClass(Class<T> clazz) {
- String className = getRecordName(clazz);
- return getNodePath(baseZNode, className);
- }
-
- /**
- * Creates a record from a string returned by ZooKeeper.
- *
- * @param data The data to write.
- * @param stat Stat of the data record to create.
- * @param clazz The data record type to create.
- * @return The created record.
- * @throws IOException
- */
- private <T extends BaseRecord> T createRecord(
- String data, Stat stat, Class<T> clazz) throws IOException {
- T record = newRecord(data, clazz, false);
- record.setDateCreated(stat.getCtime());
- record.setDateModified(stat.getMtime());
- return record;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/package-info.java
deleted file mode 100644
index a18433e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/package-info.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Implementations of state store data providers/drivers. Each driver is
- * responsible for maintaining, querying, updating and deleting persistent data
- * records. Data records are defined as subclasses of
- * {@link org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord}.
- * Each driver implements the
- * {@link org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver
- * StateStoreDriver} interface.
- * <p>
- * Currently supported drivers:
- * <ul>
- * <li>{@link StateStoreFileImpl} A file-based data storage backend.
- * <li>{@link StateStoreZooKeeperImpl} A zookeeper based data storage backend.
- * </ul>
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/package-info.java
deleted file mode 100644
index da998b5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/package-info.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * The state store uses modular data storage classes derived from
- * StateStoreDriver to handle querying, updating and deleting data records. The
- * data storage driver is initialized and maintained by the StateStoreService.
- * The state store supports fetching all records of a type, filtering by column
- * values or fetching a single record by its primary key.
- * <p>
- * Each data storage backend is required to implement the methods contained in
- * the StateStoreDriver interface. These methods allow the querying, updating,
- * inserting and deleting of data records into the state store.
- */
-
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-
-package org.apache.hadoop.hdfs.server.federation.store.driver;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MembershipStoreImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MembershipStoreImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MembershipStoreImpl.java
deleted file mode 100644
index 57b7b61..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MembershipStoreImpl.java
+++ /dev/null
@@ -1,310 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.impl;
-
-import static org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils.filterMultiple;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
-import org.apache.hadoop.hdfs.server.federation.store.MembershipStore;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreCache;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateNamenodeRegistrationRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateNamenodeRegistrationResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
-import org.apache.hadoop.hdfs.server.federation.store.records.Query;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Implementation of the {@link MembershipStore} State Store API.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class MembershipStoreImpl
- extends MembershipStore implements StateStoreCache {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(MembershipStoreImpl.class);
-
-
- /** Reported namespaces that are not decommissioned. */
- private final Set<FederationNamespaceInfo> activeNamespaces;
-
- /** Namenodes (after evaluating the quorum) that are active in the cluster. */
- private final Map<String, MembershipState> activeRegistrations;
- /** Namenode status reports (raw) that were discarded for being too old. */
- private final Map<String, MembershipState> expiredRegistrations;
-
- /** Lock to access the local memory cache. */
- private final ReadWriteLock cacheReadWriteLock =
- new ReentrantReadWriteLock();
- private final Lock cacheReadLock = cacheReadWriteLock.readLock();
- private final Lock cacheWriteLock = cacheReadWriteLock.writeLock();
-
-
- public MembershipStoreImpl(StateStoreDriver driver) {
- super(driver);
-
- this.activeRegistrations = new HashMap<>();
- this.expiredRegistrations = new HashMap<>();
- this.activeNamespaces = new TreeSet<>();
- }
-
- @Override
- public GetNamenodeRegistrationsResponse getExpiredNamenodeRegistrations(
- GetNamenodeRegistrationsRequest request) throws IOException {
-
- GetNamenodeRegistrationsResponse response =
- GetNamenodeRegistrationsResponse.newInstance();
- cacheReadLock.lock();
- try {
- Collection<MembershipState> vals = this.expiredRegistrations.values();
- List<MembershipState> copyVals = new ArrayList<>(vals);
- response.setNamenodeMemberships(copyVals);
- } finally {
- cacheReadLock.unlock();
- }
- return response;
- }
-
- @Override
- public GetNamespaceInfoResponse getNamespaceInfo(
- GetNamespaceInfoRequest request) throws IOException {
-
- Set<FederationNamespaceInfo> namespaces = new HashSet<>();
- try {
- cacheReadLock.lock();
- namespaces.addAll(activeNamespaces);
- } finally {
- cacheReadLock.unlock();
- }
-
- GetNamespaceInfoResponse response =
- GetNamespaceInfoResponse.newInstance(namespaces);
- return response;
- }
-
- @Override
- public GetNamenodeRegistrationsResponse getNamenodeRegistrations(
- final GetNamenodeRegistrationsRequest request) throws IOException {
-
- // TODO Cache some common queries and sorts
- List<MembershipState> ret = null;
-
- cacheReadLock.lock();
- try {
- Collection<MembershipState> registrations = activeRegistrations.values();
- MembershipState partialMembership = request.getPartialMembership();
- if (partialMembership == null) {
- ret = new ArrayList<>(registrations);
- } else {
- Query<MembershipState> query = new Query<>(partialMembership);
- ret = filterMultiple(query, registrations);
- }
- } finally {
- cacheReadLock.unlock();
- }
- // Sort in ascending update date order
- Collections.sort(ret);
-
- GetNamenodeRegistrationsResponse response =
- GetNamenodeRegistrationsResponse.newInstance(ret);
- return response;
- }
-
- @Override
- public NamenodeHeartbeatResponse namenodeHeartbeat(
- NamenodeHeartbeatRequest request) throws IOException {
-
- MembershipState record = request.getNamenodeMembership();
- String nnId = record.getNamenodeKey();
- MembershipState existingEntry = null;
- cacheReadLock.lock();
- try {
- existingEntry = this.activeRegistrations.get(nnId);
- } finally {
- cacheReadLock.unlock();
- }
-
- if (existingEntry != null) {
- if (existingEntry.getState() != record.getState()) {
- LOG.info("NN registration state has changed: {} -> {}",
- existingEntry, record);
- } else {
- LOG.debug("Updating NN registration: {} -> {}", existingEntry, record);
- }
- } else {
- LOG.info("Inserting new NN registration: {}", record);
- }
-
- boolean status = getDriver().put(record, true, false);
-
- NamenodeHeartbeatResponse response =
- NamenodeHeartbeatResponse.newInstance(status);
- return response;
- }
-
- @Override
- public boolean loadCache(boolean force) throws IOException {
- super.loadCache(force);
-
- // Update local cache atomically
- cacheWriteLock.lock();
- try {
- this.activeRegistrations.clear();
- this.expiredRegistrations.clear();
- this.activeNamespaces.clear();
-
- // Build list of NN registrations: nnId -> registration list
- Map<String, List<MembershipState>> nnRegistrations = new HashMap<>();
- List<MembershipState> cachedRecords = getCachedRecords();
- for (MembershipState membership : cachedRecords) {
- String nnId = membership.getNamenodeKey();
- if (membership.getState() == FederationNamenodeServiceState.EXPIRED) {
- // Expired, RPC service does not use these
- String key = membership.getPrimaryKey();
- this.expiredRegistrations.put(key, membership);
- } else {
- // This is a valid NN registration, build a list of all registrations
- // using the NN id to use for the quorum calculation.
- List<MembershipState> nnRegistration =
- nnRegistrations.get(nnId);
- if (nnRegistration == null) {
- nnRegistration = new LinkedList<>();
- nnRegistrations.put(nnId, nnRegistration);
- }
- nnRegistration.add(membership);
- String bpId = membership.getBlockPoolId();
- String cId = membership.getClusterId();
- String nsId = membership.getNameserviceId();
- FederationNamespaceInfo nsInfo =
- new FederationNamespaceInfo(bpId, cId, nsId);
- this.activeNamespaces.add(nsInfo);
- }
- }
-
- // Calculate most representative entry for each active NN id
- for (List<MembershipState> nnRegistration : nnRegistrations.values()) {
- // Run quorum based on NN state
- MembershipState representativeRecord =
- getRepresentativeQuorum(nnRegistration);
- String nnKey = representativeRecord.getNamenodeKey();
- this.activeRegistrations.put(nnKey, representativeRecord);
- }
- LOG.debug("Refreshed {} NN registrations from State Store",
- cachedRecords.size());
- } finally {
- cacheWriteLock.unlock();
- }
- return true;
- }
-
- @Override
- public UpdateNamenodeRegistrationResponse updateNamenodeRegistration(
- UpdateNamenodeRegistrationRequest request) throws IOException {
-
- boolean status = false;
- cacheWriteLock.lock();
- try {
- String namenode = MembershipState.getNamenodeKey(
- request.getNameserviceId(), request.getNamenodeId());
- MembershipState member = this.activeRegistrations.get(namenode);
- if (member != null) {
- member.setState(request.getState());
- status = true;
- }
- } finally {
- cacheWriteLock.unlock();
- }
- UpdateNamenodeRegistrationResponse response =
- UpdateNamenodeRegistrationResponse.newInstance(status);
- return response;
- }
-
- /**
- * Picks the most recent entry in the subset that is most agreeable on the
- * specified field. 1) If a majority of the collection has the same value for
- * the field, the first sorted entry within the subset the matches the
- * majority value 2) Otherwise the first sorted entry in the set of all
- * entries
- *
- * @param records - Collection of state store record objects of the same type
- * @return record that is most representative of the field name
- */
- private MembershipState getRepresentativeQuorum(
- Collection<MembershipState> records) {
-
- // Collate objects by field value: field value -> order set of records
- Map<FederationNamenodeServiceState, TreeSet<MembershipState>> occurenceMap =
- new HashMap<>();
- for (MembershipState record : records) {
- FederationNamenodeServiceState state = record.getState();
- TreeSet<MembershipState> matchingSet = occurenceMap.get(state);
- if (matchingSet == null) {
- // TreeSet orders elements by descending date via comparators
- matchingSet = new TreeSet<>();
- occurenceMap.put(state, matchingSet);
- }
- matchingSet.add(record);
- }
-
- // Select largest group
- TreeSet<MembershipState> largestSet = new TreeSet<>();
- for (TreeSet<MembershipState> matchingSet : occurenceMap.values()) {
- if (largestSet.size() < matchingSet.size()) {
- largestSet = matchingSet;
- }
- }
-
- // If quorum, use the newest element here
- if (largestSet.size() > records.size() / 2) {
- return largestSet.first();
- // Otherwise, return most recent by class comparator
- } else if (records.size() > 0) {
- TreeSet<MembershipState> sortedList = new TreeSet<>(records);
- LOG.debug("Quorum failed, using most recent: {}", sortedList.first());
- return sortedList.first();
- } else {
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MountTableStoreImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MountTableStoreImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MountTableStoreImpl.java
deleted file mode 100644
index eb117d6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MountTableStoreImpl.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.impl;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.hdfs.server.federation.router.RouterAdminServer;
-import org.apache.hadoop.hdfs.server.federation.router.RouterPermissionChecker;
-import org.apache.hadoop.hdfs.server.federation.store.MountTableStore;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-import org.apache.hadoop.hdfs.server.federation.store.records.Query;
-import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.util.Time;
-
-/**
- * Implementation of the {@link MountTableStore} state store API.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class MountTableStoreImpl extends MountTableStore {
-
- public MountTableStoreImpl(StateStoreDriver driver) {
- super(driver);
- }
-
- @Override
- public AddMountTableEntryResponse addMountTableEntry(
- AddMountTableEntryRequest request) throws IOException {
- MountTable mountTable = request.getEntry();
- if (mountTable != null) {
- RouterPermissionChecker pc = RouterAdminServer.getPermissionChecker();
- if (pc != null) {
- pc.checkPermission(mountTable, FsAction.WRITE);
- }
- }
-
- boolean status = getDriver().put(mountTable, false, true);
- AddMountTableEntryResponse response =
- AddMountTableEntryResponse.newInstance();
- response.setStatus(status);
- return response;
- }
-
- @Override
- public UpdateMountTableEntryResponse updateMountTableEntry(
- UpdateMountTableEntryRequest request) throws IOException {
- MountTable mountTable = request.getEntry();
- if (mountTable != null) {
- RouterPermissionChecker pc = RouterAdminServer.getPermissionChecker();
- if (pc != null) {
- pc.checkPermission(mountTable, FsAction.WRITE);
- }
- }
-
- boolean status = getDriver().put(mountTable, true, true);
- UpdateMountTableEntryResponse response =
- UpdateMountTableEntryResponse.newInstance();
- response.setStatus(status);
- return response;
- }
-
- @Override
- public RemoveMountTableEntryResponse removeMountTableEntry(
- RemoveMountTableEntryRequest request) throws IOException {
- final String srcPath = request.getSrcPath();
- final MountTable partial = MountTable.newInstance();
- partial.setSourcePath(srcPath);
- final Query<MountTable> query = new Query<>(partial);
- final MountTable deleteEntry = getDriver().get(getRecordClass(), query);
-
- boolean status = false;
- if (deleteEntry != null) {
- RouterPermissionChecker pc = RouterAdminServer.getPermissionChecker();
- if (pc != null) {
- pc.checkPermission(deleteEntry, FsAction.WRITE);
- }
- status = getDriver().remove(deleteEntry);
- }
-
- RemoveMountTableEntryResponse response =
- RemoveMountTableEntryResponse.newInstance();
- response.setStatus(status);
- return response;
- }
-
- @Override
- public GetMountTableEntriesResponse getMountTableEntries(
- GetMountTableEntriesRequest request) throws IOException {
- RouterPermissionChecker pc =
- RouterAdminServer.getPermissionChecker();
- // Get all values from the cache
- List<MountTable> records = getCachedRecords();
-
- // Sort and filter
- Collections.sort(records, MountTable.SOURCE_COMPARATOR);
- String reqSrcPath = request.getSrcPath();
- if (reqSrcPath != null && !reqSrcPath.isEmpty()) {
- // Return only entries beneath this path
- Iterator<MountTable> it = records.iterator();
- while (it.hasNext()) {
- MountTable record = it.next();
- String srcPath = record.getSourcePath();
- if (!srcPath.startsWith(reqSrcPath)) {
- it.remove();
- } else if (pc != null) {
- // do the READ permission check
- try {
- pc.checkPermission(record, FsAction.READ);
- } catch (AccessControlException ignored) {
- // Remove this mount table entry if it cannot
- // be accessed by current user.
- it.remove();
- }
- }
- }
- }
-
- GetMountTableEntriesResponse response =
- GetMountTableEntriesResponse.newInstance();
- response.setEntries(records);
- response.setTimestamp(Time.now());
- return response;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/RouterStoreImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/RouterStoreImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/RouterStoreImpl.java
deleted file mode 100644
index d58c288..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/RouterStoreImpl.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.impl;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.server.federation.store.RouterStore;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationsRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationsResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.Query;
-import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
-import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
-
-/**
- * Implementation of the {@link RouterStore} state store API.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class RouterStoreImpl extends RouterStore {
-
- public RouterStoreImpl(StateStoreDriver driver) {
- super(driver);
- }
-
- @Override
- public GetRouterRegistrationResponse getRouterRegistration(
- GetRouterRegistrationRequest request) throws IOException {
-
- final RouterState partial = RouterState.newInstance();
- partial.setAddress(request.getRouterId());
- final Query<RouterState> query = new Query<RouterState>(partial);
- RouterState record = getDriver().get(getRecordClass(), query);
- if (record != null) {
- overrideExpiredRecord(record);
- }
- GetRouterRegistrationResponse response =
- GetRouterRegistrationResponse.newInstance();
- response.setRouter(record);
- return response;
- }
-
- @Override
- public GetRouterRegistrationsResponse getRouterRegistrations(
- GetRouterRegistrationsRequest request) throws IOException {
-
- // Get all values from the cache
- QueryResult<RouterState> recordsAndTimeStamp =
- getCachedRecordsAndTimeStamp();
- List<RouterState> records = recordsAndTimeStamp.getRecords();
- long timestamp = recordsAndTimeStamp.getTimestamp();
-
- // Generate response
- GetRouterRegistrationsResponse response =
- GetRouterRegistrationsResponse.newInstance();
- response.setRouters(records);
- response.setTimestamp(timestamp);
- return response;
- }
-
- @Override
- public RouterHeartbeatResponse routerHeartbeat(RouterHeartbeatRequest request)
- throws IOException {
-
- RouterState record = request.getRouter();
- boolean status = getDriver().put(record, true, false);
- RouterHeartbeatResponse response =
- RouterHeartbeatResponse.newInstance(status);
- return response;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/package-info.java
deleted file mode 100644
index 1a50d15..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/package-info.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Contains implementations of the state store API interfaces. All classes
- * derive from {@link
- * org.apache.hadoop.hdfs.server.federation.store.RecordStore}. The API
- * definitions are contained in the
- * org.apache.hadoop.hdfs.server.federation.store package.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-package org.apache.hadoop.hdfs.server.federation.store.impl;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/package-info.java
deleted file mode 100644
index 949ec7c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/package-info.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * The federation state store tracks persistent values that are shared between
- * multiple routers.
- * <p>
- * Data is stored in data records that inherit from a common class. Data records
- * are serialized when written to the data store using a modular serialization
- * implementation. The default is profobuf serialization. Data is stored as rows
- * of records of the same type with each data member in a record representing a
- * column.
- * <p>
- * The state store uses a modular data storage
- * {@link org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver
- * StateStoreDriver} to handle querying, updating and deleting data records. The
- * data storage driver is initialized and maintained by the
- * {@link org.apache.hadoop.hdfs.server.federation.store.
- * StateStoreService FederationStateStoreService}. The state store
- * supports fetching all records of a type, filtering by column values or
- * fetching a single record by its primary key.
- * <p>
- * The state store contains several API interfaces, one for each data records
- * type.
- * <p>
- * <ul>
- * <li>FederationMembershipStateStore: state of all Namenodes in the federation.
- * Uses the MembershipState record.
- * <li>FederationMountTableStore: Mount table mapping paths in the global
- * namespace to individual subcluster paths. Uses the MountTable record.
- * <li>RouterStateStore: State of all routers in the federation. Uses the
- * RouterState record.
- * </ul>
- * <p>
- * Each API is defined in a separate interface. The implementations of these
- * interfaces are responsible for accessing the
- * {@link org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver
- * StateStoreDriver} to query, update and delete data records.
- */
-
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-
-package org.apache.hadoop.hdfs.server.federation.store;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntryRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntryRequest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntryRequest.java
deleted file mode 100644
index 2d9f102..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntryRequest.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-
-/**
- * API request for adding a mount table entry to the state store.
- */
-public abstract class AddMountTableEntryRequest {
-
- public static AddMountTableEntryRequest newInstance() {
- return StateStoreSerializer.newRecord(AddMountTableEntryRequest.class);
- }
-
- public static AddMountTableEntryRequest newInstance(MountTable newEntry) {
- AddMountTableEntryRequest request = newInstance();
- request.setEntry(newEntry);
- return request;
- }
-
- @Public
- @Unstable
- public abstract MountTable getEntry();
-
- @Public
- @Unstable
- public abstract void setEntry(MountTable mount);
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
deleted file mode 100644
index d282a7d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
+++ /dev/null
@@ -1,2214 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HANDLER_COUNT_DEFAULT;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HANDLER_COUNT_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HANDLER_QUEUE_SIZE_DEFAULT;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HANDLER_QUEUE_SIZE_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_READER_COUNT_DEFAULT;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_READER_COUNT_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_READER_QUEUE_SIZE_DEFAULT;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_READER_QUEUE_SIZE_KEY;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.lang.reflect.Array;
-import java.net.InetSocketAddress;
-import java.util.Collection;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.TreeMap;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.crypto.CryptoProtocolVersion;
-import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries;
-import org.apache.hadoop.fs.CacheFlag;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.fs.ContentSummary;
-import org.apache.hadoop.fs.CreateFlag;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
-import org.apache.hadoop.fs.FsServerDefaults;
-import org.apache.hadoop.fs.Options;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.QuotaUsage;
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.fs.XAttr;
-import org.apache.hadoop.fs.XAttrSetFlag;
-import org.apache.hadoop.fs.permission.AclEntry;
-import org.apache.hadoop.fs.permission.AclStatus;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.AddBlockFlag;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.inotify.EventBatchList;
-import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse;
-import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
-import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
-import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
-import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
-import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
-import org.apache.hadoop.hdfs.protocol.ClientProtocol;
-import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-import org.apache.hadoop.hdfs.protocol.DirectoryListing;
-import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats;
-import org.apache.hadoop.hdfs.protocol.EncryptionZone;
-import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
-import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
-import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
-import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
-import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
-import org.apache.hadoop.hdfs.protocol.LocatedBlock;
-import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
-import org.apache.hadoop.hdfs.protocol.OpenFilesIterator;
-import org.apache.hadoop.hdfs.protocol.OpenFilesIterator.OpenFilesType;
-import org.apache.hadoop.hdfs.protocol.ReplicatedBlockStats;
-import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
-import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
-import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing;
-import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
-import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ClientNamenodeProtocol;
-import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB;
-import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB;
-import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
-import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCMetrics;
-import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
-import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
-import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException;
-import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
-import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException;
-import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
-import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
-import org.apache.hadoop.io.EnumSetWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.RPC.Server;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.ipc.StandbyException;
-import org.apache.hadoop.net.NodeBase;
-import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.protobuf.BlockingService;
-
-/**
- * This class is responsible for handling all of the RPC calls to the It is
- * created, started, and stopped by {@link Router}. It implements the
- * {@link ClientProtocol} to mimic a
- * {@link org.apache.hadoop.hdfs.server.namenode.NameNode NameNode} and proxies
- * the requests to the active
- * {@link org.apache.hadoop.hdfs.server.namenode.NameNode NameNode}.
- */
-public class RouterRpcServer extends AbstractService implements ClientProtocol {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(RouterRpcServer.class);
-
-
- /** Configuration for the RPC server. */
- private Configuration conf;
-
- /** Identifier for the super user. */
- private final String superUser;
- /** Identifier for the super group. */
- private final String superGroup;
-
- /** Router using this RPC server. */
- private final Router router;
-
- /** The RPC server that listens to requests from clients. */
- private final Server rpcServer;
- /** The address for this RPC server. */
- private final InetSocketAddress rpcAddress;
-
- /** RPC clients to connect to the Namenodes. */
- private final RouterRpcClient rpcClient;
-
- /** Monitor metrics for the RPC calls. */
- private final RouterRpcMonitor rpcMonitor;
-
-
- /** Interface to identify the active NN for a nameservice or blockpool ID. */
- private final ActiveNamenodeResolver namenodeResolver;
-
- /** Interface to map global name space to HDFS subcluster name spaces. */
- private final FileSubclusterResolver subclusterResolver;
-
- /** If we are in safe mode, fail requests as if a standby NN. */
- private volatile boolean safeMode;
-
- /** Category of the operation that a thread is executing. */
- private final ThreadLocal<OperationCategory> opCategory = new ThreadLocal<>();
-
- // Modules implementing groups of RPC calls
- /** Router Quota calls. */
- private final Quota quotaCall;
- /** Erasure coding calls. */
- private final ErasureCoding erasureCoding;
-
-
- /**
- * Construct a router RPC server.
- *
- * @param configuration HDFS Configuration.
- * @param nnResolver The NN resolver instance to determine active NNs in HA.
- * @param fileResolver File resolver to resolve file paths to subclusters.
- * @throws IOException If the RPC server could not be created.
- */
- public RouterRpcServer(Configuration configuration, Router router,
- ActiveNamenodeResolver nnResolver, FileSubclusterResolver fileResolver)
- throws IOException {
- super(RouterRpcServer.class.getName());
-
- this.conf = configuration;
- this.router = router;
- this.namenodeResolver = nnResolver;
- this.subclusterResolver = fileResolver;
-
- // User and group for reporting
- this.superUser = System.getProperty("user.name");
- this.superGroup = this.conf.get(
- DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_KEY,
- DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT);
-
- // RPC server settings
- int handlerCount = this.conf.getInt(DFS_ROUTER_HANDLER_COUNT_KEY,
- DFS_ROUTER_HANDLER_COUNT_DEFAULT);
-
- int readerCount = this.conf.getInt(DFS_ROUTER_READER_COUNT_KEY,
- DFS_ROUTER_READER_COUNT_DEFAULT);
-
- int handlerQueueSize = this.conf.getInt(DFS_ROUTER_HANDLER_QUEUE_SIZE_KEY,
- DFS_ROUTER_HANDLER_QUEUE_SIZE_DEFAULT);
-
- // Override Hadoop Common IPC setting
- int readerQueueSize = this.conf.getInt(DFS_ROUTER_READER_QUEUE_SIZE_KEY,
- DFS_ROUTER_READER_QUEUE_SIZE_DEFAULT);
- this.conf.setInt(
- CommonConfigurationKeys.IPC_SERVER_RPC_READ_CONNECTION_QUEUE_SIZE_KEY,
- readerQueueSize);
-
- RPC.setProtocolEngine(this.conf, ClientNamenodeProtocolPB.class,
- ProtobufRpcEngine.class);
-
- ClientNamenodeProtocolServerSideTranslatorPB
- clientProtocolServerTranslator =
- new ClientNamenodeProtocolServerSideTranslatorPB(this);
- BlockingService clientNNPbService = ClientNamenodeProtocol
- .newReflectiveBlockingService(clientProtocolServerTranslator);
-
- InetSocketAddress confRpcAddress = conf.getSocketAddr(
- DFSConfigKeys.DFS_ROUTER_RPC_BIND_HOST_KEY,
- DFSConfigKeys.DFS_ROUTER_RPC_ADDRESS_KEY,
- DFSConfigKeys.DFS_ROUTER_RPC_ADDRESS_DEFAULT,
- DFSConfigKeys.DFS_ROUTER_RPC_PORT_DEFAULT);
- LOG.info("RPC server binding to {} with {} handlers for Router {}",
- confRpcAddress, handlerCount, this.router.getRouterId());
-
- this.rpcServer = new RPC.Builder(this.conf)
- .setProtocol(ClientNamenodeProtocolPB.class)
- .setInstance(clientNNPbService)
- .setBindAddress(confRpcAddress.getHostName())
- .setPort(confRpcAddress.getPort())
- .setNumHandlers(handlerCount)
- .setnumReaders(readerCount)
- .setQueueSizePerHandler(handlerQueueSize)
- .setVerbose(false)
- .build();
- // We don't want the server to log the full stack trace for some exceptions
- this.rpcServer.addTerseExceptions(
- RemoteException.class,
- StandbyException.class,
- SafeModeException.class,
- FileNotFoundException.class,
- FileAlreadyExistsException.class,
- AccessControlException.class,
- LeaseExpiredException.class,
- NotReplicatedYetException.class,
- IOException.class);
-
- // The RPC-server port can be ephemeral... ensure we have the correct info
- InetSocketAddress listenAddress = this.rpcServer.getListenerAddress();
- this.rpcAddress = new InetSocketAddress(
- confRpcAddress.getHostName(), listenAddress.getPort());
-
- // Create metrics monitor
- Class<? extends RouterRpcMonitor> rpcMonitorClass = this.conf.getClass(
- DFSConfigKeys.DFS_ROUTER_METRICS_CLASS,
- DFSConfigKeys.DFS_ROUTER_METRICS_CLASS_DEFAULT,
- RouterRpcMonitor.class);
- this.rpcMonitor = ReflectionUtils.newInstance(rpcMonitorClass, conf);
-
- // Create the client
- this.rpcClient = new RouterRpcClient(this.conf, this.router.getRouterId(),
- this.namenodeResolver, this.rpcMonitor);
-
- // Initialize modules
- this.quotaCall = new Quota(this.router, this);
- this.erasureCoding = new ErasureCoding(this);
- }
-
- @Override
- protected void serviceInit(Configuration configuration) throws Exception {
- this.conf = configuration;
-
- if (this.rpcMonitor == null) {
- LOG.error("Cannot instantiate Router RPC metrics class");
- } else {
- this.rpcMonitor.init(this.conf, this, this.router.getStateStore());
- }
-
- super.serviceInit(configuration);
- }
-
- @Override
- protected void serviceStart() throws Exception {
- if (this.rpcServer != null) {
- this.rpcServer.start();
- LOG.info("Router RPC up at: {}", this.getRpcAddress());
- }
- super.serviceStart();
- }
-
- @Override
- protected void serviceStop() throws Exception {
- if (this.rpcServer != null) {
- this.rpcServer.stop();
- }
- if (rpcMonitor != null) {
- this.rpcMonitor.close();
- }
- super.serviceStop();
- }
-
- /**
- * Get the RPC client to the Namenode.
- *
- * @return RPC clients to the Namenodes.
- */
- public RouterRpcClient getRPCClient() {
- return rpcClient;
- }
-
- /**
- * Get the RPC monitor and metrics.
- *
- * @return RPC monitor and metrics.
- */
- public RouterRpcMonitor getRPCMonitor() {
- return rpcMonitor;
- }
-
- /**
- * Allow access to the client RPC server for testing.
- *
- * @return The RPC server.
- */
- @VisibleForTesting
- public Server getServer() {
- return rpcServer;
- }
-
- /**
- * Get the RPC address of the service.
- *
- * @return RPC service address.
- */
- public InetSocketAddress getRpcAddress() {
- return rpcAddress;
- }
-
- /**
- * Check if the Router is in safe mode. We should only see READ, WRITE, and
- * UNCHECKED. It includes a default handler when we haven't implemented an
- * operation. If not supported, it always throws an exception reporting the
- * operation.
- *
- * @param op Category of the operation to check.
- * @param supported If the operation is supported or not. If not, it will
- * throw an UnsupportedOperationException.
- * @throws SafeModeException If the Router is in safe mode and cannot serve
- * client requests.
- * @throws UnsupportedOperationException If the operation is not supported.
- */
- protected void checkOperation(OperationCategory op, boolean supported)
- throws RouterSafeModeException, UnsupportedOperationException {
- checkOperation(op);
-
- if (!supported) {
- if (rpcMonitor != null) {
- rpcMonitor.proxyOpNotImplemented();
- }
- String methodName = getMethodName();
- throw new UnsupportedOperationException(
- "Operation \"" + methodName + "\" is not supported");
- }
- }
-
- /**
- * Check if the Router is in safe mode. We should only see READ, WRITE, and
- * UNCHECKED. This function should be called by all ClientProtocol functions.
- *
- * @param op Category of the operation to check.
- * @throws SafeModeException If the Router is in safe mode and cannot serve
- * client requests.
- */
- protected void checkOperation(OperationCategory op)
- throws RouterSafeModeException {
- // Log the function we are currently calling.
- if (rpcMonitor != null) {
- rpcMonitor.startOp();
- }
- // Log the function we are currently calling.
- if (LOG.isDebugEnabled()) {
- String methodName = getMethodName();
- LOG.debug("Proxying operation: {}", methodName);
- }
-
- // Store the category of the operation category for this thread
- opCategory.set(op);
-
- // We allow unchecked and read operations
- if (op == OperationCategory.UNCHECKED || op == OperationCategory.READ) {
- return;
- }
-
- if (safeMode) {
- // Throw standby exception, router is not available
- if (rpcMonitor != null) {
- rpcMonitor.routerFailureSafemode();
- }
- throw new RouterSafeModeException(router.getRouterId(), op);
- }
- }
-
- /**
- * In safe mode all RPC requests will fail and return a standby exception.
- * The client will try another Router, similar to the client retry logic for
- * HA.
- *
- * @param mode True if enabled, False if disabled.
- */
- public void setSafeMode(boolean mode) {
- this.safeMode = mode;
- }
-
- /**
- * Check if the Router is in safe mode and cannot serve RPC calls.
- *
- * @return If the Router is in safe mode.
- */
- public boolean isInSafeMode() {
- return this.safeMode;
- }
-
- @Override // ClientProtocol
- public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
- throws IOException {
- checkOperation(OperationCategory.WRITE, false);
- return null;
- }
-
- /**
- * The the delegation token from each name service.
- * @param renewer
- * @return Name service -> Token.
- * @throws IOException
- */
- public Map<FederationNamespaceInfo, Token<DelegationTokenIdentifier>>
- getDelegationTokens(Text renewer) throws IOException {
- checkOperation(OperationCategory.WRITE, false);
- return null;
- }
-
- @Override // ClientProtocol
- public long renewDelegationToken(Token<DelegationTokenIdentifier> token)
- throws IOException {
- checkOperation(OperationCategory.WRITE, false);
- return 0;
- }
-
- @Override // ClientProtocol
- public void cancelDelegationToken(Token<DelegationTokenIdentifier> token)
- throws IOException {
- checkOperation(OperationCategory.WRITE, false);
- }
-
- @Override // ClientProtocol
- public LocatedBlocks getBlockLocations(String src, final long offset,
- final long length) throws IOException {
- checkOperation(OperationCategory.READ);
-
- List<RemoteLocation> locations = getLocationsForPath(src, false);
- RemoteMethod remoteMethod = new RemoteMethod("getBlockLocations",
- new Class<?>[] {String.class, long.class, long.class},
- new RemoteParam(), offset, length);
- return (LocatedBlocks) rpcClient.invokeSequential(locations, remoteMethod,
- LocatedBlocks.class, null);
- }
-
- @Override // ClientProtocol
- public FsServerDefaults getServerDefaults() throws IOException {
- checkOperation(OperationCategory.READ);
-
- RemoteMethod method = new RemoteMethod("getServerDefaults");
- String ns = subclusterResolver.getDefaultNamespace();
- return (FsServerDefaults) rpcClient.invokeSingle(ns, method);
- }
-
- @Override // ClientProtocol
- public HdfsFileStatus create(String src, FsPermission masked,
- String clientName, EnumSetWritable<CreateFlag> flag,
- boolean createParent, short replication, long blockSize,
- CryptoProtocolVersion[] supportedVersions, String ecPolicyName)
- throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- RemoteLocation createLocation = getCreateLocation(src);
- RemoteMethod method = new RemoteMethod("create",
- new Class<?>[] {String.class, FsPermission.class, String.class,
- EnumSetWritable.class, boolean.class, short.class,
- long.class, CryptoProtocolVersion[].class,
- String.class},
- createLocation.getDest(), masked, clientName, flag, createParent,
- replication, blockSize, supportedVersions, ecPolicyName);
- return (HdfsFileStatus) rpcClient.invokeSingle(createLocation, method);
- }
-
- /**
- * Get the location to create a file. It checks if the file already existed
- * in one of the locations.
- *
- * @param src Path of the file to check.
- * @return The remote location for this file.
- * @throws IOException If the file has no creation location.
- */
- private RemoteLocation getCreateLocation(final String src)
- throws IOException {
-
- final List<RemoteLocation> locations = getLocationsForPath(src, true);
- if (locations == null || locations.isEmpty()) {
- throw new IOException("Cannot get locations to create " + src);
- }
-
- RemoteLocation createLocation = locations.get(0);
- if (locations.size() > 1) {
- try {
- // Check if this file already exists in other subclusters
- LocatedBlocks existingLocation = getBlockLocations(src, 0, 1);
- if (existingLocation != null) {
- // Forward to the existing location and let the NN handle the error
- LocatedBlock existingLocationLastLocatedBlock =
- existingLocation.getLastLocatedBlock();
- if (existingLocationLastLocatedBlock == null) {
- // The block has no blocks yet, check for the meta data
- for (RemoteLocation location : locations) {
- RemoteMethod method = new RemoteMethod("getFileInfo",
- new Class<?>[] {String.class}, new RemoteParam());
- if (rpcClient.invokeSingle(location, method) != null) {
- createLocation = location;
- break;
- }
- }
- } else {
- ExtendedBlock existingLocationLastBlock =
- existingLocationLastLocatedBlock.getBlock();
- String blockPoolId = existingLocationLastBlock.getBlockPoolId();
- createLocation = getLocationForPath(src, true, blockPoolId);
- }
- }
- } catch (FileNotFoundException fne) {
- // Ignore if the file is not found
- }
- }
- return createLocation;
- }
-
- // Medium
- @Override // ClientProtocol
- public LastBlockWithStatus append(String src, final String clientName,
- final EnumSetWritable<CreateFlag> flag) throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- List<RemoteLocation> locations = getLocationsForPath(src, true);
- RemoteMethod method = new RemoteMethod("append",
- new Class<?>[] {String.class, String.class, EnumSetWritable.class},
- new RemoteParam(), clientName, flag);
- return (LastBlockWithStatus) rpcClient.invokeSequential(
- locations, method, LastBlockWithStatus.class, null);
- }
-
- // Low
- @Override // ClientProtocol
- public boolean recoverLease(String src, String clientName)
- throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- final List<RemoteLocation> locations = getLocationsForPath(src, true);
- RemoteMethod method = new RemoteMethod("recoverLease",
- new Class<?>[] {String.class, String.class}, new RemoteParam(),
- clientName);
- Object result = rpcClient.invokeSequential(
- locations, method, Boolean.class, Boolean.TRUE);
- return (boolean) result;
- }
-
- @Override // ClientProtocol
- public boolean setReplication(String src, short replication)
- throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- List<RemoteLocation> locations = getLocationsForPath(src, true);
- RemoteMethod method = new RemoteMethod("setReplication",
- new Class<?>[] {String.class, short.class}, new RemoteParam(),
- replication);
- Object result = rpcClient.invokeSequential(
- locations, method, Boolean.class, Boolean.TRUE);
- return (boolean) result;
- }
-
- @Override
- public void setStoragePolicy(String src, String policyName)
- throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- List<RemoteLocation> locations = getLocationsForPath(src, true);
- RemoteMethod method = new RemoteMethod("setStoragePolicy",
- new Class<?>[] {String.class, String.class},
- new RemoteParam(), policyName);
- rpcClient.invokeSequential(locations, method, null, null);
- }
-
- @Override
- public BlockStoragePolicy[] getStoragePolicies() throws IOException {
- checkOperation(OperationCategory.READ);
-
- RemoteMethod method = new RemoteMethod("getStoragePolicies");
- String ns = subclusterResolver.getDefaultNamespace();
- return (BlockStoragePolicy[]) rpcClient.invokeSingle(ns, method);
- }
-
- @Override // ClientProtocol
- public void setPermission(String src, FsPermission permissions)
- throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- final List<RemoteLocation> locations = getLocationsForPath(src, true);
- RemoteMethod method = new RemoteMethod("setPermission",
- new Class<?>[] {String.class, FsPermission.class},
- new RemoteParam(), permissions);
- rpcClient.invokeSequential(locations, method);
- }
-
- @Override // ClientProtocol
- public void setOwner(String src, String username, String groupname)
- throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- final List<RemoteLocation> locations = getLocationsForPath(src, true);
- RemoteMethod method = new RemoteMethod("setOwner",
- new Class<?>[] {String.class, String.class, String.class},
- new RemoteParam(), username, groupname);
- rpcClient.invokeSequential(locations, method);
- }
-
- /**
- * Excluded and favored nodes are not verified and will be ignored by
- * placement policy if they are not in the same nameservice as the file.
- */
- @Override // ClientProtocol
- public LocatedBlock addBlock(String src, String clientName,
- ExtendedBlock previous, DatanodeInfo[] excludedNodes, long fileId,
- String[] favoredNodes, EnumSet<AddBlockFlag> addBlockFlags)
- throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- final List<RemoteLocation> locations = getLocationsForPath(src, true);
- RemoteMethod method = new RemoteMethod("addBlock",
- new Class<?>[] {String.class, String.class, ExtendedBlock.class,
- DatanodeInfo[].class, long.class, String[].class,
- EnumSet.class},
- new RemoteParam(), clientName, previous, excludedNodes, fileId,
- favoredNodes, addBlockFlags);
- // TODO verify the excludedNodes and favoredNodes are acceptable to this NN
- return (LocatedBlock) rpcClient.invokeSequential(
- locations, method, LocatedBlock.class, null);
- }
-
- /**
- * Excluded nodes are not verified and will be ignored by placement if they
- * are not in the same nameservice as the file.
- */
- @Override // ClientProtocol
- public LocatedBlock getAdditionalDatanode(final String src, final long fileId,
- final ExtendedBlock blk, final DatanodeInfo[] existings,
- final String[] existingStorageIDs, final DatanodeInfo[] excludes,
- final int numAdditionalNodes, final String clientName)
- throws IOException {
- checkOperation(OperationCategory.READ);
-
- final List<RemoteLocation> locations = getLocationsForPath(src, false);
- RemoteMethod method = new RemoteMethod("getAdditionalDatanode",
- new Class<?>[] {String.class, long.class, ExtendedBlock.class,
- DatanodeInfo[].class, String[].class,
- DatanodeInfo[].class, int.class, String.class},
- new RemoteParam(), fileId, blk, existings, existingStorageIDs, excludes,
- numAdditionalNodes, clientName);
- return (LocatedBlock) rpcClient.invokeSequential(
- locations, method, LocatedBlock.class, null);
- }
-
- @Override // ClientProtocol
- public void abandonBlock(ExtendedBlock b, long fileId, String src,
- String holder) throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- RemoteMethod method = new RemoteMethod("abandonBlock",
- new Class<?>[] {ExtendedBlock.class, long.class, String.class,
- String.class},
- b, fileId, new RemoteParam(), holder);
- rpcClient.invokeSingle(b, method);
- }
-
- @Override // ClientProtocol
- public boolean complete(String src, String clientName, ExtendedBlock last,
- long fileId) throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- final List<RemoteLocation> locations = getLocationsForPath(src, true);
- RemoteMethod method = new RemoteMethod("complete",
- new Class<?>[] {String.class, String.class, ExtendedBlock.class,
- long.class},
- new RemoteParam(), clientName, last, fileId);
- // Complete can return true/false, so don't expect a result
- return ((Boolean) rpcClient.invokeSequential(
- locations, method, Boolean.class, null)).booleanValue();
- }
-
- @Override // ClientProtocol
- public LocatedBlock updateBlockForPipeline(
- ExtendedBlock block, String clientName) throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- RemoteMethod method = new RemoteMethod("updateBlockForPipeline",
- new Class<?>[] {ExtendedBlock.class, String.class},
- block, clientName);
- return (LocatedBlock) rpcClient.invokeSingle(block, method);
- }
-
- /**
- * Datanode are not verified to be in the same nameservice as the old block.
- * TODO This may require validation.
- */
- @Override // ClientProtocol
- public void updatePipeline(String clientName, ExtendedBlock oldBlock,
- ExtendedBlock newBlock, DatanodeID[] newNodes, String[] newStorageIDs)
- throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- RemoteMethod method = new RemoteMethod("updatePipeline",
- new Class<?>[] {String.class, ExtendedBlock.class, ExtendedBlock.class,
- DatanodeID[].class, String[].class},
- clientName, oldBlock, newBlock, newNodes, newStorageIDs);
- rpcClient.invokeSingle(oldBlock, method);
- }
-
- @Override // ClientProtocol
- public long getPreferredBlockSize(String src) throws IOException {
- checkOperation(OperationCategory.READ);
-
- final List<RemoteLocation> locations = getLocationsForPath(src, true);
- RemoteMethod method = new RemoteMethod("getPreferredBlockSize",
- new Class<?>[] {String.class}, new RemoteParam());
- return ((Long) rpcClient.invokeSequential(
- locations, method, Long.class, null)).longValue();
- }
-
- /**
- * Determines combinations of eligible src/dst locations for a rename. A
- * rename cannot change the namespace. Renames are only allowed if there is an
- * eligible dst location in the same namespace as the source.
- *
- * @param srcLocations List of all potential source destinations where the
- * path may be located. On return this list is trimmed to include
- * only the paths that have corresponding destinations in the same
- * namespace.
- * @param dst The destination path
- * @return A map of all eligible source namespaces and their corresponding
- * replacement value.
- * @throws IOException If the dst paths could not be determined.
- */
- private RemoteParam getRenameDestinations(
- final List<RemoteLocation> srcLocations, final String dst)
- throws IOException {
-
- final List<RemoteLocation> dstLocations = getLocationsForPath(dst, true);
- final Map<RemoteLocation, String> dstMap = new HashMap<>();
-
- Iterator<RemoteLocation> iterator = srcLocations.iterator();
- while (iterator.hasNext()) {
- RemoteLocation srcLocation = iterator.next();
- RemoteLocation eligibleDst =
- getFirstMatchingLocation(srcLocation, dstLocations);
- if (eligibleDst != null) {
- // Use this dst for this source location
- dstMap.put(srcLocation, eligibleDst.getDest());
- } else {
- // This src destination is not valid, remove from the source list
- iterator.remove();
- }
- }
- return new RemoteParam(dstMap);
- }
-
- /**
- * Get first matching location.
- *
- * @param location Location we are looking for.
- * @param locations List of locations.
- * @return The first matchin location in the list.
- */
- private RemoteLocation getFirstMatchingLocation(RemoteLocation location,
- List<RemoteLocation> locations) {
- for (RemoteLocation loc : locations) {
- if (loc.getNameserviceId().equals(location.getNameserviceId())) {
- // Return first matching location
- return loc;
- }
- }
- return null;
- }
-
- @Deprecated
- @Override // ClientProtocol
- public boolean rename(final String src, final String dst)
- throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- final List<RemoteLocation> srcLocations = getLocationsForPath(src, true);
- // srcLocations may be trimmed by getRenameDestinations()
- final List<RemoteLocation> locs = new LinkedList<>(srcLocations);
- RemoteParam dstParam = getRenameDestinations(locs, dst);
- if (locs.isEmpty()) {
- throw new IOException(
- "Rename of " + src + " to " + dst + " is not allowed," +
- " no eligible destination in the same namespace was found.");
- }
- RemoteMethod method = new RemoteMethod("rename",
- new Class<?>[] {String.class, String.class},
- new RemoteParam(), dstParam);
- return ((Boolean) rpcClient.invokeSequential(
- locs, method, Boolean.class, Boolean.TRUE)).booleanValue();
- }
-
- @Override // ClientProtocol
- public void rename2(final String src, final String dst,
- final Options.Rename... options) throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- final List<RemoteLocation> srcLocations = getLocationsForPath(src, true);
- // srcLocations may be trimmed by getRenameDestinations()
- final List<RemoteLocation> locs = new LinkedList<>(srcLocations);
- RemoteParam dstParam = getRenameDestinations(locs, dst);
- if (locs.isEmpty()) {
- throw new IOException(
- "Rename of " + src + " to " + dst + " is not allowed," +
- " no eligible destination in the same namespace was found.");
- }
- RemoteMethod method = new RemoteMethod("rename2",
- new Class<?>[] {String.class, String.class, options.getClass()},
- new RemoteParam(), dstParam, options);
- rpcClient.invokeSequential(locs, method, null, null);
- }
-
- @Override // ClientProtocol
- public void concat(String trg, String[] src) throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- // See if the src and target files are all in the same namespace
- LocatedBlocks targetBlocks = getBlockLocations(trg, 0, 1);
- if (targetBlocks == null) {
- throw new IOException("Cannot locate blocks for target file - " + trg);
- }
- LocatedBlock lastLocatedBlock = targetBlocks.getLastLocatedBlock();
- String targetBlockPoolId = lastLocatedBlock.getBlock().getBlockPoolId();
- for (String source : src) {
- LocatedBlocks sourceBlocks = getBlockLocations(source, 0, 1);
- if (sourceBlocks == null) {
- throw new IOException(
- "Cannot located blocks for source file " + source);
- }
- String sourceBlockPoolId =
- sourceBlocks.getLastLocatedBlock().getBlock().getBlockPoolId();
- if (!sourceBlockPoolId.equals(targetBlockPoolId)) {
- throw new IOException("Cannot concatenate source file " + source
- + " because it is located in a different namespace"
- + " with block pool id " + sourceBlockPoolId
- + " from the target file with block pool id "
- + targetBlockPoolId);
- }
- }
-
- // Find locations in the matching namespace.
- final RemoteLocation targetDestination =
- getLocationForPath(trg, true, targetBlockPoolId);
- String[] sourceDestinations = new String[src.length];
- for (int i = 0; i < src.length; i++) {
- String sourceFile = src[i];
- RemoteLocation location =
- getLocationForPath(sourceFile, true, targetBlockPoolId);
- sourceDestinations[i] = location.getDest();
- }
- // Invoke
- RemoteMethod method = new RemoteMethod("concat",
- new Class<?>[] {String.class, String[].class},
- targetDestination.getDest(), sourceDestinations);
- rpcClient.invokeSingle(targetDestination, method);
- }
-
- @Override // ClientProtocol
- public boolean truncate(String src, long newLength, String clientName)
- throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- final List<RemoteLocation> locations = getLocationsForPath(src, true);
- RemoteMethod method = new RemoteMethod("truncate",
- new Class<?>[] {String.class, long.class, String.class},
- new RemoteParam(), newLength, clientName);
- return ((Boolean) rpcClient.invokeSequential(locations, method,
- Boolean.class, Boolean.TRUE)).booleanValue();
- }
-
- @Override // ClientProtocol
- public boolean delete(String src, boolean recursive) throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- final List<RemoteLocation> locations = getLocationsForPath(src, true);
- RemoteMethod method = new RemoteMethod("delete",
- new Class<?>[] {String.class, boolean.class}, new RemoteParam(),
- recursive);
- return ((Boolean) rpcClient.invokeSequential(locations, method,
- Boolean.class, Boolean.TRUE)).booleanValue();
- }
-
- @Override // ClientProtocol
- public boolean mkdirs(String src, FsPermission masked, boolean createParent)
- throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- final List<RemoteLocation> locations = getLocationsForPath(src, true);
- if (locations.size() > 1) {
- // Check if this directory already exists
- try {
- HdfsFileStatus fileStatus = getFileInfo(src);
- if (fileStatus != null) {
- // When existing, the NN doesn't return an exception; return true
- return true;
- }
- } catch (IOException ioe) {
- // Can't query if this file exists or not.
- LOG.error("Error requesting file info for path {} while proxing mkdirs",
- src, ioe);
- }
- }
-
- RemoteLocation firstLocation = locations.get(0);
- RemoteMethod method = new RemoteMethod("mkdirs",
- new Class<?>[] {String.class, FsPermission.class, boolean.class},
- new RemoteParam(), masked, createParent);
- return ((Boolean) rpcClient.invokeSingle(firstLocation, method))
- .booleanValue();
- }
-
- @Override // ClientProtocol
- public void renewLease(String clientName) throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- RemoteMethod method = new RemoteMethod("renewLease",
- new Class<?>[] {String.class}, clientName);
- Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
- rpcClient.invokeConcurrent(nss, method, false, false);
- }
-
- @Override // ClientProtocol
- public DirectoryListing getListing(String src, byte[] startAfter,
- boolean needLocation) throws IOException {
- checkOperation(OperationCategory.READ);
-
- // Locate the dir and fetch the listing
- final List<RemoteLocation> locations = getLocationsForPath(src, true);
- RemoteMethod method = new RemoteMethod("getListing",
- new Class<?>[] {String.class, startAfter.getClass(), boolean.class},
- new RemoteParam(), startAfter, needLocation);
- Map<RemoteLocation, DirectoryListing> listings =
- rpcClient.invokeConcurrent(
- locations, method, false, false, DirectoryListing.class);
-
- Map<String, HdfsFileStatus> nnListing = new TreeMap<>();
- int totalRemainingEntries = 0;
- int remainingEntries = 0;
- boolean namenodeListingExists = false;
- if (listings != null) {
- // Check the subcluster listing with the smallest name
- String lastName = null;
- for (Entry<RemoteLocation, DirectoryListing> entry :
- listings.entrySet()) {
- RemoteLocation location = entry.getKey();
- DirectoryListing listing = entry.getValue();
- if (listing == null) {
- LOG.debug("Cannot get listing from {}", location);
- } else {
- totalRemainingEntries += listing.getRemainingEntries();
- HdfsFileStatus[] partialListing = listing.getPartialListing();
- int length = partialListing.length;
- if (length > 0) {
- HdfsFileStatus lastLocalEntry = partialListing[length-1];
- String lastLocalName = lastLocalEntry.getLocalName();
- if (lastName == null || lastName.compareTo(lastLocalName) > 0) {
- lastName = lastLocalName;
- }
- }
- }
- }
-
- // Add existing entries
- for (Object value : listings.values()) {
- DirectoryListing listing = (DirectoryListing) value;
- if (listing != null) {
- namenodeListingExists = true;
- for (HdfsFileStatus file : listing.getPartialListing()) {
- String filename = file.getLocalName();
- if (totalRemainingEntries > 0 && filename.compareTo(lastName) > 0) {
- // Discarding entries further than the lastName
- remainingEntries++;
- } else {
- nnListing.put(filename, file);
- }
- }
- remainingEntries += listing.getRemainingEntries();
- }
- }
- }
-
- // Add mount points at this level in the tree
- final List<String> children = subclusterResolver.getMountPoints(src);
- if (children != null) {
- // Get the dates for each mount point
- Map<String, Long> dates = getMountPointDates(src);
-
- // Create virtual folder with the mount name
- for (String child : children) {
- long date = 0;
- if (dates != null && dates.containsKey(child)) {
- date = dates.get(child);
- }
- // TODO add number of children
- HdfsFileStatus dirStatus = getMountPointStatus(child, 0, date);
-
- // This may overwrite existing listing entries with the mount point
- // TODO don't add if already there?
- nnListing.put(child, dirStatus);
- }
- }
-
- if (!namenodeListingExists && nnListing.size() == 0) {
- // NN returns a null object if the directory cannot be found and has no
- // listing. If we didn't retrieve any NN listing data, and there are no
- // mount points here, return null.
- return null;
- }
-
- // Generate combined listing
- HdfsFileStatus[] combinedData = new HdfsFileStatus[nnListing.size()];
- combinedData = nnListing.values().toArray(combinedData);
- return new DirectoryListing(combinedData, remainingEntries);
- }
-
- @Override // ClientProtocol
- public HdfsFileStatus getFileInfo(String src) throws IOException {
- checkOperation(OperationCategory.READ);
-
- final List<RemoteLocation> locations = getLocationsForPath(src, false);
- RemoteMethod method = new RemoteMethod("getFileInfo",
- new Class<?>[] {String.class}, new RemoteParam());
- HdfsFileStatus ret = (HdfsFileStatus) rpcClient.invokeSequential(
- locations, method, HdfsFileStatus.class, null);
-
- // If there is no real path, check mount points
- if (ret == null) {
- List<String> children = subclusterResolver.getMountPoints(src);
- if (children != null && !children.isEmpty()) {
- Map<String, Long> dates = getMountPointDates(src);
- long date = 0;
- if (dates != null && dates.containsKey(src)) {
- date = dates.get(src);
- }
- ret = getMountPointStatus(src, children.size(), date);
- }
- }
-
- return ret;
- }
-
- @Override // ClientProtocol
- public boolean isFileClosed(String src) throws IOException {
- checkOperation(OperationCategory.READ);
-
- final List<RemoteLocation> locations = getLocationsForPath(src, false);
- RemoteMethod method = new RemoteMethod("isFileClosed",
- new Class<?>[] {String.class}, new RemoteParam());
- return ((Boolean) rpcClient.invokeSequential(
- locations, method, Boolean.class, Boolean.TRUE)).booleanValue();
- }
-
- @Override // ClientProtocol
- public HdfsFileStatus getFileLinkInfo(String src) throws IOException {
- checkOperation(OperationCategory.READ);
-
- final List<RemoteLocation> locations = getLocationsForPath(src, false);
- RemoteMethod method = new RemoteMethod("getFileLinkInfo",
- new Class<?>[] {String.class}, new RemoteParam());
- return (HdfsFileStatus) rpcClient.invokeSequential(
- locations, method, HdfsFileStatus.class, null);
- }
-
- @Override
- public HdfsLocatedFileStatus getLocatedFileInfo(String src,
- boolean needBlockToken) throws IOException {
- checkOperation(OperationCategory.READ);
- final List<RemoteLocation> locations = getLocationsForPath(src, false);
- RemoteMethod method = new RemoteMethod("getLocatedFileInfo",
- new Class<?>[] {String.class, boolean.class}, new RemoteParam(),
- Boolean.valueOf(needBlockToken));
- return (HdfsLocatedFileStatus) rpcClient.invokeSequential(
- locations, method, HdfsFileStatus.class, null);
- }
-
- @Override // ClientProtocol
- public long[] getStats() throws IOException {
- checkOperation(OperationCategory.UNCHECKED);
-
- RemoteMethod method = new RemoteMethod("getStats");
- Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
- Map<FederationNamespaceInfo, long[]> results =
- rpcClient.invokeConcurrent(nss, method, true, false, long[].class);
- long[] combinedData = new long[STATS_ARRAY_LENGTH];
- for (long[] data : results.values()) {
- for (int i = 0; i < combinedData.length && i < data.length; i++) {
- if (data[i] >= 0) {
- combinedData[i] += data[i];
- }
- }
- }
- return combinedData;
- }
-
- @Override // ClientProtocol
- public DatanodeInfo[] getDatanodeReport(DatanodeReportType type)
- throws IOException {
- checkOperation(OperationCategory.UNCHECKED);
- return getDatanodeReport(type, 0);
- }
-
- /**
- * Get the datanode report with a timeout.
- * @param type Type of the datanode.
- * @param timeOutMs Time out for the reply in milliseconds.
- * @return List of datanodes.
- * @throws IOException If it cannot get the report.
- */
- public DatanodeInfo[] getDatanodeReport(
- DatanodeReportType type, long timeOutMs) throws IOException {
- checkOperation(OperationCategory.UNCHECKED);
-
- Map<String, DatanodeInfo> datanodesMap = new LinkedHashMap<>();
- RemoteMethod method = new RemoteMethod("getDatanodeReport",
- new Class<?>[] {DatanodeReportType.class}, type);
-
- Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
- Map<FederationNamespaceInfo, DatanodeInfo[]> results =
- rpcClient.invokeConcurrent(
- nss, method, true, false, timeOutMs, DatanodeInfo[].class);
- for (Entry<FederationNamespaceInfo, DatanodeInfo[]> entry :
- results.entrySet()) {
- FederationNamespaceInfo ns = entry.getKey();
- DatanodeInfo[] result = entry.getValue();
- for (DatanodeInfo node : result) {
- String nodeId = node.getXferAddr();
- if (!datanodesMap.containsKey(nodeId)) {
- // Add the subcluster as a suffix to the network location
- node.setNetworkLocation(
- NodeBase.PATH_SEPARATOR_STR + ns.getNameserviceId() +
- node.getNetworkLocation());
- datanodesMap.put(nodeId, node);
- } else {
- LOG.debug("{} is in multiple subclusters", nodeId);
- }
- }
- }
- // Map -> Array
- Collection<DatanodeInfo> datanodes = datanodesMap.values();
- DatanodeInfo[] combinedData = new DatanodeInfo[datanodes.size()];
- combinedData = datanodes.toArray(combinedData);
- return combinedData;
- }
-
- @Override // ClientProtocol
- public DatanodeStorageReport[] getDatanodeStorageReport(
- DatanodeReportType type) throws IOException {
- checkOperation(OperationCategory.UNCHECKED);
-
- Map<String, DatanodeStorageReport[]> dnSubcluster =
- getDatanodeStorageReportMap(type);
-
- // Avoid repeating machines in multiple subclusters
- Map<String, DatanodeStorageReport> datanodesMap = new LinkedHashMap<>();
- for (DatanodeStorageReport[] dns : dnSubcluster.values()) {
- for (DatanodeStorageReport dn : dns) {
- DatanodeInfo dnInfo = dn.getDatanodeInfo();
- String nodeId = dnInfo.getXferAddr();
- if (!datanodesMap.containsKey(nodeId)) {
- datanodesMap.put(nodeId, dn);
- }
- // TODO merge somehow, right now it just takes the first one
- }
- }
-
- Collection<DatanodeStorageReport> datanodes = datanodesMap.values();
- DatanodeStorageReport[] combinedData =
- new DatanodeStorageReport[datanodes.size()];
- combinedData = datanodes.toArray(combinedData);
- return combinedData;
- }
-
- /**
- * Get the list of datanodes per subcluster.
- *
- * @param type Type of the datanodes to get.
- * @return nsId -> datanode list.
- * @throws IOException
- */
- public Map<String, DatanodeStorageReport[]> getDatanodeStorageReportMap(
- DatanodeReportType type) throws IOException {
-
- Map<String, DatanodeStorageReport[]> ret = new LinkedHashMap<>();
- RemoteMethod method = new RemoteMethod("getDatanodeStorageReport",
- new Class<?>[] {DatanodeReportType.class}, type);
- Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
- Map<FederationNamespaceInfo, DatanodeStorageReport[]> results =
- rpcClient.invokeConcurrent(
- nss, method, true, false, DatanodeStorageReport[].class);
- for (Entry<FederationNamespaceInfo, DatanodeStorageReport[]> entry :
- results.entrySet()) {
- FederationNamespaceInfo ns = entry.getKey();
- String nsId = ns.getNameserviceId();
- DatanodeStorageReport[] result = entry.getValue();
- ret.put(nsId, result);
- }
- return ret;
- }
-
- @Override // ClientProtocol
- public boolean setSafeMode(SafeModeAction action, boolean isChecked)
- throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- // Set safe mode in all the name spaces
- RemoteMethod method = new RemoteMethod("setSafeMode",
- new Class<?>[] {SafeModeAction.class, boolean.class},
- action, isChecked);
- Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
- Map<FederationNamespaceInfo, Boolean> results =
- rpcClient.invokeConcurrent(nss, method, true, true, boolean.class);
-
- // We only report true if all the name space are in safe mode
- int numSafemode = 0;
- for (boolean safemode : results.values()) {
- if (safemode) {
- numSafemode++;
- }
- }
- return numSafemode == results.size();
- }
-
- @Override // ClientProtocol
- public boolean restoreFailedStorage(String arg) throws IOException {
- checkOperation(OperationCategory.UNCHECKED);
-
- RemoteMethod method = new RemoteMethod("restoreFailedStorage",
- new Class<?>[] {String.class}, arg);
- final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
- Map<FederationNamespaceInfo, Boolean> ret =
- rpcClient.invokeConcurrent(nss, method, true, false, boolean.class);
-
- boolean success = true;
- for (boolean s : ret.values()) {
- if (!s) {
- success = false;
- break;
- }
- }
- return success;
- }
-
- @Override // ClientProtocol
- public boolean saveNamespace(long timeWindow, long txGap) throws IOException {
- checkOperation(OperationCategory.UNCHECKED);
-
- RemoteMethod method = new RemoteMethod("saveNamespace",
- new Class<?>[] {Long.class, Long.class}, timeWindow, txGap);
- final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
- Map<FederationNamespaceInfo, Boolean> ret =
- rpcClient.invokeConcurrent(nss, method, true, false, boolean.class);
-
- boolean success = true;
- for (boolean s : ret.values()) {
- if (!s) {
- success = false;
- break;
- }
- }
- return success;
- }
-
- @Override // ClientProtocol
- public long rollEdits() throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- RemoteMethod method = new RemoteMethod("rollEdits", new Class<?>[] {});
- final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
- Map<FederationNamespaceInfo, Long> ret =
- rpcClient.invokeConcurrent(nss, method, true, false, long.class);
-
- // Return the maximum txid
- long txid = 0;
- for (long t : ret.values()) {
- if (t > txid) {
- txid = t;
- }
- }
- return txid;
- }
-
- @Override // ClientProtocol
- public void refreshNodes() throws IOException {
- checkOperation(OperationCategory.UNCHECKED);
-
- RemoteMethod method = new RemoteMethod("refreshNodes", new Class<?>[] {});
- final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
- rpcClient.invokeConcurrent(nss, method, true, true);
- }
-
- @Override // ClientProtocol
- public void finalizeUpgrade() throws IOException {
- checkOperation(OperationCategory.UNCHECKED);
-
- RemoteMethod method = new RemoteMethod("finalizeUpgrade",
- new Class<?>[] {});
- final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
- rpcClient.invokeConcurrent(nss, method, true, false);
- }
-
- @Override // ClientProtocol
- public RollingUpgradeInfo rollingUpgrade(RollingUpgradeAction action)
- throws IOException {
- checkOperation(OperationCategory.READ);
-
- RemoteMethod method = new RemoteMethod("rollingUpgrade",
- new Class<?>[] {RollingUpgradeAction.class}, action);
- final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
- Map<FederationNamespaceInfo, RollingUpgradeInfo> ret =
- rpcClient.invokeConcurrent(
- nss, method, true, false, RollingUpgradeInfo.class);
-
- // Return the first rolling upgrade info
- RollingUpgradeInfo info = null;
- for (RollingUpgradeInfo infoNs : ret.values()) {
- if (info == null && infoNs != null) {
- info = infoNs;
- }
- }
- return info;
- }
-
- @Override // ClientProtocol
- public void metaSave(String filename) throws IOException {
- checkOperation(OperationCategory.UNCHECKED);
-
- RemoteMethod method = new RemoteMethod("metaSave",
- new Class<?>[] {String.class}, filename);
- final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
- rpcClient.invokeConcurrent(nss, method, true, false);
- }
-
- @Override // ClientProtocol
- public CorruptFileBlocks listCorruptFileBlocks(String path, String cookie)
- throws IOException {
- checkOperation(OperationCategory.READ);
-
- final List<RemoteLocation> locations = getLocationsForPath(path, false);
- RemoteMethod method = new RemoteMethod("listCorruptFileBlocks",
- new Class<?>[] {String.class, String.class},
- new RemoteParam(), cookie);
- return (CorruptFileBlocks) rpcClient.invokeSequential(
- locations, method, CorruptFileBlocks.class, null);
- }
-
- @Override // ClientProtocol
- public void setBalancerBandwidth(long bandwidth) throws IOException {
- checkOperation(OperationCategory.UNCHECKED);
-
- RemoteMethod method = new RemoteMethod("setBalancerBandwidth",
- new Class<?>[] {Long.class}, bandwidth);
- final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
- rpcClient.invokeConcurrent(nss, method, true, false);
- }
-
- @Override // ClientProtocol
- public ContentSummary getContentSummary(String path) throws IOException {
- checkOperation(OperationCategory.READ);
-
- // Get the summaries from regular files
- Collection<ContentSummary> summaries = new LinkedList<>();
- FileNotFoundException notFoundException = null;
- try {
- final List<RemoteLocation> locations = getLocationsForPath(path, false);
- RemoteMethod method = new RemoteMethod("getContentSummary",
- new Class<?>[] {String.class}, new RemoteParam());
- Map<RemoteLocation, ContentSummary> results =
- rpcClient.invokeConcurrent(
- locations, method, false, false, ContentSummary.class);
- summaries.addAll(results.values());
- } catch (FileNotFoundException e) {
- notFoundException = e;
- }
-
- // Add mount points at this level in the tree
- final List<String> children = subclusterResolver.getMountPoints(path);
- if (children != null) {
- for (String child : children) {
- Path childPath = new Path(path, child);
- try {
- ContentSummary mountSummary = getContentSummary(childPath.toString());
- if (mountSummary != null) {
- summaries.add(mountSummary);
- }
- } catch (Exception e) {
- LOG.error("Cannot get content summary for mount {}: {}",
- childPath, e.getMessage());
- }
- }
- }
-
- // Throw original exception if no original nor mount points
- if (summaries.isEmpty() && notFoundException != null) {
- throw notFoundException;
- }
-
- return aggregateContentSummary(summaries);
- }
-
- /**
- * Aggregate content summaries for each subcluster.
- *
- * @param summaries Collection of individual summaries.
- * @return Aggregated content summary.
- */
- private ContentSummary aggregateContentSummary(
- Collection<ContentSummary> summaries) {
- if (summaries.size() == 1) {
- return summaries.iterator().next();
- }
-
- long length = 0;
- long fileCount = 0;
- long directoryCount = 0;
- long quota = 0;
- long spaceConsumed = 0;
- long spaceQuota = 0;
-
- for (ContentSummary summary : summaries) {
- length += summary.getLength();
- fileCount += summary.getFileCount();
- directoryCount += summary.getDirectoryCount();
- quota += summary.getQuota();
- spaceConsumed += summary.getSpaceConsumed();
- spaceQuota += summary.getSpaceQuota();
- }
-
- ContentSummary ret = new ContentSummary.Builder()
- .length(length)
- .fileCount(fileCount)
- .directoryCount(directoryCount)
- .quota(quota)
- .spaceConsumed(spaceConsumed)
- .spaceQuota(spaceQuota)
- .build();
- return ret;
- }
-
- @Override // ClientProtocol
- public void fsync(String src, long fileId, String clientName,
- long lastBlockLength) throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- final List<RemoteLocation> locations = getLocationsForPath(src, true);
- RemoteMethod method = new RemoteMethod("fsync",
- new Class<?>[] {String.class, long.class, String.class, long.class },
- new RemoteParam(), fileId, clientName, lastBlockLength);
- rpcClient.invokeSequential(locations, method);
- }
-
- @Override // ClientProtocol
- public void setTimes(String src, long mtime, long atime) throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- final List<RemoteLocation> locations = getLocationsForPath(src, true);
- RemoteMethod method = new RemoteMethod("setTimes",
- new Class<?>[] {String.class, long.class, long.class},
- new RemoteParam(), mtime, atime);
- rpcClient.invokeSequential(locations, method);
- }
-
- @Override // ClientProtocol
- public void createSymlink(String target, String link, FsPermission dirPerms,
- boolean createParent) throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- // TODO Verify that the link location is in the same NS as the targets
- final List<RemoteLocation> targetLocations =
- getLocationsForPath(target, true);
- final List<RemoteLocation> linkLocations =
- getLocationsForPath(link, true);
- RemoteLocation linkLocation = linkLocations.get(0);
- RemoteMethod method = new RemoteMethod("createSymlink",
- new Class<?>[] {String.class, String.class, FsPermission.class,
- boolean.class},
- new RemoteParam(), linkLocation.getDest(), dirPerms, createParent);
- rpcClient.invokeSequential(targetLocations, method);
- }
-
- @Override // ClientProtocol
- public String getLinkTarget(String path) throws IOException {
- checkOperation(OperationCategory.READ);
-
- final List<RemoteLocation> locations = getLocationsForPath(path, true);
- RemoteMethod method = new RemoteMethod("getLinkTarget",
- new Class<?>[] {String.class}, new RemoteParam());
- return (String) rpcClient.invokeSequential(
- locations, method, String.class, null);
- }
-
- @Override // Client Protocol
- public void allowSnapshot(String snapshotRoot) throws IOException {
- checkOperation(OperationCategory.WRITE, false);
- }
-
- @Override // Client Protocol
- public void disallowSnapshot(String snapshot) throws IOException {
- checkOperation(OperationCategory.WRITE, false);
- }
-
- @Override // ClientProtocol
- public void renameSnapshot(String snapshotRoot, String snapshotOldName,
- String snapshotNewName) throws IOException {
- checkOperation(OperationCategory.WRITE, false);
- }
-
- @Override // Client Protocol
- public SnapshottableDirectoryStatus[] getSnapshottableDirListing()
- throws IOException {
- checkOperation(OperationCategory.READ, false);
- return null;
- }
-
- @Override // ClientProtocol
- public SnapshotDiffReport getSnapshotDiffReport(String snapshotRoot,
- String earlierSnapshotName, String laterSnapshotName) throws IOException {
- checkOperation(OperationCategory.READ, false);
- return null;
- }
-
- @Override // ClientProtocol
- public SnapshotDiffReportListing getSnapshotDiffReportListing(
- String snapshotRoot, String earlierSnapshotName, String laterSnapshotName,
- byte[] startPath, int index) throws IOException {
- checkOperation(OperationCategory.READ, false);
- return null;
- }
-
- @Override // ClientProtocol
- public long addCacheDirective(CacheDirectiveInfo path,
- EnumSet<CacheFlag> flags) throws IOException {
- checkOperation(OperationCategory.WRITE, false);
- return 0;
- }
-
- @Override // ClientProtocol
- public void modifyCacheDirective(CacheDirectiveInfo directive,
- EnumSet<CacheFlag> flags) throws IOException {
- checkOperation(OperationCategory.WRITE, false);
- }
-
- @Override // ClientProtocol
- public void removeCacheDirective(long id) throws IOException {
- checkOperation(OperationCategory.WRITE, false);
- }
-
- @Override // ClientProtocol
- public BatchedEntries<CacheDirectiveEntry> listCacheDirectives(
- long prevId, CacheDirectiveInfo filter) throws IOException {
- checkOperation(OperationCategory.READ, false);
- return null;
- }
-
- @Override // ClientProtocol
- public void addCachePool(CachePoolInfo info) throws IOException {
- checkOperation(OperationCategory.WRITE, false);
- }
-
- @Override // ClientProtocol
- public void modifyCachePool(CachePoolInfo info) throws IOException {
- checkOperation(OperationCategory.WRITE, false);
- }
-
- @Override // ClientProtocol
- public void removeCachePool(String cachePoolName) throws IOException {
- checkOperation(OperationCategory.WRITE, false);
- }
-
- @Override // ClientProtocol
- public BatchedEntries<CachePoolEntry> listCachePools(String prevKey)
- throws IOException {
- checkOperation(OperationCategory.READ, false);
- return null;
- }
-
- @Override // ClientProtocol
- public void modifyAclEntries(String src, List<AclEntry> aclSpec)
- throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- // TODO handle virtual directories
- final List<RemoteLocation> locations = getLocationsForPath(src, true);
- RemoteMethod method = new RemoteMethod("modifyAclEntries",
- new Class<?>[] {String.class, List.class},
- new RemoteParam(), aclSpec);
- rpcClient.invokeSequential(locations, method, null, null);
- }
-
- @Override // ClienProtocol
- public void removeAclEntries(String src, List<AclEntry> aclSpec)
- throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- // TODO handle virtual directories
- final List<RemoteLocation> locations = getLocationsForPath(src, true);
- RemoteMethod method = new RemoteMethod("removeAclEntries",
- new Class<?>[] {String.class, List.class},
- new RemoteParam(), aclSpec);
- rpcClient.invokeSequential(locations, method, null, null);
- }
-
- @Override // ClientProtocol
- public void removeDefaultAcl(String src) throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- // TODO handle virtual directories
- final List<RemoteLocation> locations = getLocationsForPath(src, true);
- RemoteMethod method = new RemoteMethod("removeDefaultAcl",
- new Class<?>[] {String.class}, new RemoteParam());
- rpcClient.invokeSequential(locations, method);
- }
-
- @Override // ClientProtocol
- public void removeAcl(String src) throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- // TODO handle virtual directories
- final List<RemoteLocation> locations = getLocationsForPath(src, true);
- RemoteMethod method = new RemoteMethod("removeAcl",
- new Class<?>[] {String.class}, new RemoteParam());
- rpcClient.invokeSequential(locations, method);
- }
-
- @Override // ClientProtocol
- public void setAcl(String src, List<AclEntry> aclSpec) throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- // TODO handle virtual directories
- final List<RemoteLocation> locations = getLocationsForPath(src, true);
- RemoteMethod method = new RemoteMethod(
- "setAcl", new Class<?>[] {String.class, List.class},
- new RemoteParam(), aclSpec);
- rpcClient.invokeSequential(locations, method);
- }
-
- @Override // ClientProtocol
- public AclStatus getAclStatus(String src) throws IOException {
- checkOperation(OperationCategory.READ);
-
- // TODO handle virtual directories
- final List<RemoteLocation> locations = getLocationsForPath(src, false);
- RemoteMethod method = new RemoteMethod("getAclStatus",
- new Class<?>[] {String.class}, new RemoteParam());
- return (AclStatus) rpcClient.invokeSequential(
- locations, method, AclStatus.class, null);
- }
-
- @Override // ClientProtocol
- public void createEncryptionZone(String src, String keyName)
- throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- // TODO handle virtual directories
- final List<RemoteLocation> locations = getLocationsForPath(src, true);
- RemoteMethod method = new RemoteMethod("createEncryptionZone",
- new Class<?>[] {String.class, String.class},
- new RemoteParam(), keyName);
- rpcClient.invokeSequential(locations, method);
- }
-
- @Override // ClientProtocol
- public EncryptionZone getEZForPath(String src) throws IOException {
- checkOperation(OperationCategory.READ);
-
- // TODO handle virtual directories
- final List<RemoteLocation> locations = getLocationsForPath(src, false);
- RemoteMethod method = new RemoteMethod("getEZForPath",
- new Class<?>[] {String.class}, new RemoteParam());
- return (EncryptionZone) rpcClient.invokeSequential(
- locations, method, EncryptionZone.class, null);
- }
-
- @Override // ClientProtocol
- public BatchedEntries<EncryptionZone> listEncryptionZones(long prevId)
- throws IOException {
- checkOperation(OperationCategory.READ, false);
- return null;
- }
-
- @Override // ClientProtocol
- public void reencryptEncryptionZone(String zone, ReencryptAction action)
- throws IOException {
- checkOperation(OperationCategory.WRITE, false);
- }
-
- @Override // ClientProtocol
- public BatchedEntries<ZoneReencryptionStatus> listReencryptionStatus(
- long prevId) throws IOException {
- checkOperation(OperationCategory.READ, false);
- return null;
- }
-
- @Override // ClientProtocol
- public void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag)
- throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- // TODO handle virtual directories
- final List<RemoteLocation> locations = getLocationsForPath(src, true);
- RemoteMethod method = new RemoteMethod("setXAttr",
- new Class<?>[] {String.class, XAttr.class, EnumSet.class},
- new RemoteParam(), xAttr, flag);
- rpcClient.invokeSequential(locations, method);
- }
-
- @SuppressWarnings("unchecked")
- @Override // ClientProtocol
- public List<XAttr> getXAttrs(String src, List<XAttr> xAttrs)
- throws IOException {
- checkOperation(OperationCategory.READ);
-
- // TODO handle virtual directories
- final List<RemoteLocation> locations = getLocationsForPath(src, false);
- RemoteMethod method = new RemoteMethod("getXAttrs",
- new Class<?>[] {String.class, List.class}, new RemoteParam(), xAttrs);
- return (List<XAttr>) rpcClient.invokeSequential(
- locations, method, List.class, null);
- }
-
- @SuppressWarnings("unchecked")
- @Override // ClientProtocol
- public List<XAttr> listXAttrs(String src) throws IOException {
- checkOperation(OperationCategory.READ);
-
- // TODO handle virtual directories
- final List<RemoteLocation> locations = getLocationsForPath(src, false);
- RemoteMethod method = new RemoteMethod("listXAttrs",
- new Class<?>[] {String.class}, new RemoteParam());
- return (List<XAttr>) rpcClient.invokeSequential(
- locations, method, List.class, null);
- }
-
- @Override // ClientProtocol
- public void removeXAttr(String src, XAttr xAttr) throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- // TODO handle virtual directories
- final List<RemoteLocation> locations = getLocationsForPath(src, true);
- RemoteMethod method = new RemoteMethod("removeXAttr",
- new Class<?>[] {String.class, XAttr.class}, new RemoteParam(), xAttr);
- rpcClient.invokeSequential(locations, method);
- }
-
- @Override // ClientProtocol
- public void checkAccess(String path, FsAction mode) throws IOException {
- checkOperation(OperationCategory.READ);
-
- // TODO handle virtual directories
- final List<RemoteLocation> locations = getLocationsForPath(path, true);
- RemoteMethod method = new RemoteMethod("checkAccess",
- new Class<?>[] {String.class, FsAction.class},
- new RemoteParam(), mode);
- rpcClient.invokeSequential(locations, method);
- }
-
- @Override // ClientProtocol
- public long getCurrentEditLogTxid() throws IOException {
- checkOperation(OperationCategory.READ);
-
- RemoteMethod method = new RemoteMethod(
- "getCurrentEditLogTxid", new Class<?>[] {});
- final Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
- Map<FederationNamespaceInfo, Long> ret =
- rpcClient.invokeConcurrent(nss, method, true, false, long.class);
-
- // Return the maximum txid
- long txid = 0;
- for (long t : ret.values()) {
- if (t > txid) {
- txid = t;
- }
- }
- return txid;
- }
-
- @Override // ClientProtocol
- public EventBatchList getEditsFromTxid(long txid) throws IOException {
- checkOperation(OperationCategory.READ, false);
- return null;
- }
-
- @Override
- public DataEncryptionKey getDataEncryptionKey() throws IOException {
- checkOperation(OperationCategory.READ, false);
- return null;
- }
-
- @Override
- public String createSnapshot(String snapshotRoot, String snapshotName)
- throws IOException {
- checkOperation(OperationCategory.WRITE);
- return null;
- }
-
- @Override
- public void deleteSnapshot(String snapshotRoot, String snapshotName)
- throws IOException {
- checkOperation(OperationCategory.WRITE, false);
- }
-
- @Override // ClientProtocol
- public void setQuota(String path, long namespaceQuota, long storagespaceQuota,
- StorageType type) throws IOException {
- this.quotaCall.setQuota(path, namespaceQuota, storagespaceQuota, type);
- }
-
- @Override // ClientProtocol
- public QuotaUsage getQuotaUsage(String path) throws IOException {
- checkOperation(OperationCategory.READ);
- return this.quotaCall.getQuotaUsage(path);
- }
-
- @Override
- public void reportBadBlocks(LocatedBlock[] blocks) throws IOException {
- checkOperation(OperationCategory.WRITE);
-
- // Block pool id -> blocks
- Map<String, List<LocatedBlock>> blockLocations = new HashMap<>();
- for (LocatedBlock block : blocks) {
- String bpId = block.getBlock().getBlockPoolId();
- List<LocatedBlock> bpBlocks = blockLocations.get(bpId);
- if (bpBlocks == null) {
- bpBlocks = new LinkedList<>();
- blockLocations.put(bpId, bpBlocks);
- }
- bpBlocks.add(block);
- }
-
- // Invoke each block pool
- for (Entry<String, List<LocatedBlock>> entry : blockLocations.entrySet()) {
- String bpId = entry.getKey();
- List<LocatedBlock> bpBlocks = entry.getValue();
-
- LocatedBlock[] bpBlocksArray =
- bpBlocks.toArray(new LocatedBlock[bpBlocks.size()]);
- RemoteMethod method = new RemoteMethod("reportBadBlocks",
- new Class<?>[] {LocatedBlock[].class},
- new Object[] {bpBlocksArray});
- rpcClient.invokeSingleBlockPool(bpId, method);
- }
- }
-
- @Override
- public void unsetStoragePolicy(String src) throws IOException {
- checkOperation(OperationCategory.WRITE, false);
- }
-
- @Override
- public BlockStoragePolicy getStoragePolicy(String path) throws IOException {
- checkOperation(OperationCategory.READ, false);
- return null;
- }
-
- @Override // ClientProtocol
- public ErasureCodingPolicyInfo[] getErasureCodingPolicies()
- throws IOException {
- return erasureCoding.getErasureCodingPolicies();
- }
-
- @Override // ClientProtocol
- public Map<String, String> getErasureCodingCodecs() throws IOException {
- return erasureCoding.getErasureCodingCodecs();
- }
-
- @Override // ClientProtocol
- public AddErasureCodingPolicyResponse[] addErasureCodingPolicies(
- ErasureCodingPolicy[] policies) throws IOException {
- return erasureCoding.addErasureCodingPolicies(policies);
- }
-
- @Override // ClientProtocol
- public void removeErasureCodingPolicy(String ecPolicyName)
- throws IOException {
- erasureCoding.removeErasureCodingPolicy(ecPolicyName);
- }
-
- @Override // ClientProtocol
- public void disableErasureCodingPolicy(String ecPolicyName)
- throws IOException {
- erasureCoding.disableErasureCodingPolicy(ecPolicyName);
- }
-
- @Override // ClientProtocol
- public void enableErasureCodingPolicy(String ecPolicyName)
- throws IOException {
- erasureCoding.enableErasureCodingPolicy(ecPolicyName);
- }
-
- @Override // ClientProtocol
- public ErasureCodingPolicy getErasureCodingPolicy(String src)
- throws IOException {
- return erasureCoding.getErasureCodingPolicy(src);
- }
-
- @Override // ClientProtocol
- public void setErasureCodingPolicy(String src, String ecPolicyName)
- throws IOException {
- erasureCoding.setErasureCodingPolicy(src, ecPolicyName);
- }
-
- @Override // ClientProtocol
- public void unsetErasureCodingPolicy(String src) throws IOException {
- erasureCoding.unsetErasureCodingPolicy(src);
- }
-
- @Override
- public ECBlockGroupStats getECBlockGroupStats() throws IOException {
- return erasureCoding.getECBlockGroupStats();
- }
-
- @Override
- public ReplicatedBlockStats getReplicatedBlockStats() throws IOException {
- checkOperation(OperationCategory.READ, false);
- return null;
- }
-
- @Deprecated
- @Override
- public BatchedEntries<OpenFileEntry> listOpenFiles(long prevId)
- throws IOException {
- return listOpenFiles(prevId, EnumSet.of(OpenFilesType.ALL_OPEN_FILES),
- OpenFilesIterator.FILTER_PATH_DEFAULT);
- }
-
- @Override
- public BatchedEntries<OpenFileEntry> listOpenFiles(long prevId,
- EnumSet<OpenFilesType> openFilesTypes, String path) throws IOException {
- checkOperation(OperationCategory.READ, false);
- return null;
- }
-
- /**
- * Locate the location with the matching block pool id.
- *
- * @param path Path to check.
- * @param failIfLocked Fail the request if locked (top mount point).
- * @param blockPoolId The block pool ID of the namespace to search for.
- * @return Prioritized list of locations in the federated cluster.
- * @throws IOException if the location for this path cannot be determined.
- */
- private RemoteLocation getLocationForPath(
- String path, boolean failIfLocked, String blockPoolId)
- throws IOException {
-
- final List<RemoteLocation> locations =
- getLocationsForPath(path, failIfLocked);
-
- String nameserviceId = null;
- Set<FederationNamespaceInfo> namespaces =
- this.namenodeResolver.getNamespaces();
- for (FederationNamespaceInfo namespace : namespaces) {
- if (namespace.getBlockPoolId().equals(blockPoolId)) {
- nameserviceId = namespace.getNameserviceId();
- break;
- }
- }
- if (nameserviceId != null) {
- for (RemoteLocation location : locations) {
- if (location.getNameserviceId().equals(nameserviceId)) {
- return location;
- }
- }
- }
- throw new IOException(
- "Cannot locate a nameservice for block pool " + blockPoolId);
- }
-
- /**
- * Get the possible locations of a path in the federated cluster.
- *
- * @param path Path to check.
- * @param failIfLocked Fail the request if locked (top mount point).
- * @return Prioritized list of locations in the federated cluster.
- * @throws IOException If the location for this path cannot be determined.
- */
- protected List<RemoteLocation> getLocationsForPath(
- String path, boolean failIfLocked) throws IOException {
- try {
- // Check the location for this path
- final PathLocation location =
- this.subclusterResolver.getDestinationForPath(path);
- if (location == null) {
- throw new IOException("Cannot find locations for " + path + " in " +
- this.subclusterResolver);
- }
-
- // We may block some write operations
- if (opCategory.get() == OperationCategory.WRITE) {
- // Check if the path is in a read only mount point
- if (isPathReadOnly(path)) {
- if (this.rpcMonitor != null) {
- this.rpcMonitor.routerFailureReadOnly();
- }
- throw new IOException(path + " is in a read only mount point");
- }
-
- // Check quota
- if (this.router.isQuotaEnabled()) {
- RouterQuotaUsage quotaUsage = this.router.getQuotaManager()
- .getQuotaUsage(path);
- if (quotaUsage != null) {
- quotaUsage.verifyNamespaceQuota();
- quotaUsage.verifyStoragespaceQuota();
- }
- }
- }
-
- return location.getDestinations();
- } catch (IOException ioe) {
- if (this.rpcMonitor != null) {
- this.rpcMonitor.routerFailureStateStore();
- }
- throw ioe;
- }
- }
-
- /**
- * Check if a path is in a read only mount point.
- *
- * @param path Path to check.
- * @return If the path is in a read only mount point.
- */
- private boolean isPathReadOnly(final String path) {
- if (subclusterResolver instanceof MountTableResolver) {
- try {
- MountTableResolver mountTable = (MountTableResolver)subclusterResolver;
- MountTable entry = mountTable.getMountPoint(path);
- if (entry != null && entry.isReadOnly()) {
- return true;
- }
- } catch (IOException e) {
- LOG.error("Cannot get mount point: {}", e.getMessage());
- }
- }
- return false;
- }
-
- /**
- * Get the modification dates for mount points.
- *
- * @param path Name of the path to start checking dates from.
- * @return Map with the modification dates for all sub-entries.
- */
- private Map<String, Long> getMountPointDates(String path) {
- Map<String, Long> ret = new TreeMap<>();
- // TODO add when we have a Mount Table
- return ret;
- }
-
- /**
- * Create a new file status for a mount point.
- *
- * @param name Name of the mount point.
- * @param childrenNum Number of children.
- * @param date Map with the dates.
- * @return New HDFS file status representing a mount point.
- */
- private HdfsFileStatus getMountPointStatus(
- String name, int childrenNum, long date) {
- long modTime = date;
- long accessTime = date;
- FsPermission permission = FsPermission.getDirDefault();
- String owner = this.superUser;
- String group = this.superGroup;
- try {
- // TODO support users, it should be the user for the pointed folder
- UserGroupInformation ugi = getRemoteUser();
- owner = ugi.getUserName();
- group = ugi.getPrimaryGroupName();
- } catch (IOException e) {
- LOG.error("Cannot get the remote user: {}", e.getMessage());
- }
- long inodeId = 0;
- return new HdfsFileStatus.Builder()
- .isdir(true)
- .mtime(modTime)
- .atime(accessTime)
- .perm(permission)
- .owner(owner)
- .group(group)
- .symlink(new byte[0])
- .path(DFSUtil.string2Bytes(name))
- .fileId(inodeId)
- .children(childrenNum)
- .build();
- }
-
- /**
- * Get the name of the method that is calling this function.
- *
- * @return Name of the method calling this function.
- */
- private static String getMethodName() {
- final StackTraceElement[] stack = Thread.currentThread().getStackTrace();
- String methodName = stack[3].getMethodName();
- return methodName;
- }
-
- /**
- * Get the user that is invoking this operation.
- *
- * @return Remote user group information.
- * @throws IOException If we cannot get the user information.
- */
- static UserGroupInformation getRemoteUser() throws IOException {
- UserGroupInformation ugi = Server.getRemoteUser();
- return (ugi != null) ? ugi : UserGroupInformation.getCurrentUser();
- }
-
- /**
- * Merge the outputs from multiple namespaces.
- * @param map Namespace -> Output array.
- * @param clazz Class of the values.
- * @return Array with the outputs.
- */
- protected static <T> T[] merge(
- Map<FederationNamespaceInfo, T[]> map, Class<T> clazz) {
-
- // Put all results into a set to avoid repeats
- Set<T> ret = new LinkedHashSet<>();
- for (T[] values : map.values()) {
- for (T val : values) {
- ret.add(val);
- }
- }
-
- return toArray(ret, clazz);
- }
-
- /**
- * Convert a set of values into an array.
- * @param set Input set.
- * @param clazz Class of the values.
- * @return Array with the values in set.
- */
- private static <T> T[] toArray(Set<T> set, Class<T> clazz) {
- @SuppressWarnings("unchecked")
- T[] combinedData = (T[]) Array.newInstance(clazz, set.size());
- combinedData = set.toArray(combinedData);
- return combinedData;
- }
-
- /**
- * Get quota module implement.
- */
- public Quota getQuotaModule() {
- return this.quotaCall;
- }
-
- /**
- * Get RPC metrics info.
- * @return The instance of FederationRPCMetrics.
- */
- public FederationRPCMetrics getRPCMetrics() {
- return this.rpcMonitor.getRPCMetrics();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSafeModeException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSafeModeException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSafeModeException.java
deleted file mode 100644
index 7a78b5b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSafeModeException.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
-import org.apache.hadoop.ipc.StandbyException;
-
-/**
- * Exception that the Router throws when it is in safe mode. This extends
- * {@link StandbyException} for the client to try another Router when it gets
- * this exception.
- */
-public class RouterSafeModeException extends StandbyException {
-
- private static final long serialVersionUID = 453568188334993493L;
-
- /** Identifier of the Router that generated this exception. */
- private final String routerId;
-
- /**
- * Build a new Router safe mode exception.
- * @param router Identifier of the Router.
- * @param op Category of the operation (READ/WRITE).
- */
- public RouterSafeModeException(String router, OperationCategory op) {
- super("Router " + router + " is in safe mode and cannot handle " + op
- + " requests.");
- this.routerId = router;
- }
-
- /**
- * Get the id of the Router that generated this exception.
- * @return Id of the Router that generated this exception.
- */
- public String getRouterId() {
- return this.routerId;
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClient.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClient.java
new file mode 100644
index 0000000..b36e459
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClient.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolPB;
+import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolTranslatorPB;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+
+/**
+ * Client to connect to the {@link Router} via the admin protocol.
+ */
+@Private
+public class RouterClient implements Closeable {
+
+ private final RouterAdminProtocolTranslatorPB proxy;
+ private final UserGroupInformation ugi;
+
+ private static RouterAdminProtocolTranslatorPB createRouterProxy(
+ InetSocketAddress address, Configuration conf, UserGroupInformation ugi)
+ throws IOException {
+
+ RPC.setProtocolEngine(
+ conf, RouterAdminProtocolPB.class, ProtobufRpcEngine.class);
+
+ AtomicBoolean fallbackToSimpleAuth = new AtomicBoolean(false);
+ final long version = RPC.getProtocolVersion(RouterAdminProtocolPB.class);
+ RouterAdminProtocolPB proxy = RPC.getProtocolProxy(
+ RouterAdminProtocolPB.class, version, address, ugi, conf,
+ NetUtils.getDefaultSocketFactory(conf),
+ RPC.getRpcTimeout(conf), null,
+ fallbackToSimpleAuth).getProxy();
+
+ return new RouterAdminProtocolTranslatorPB(proxy);
+ }
+
+ public RouterClient(InetSocketAddress address, Configuration conf)
+ throws IOException {
+ this.ugi = UserGroupInformation.getCurrentUser();
+ this.proxy = createRouterProxy(address, conf, ugi);
+ }
+
+ public MountTableManager getMountTableManager() {
+ return proxy;
+ }
+
+ public RouterStateManager getRouterStateManager() {
+ return proxy;
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ RPC.stopProxy(proxy);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHeartbeatService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHeartbeatService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHeartbeatService.java
new file mode 100644
index 0000000..fe172c2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHeartbeatService.java
@@ -0,0 +1,166 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.store.CachedRecordStore;
+import org.apache.hadoop.hdfs.server.federation.store.MembershipStore;
+import org.apache.hadoop.hdfs.server.federation.store.MountTableStore;
+import org.apache.hadoop.hdfs.server.federation.store.RecordStore;
+import org.apache.hadoop.hdfs.server.federation.store.RouterStore;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+import org.apache.hadoop.hdfs.server.federation.store.records.StateStoreVersion;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Service to periodically update the Router current state in the State Store.
+ */
+public class RouterHeartbeatService extends PeriodicService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RouterHeartbeatService.class);
+
+ /** Router we are hearbeating. */
+ private final Router router;
+
+ /**
+ * Create a new Router heartbeat service.
+ *
+ * @param router Router to heartbeat.
+ */
+ public RouterHeartbeatService(Router router) {
+ super(RouterHeartbeatService.class.getSimpleName());
+ this.router = router;
+ }
+
+ /**
+ * Trigger the update of the Router state asynchronously.
+ */
+ protected void updateStateAsync() {
+ Thread thread = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ updateStateStore();
+ }
+ }, "Router Heartbeat Async");
+ thread.setDaemon(true);
+ thread.start();
+ }
+
+ /**
+ * Update the state of the Router in the State Store.
+ */
+ @VisibleForTesting
+ synchronized void updateStateStore() {
+ String routerId = router.getRouterId();
+ if (routerId == null) {
+ LOG.error("Cannot heartbeat for router: unknown router id");
+ return;
+ }
+ if (isStoreAvailable()) {
+ RouterStore routerStore = router.getRouterStateManager();
+ try {
+ RouterState record = RouterState.newInstance(
+ routerId, router.getStartTime(), router.getRouterState());
+ StateStoreVersion stateStoreVersion = StateStoreVersion.newInstance(
+ getStateStoreVersion(MembershipStore.class),
+ getStateStoreVersion(MountTableStore.class));
+ record.setStateStoreVersion(stateStoreVersion);
+ RouterHeartbeatRequest request =
+ RouterHeartbeatRequest.newInstance(record);
+ RouterHeartbeatResponse response = routerStore.routerHeartbeat(request);
+ if (!response.getStatus()) {
+ LOG.warn("Cannot heartbeat router {}", routerId);
+ } else {
+ LOG.debug("Router heartbeat for router {}", routerId);
+ }
+ } catch (IOException e) {
+ LOG.error("Cannot heartbeat router {}: {}", routerId, e.getMessage());
+ }
+ } else {
+ LOG.warn("Cannot heartbeat router {}: State Store unavailable", routerId);
+ }
+ }
+
+ /**
+ * Get the version of the data in the State Store.
+ *
+ * @param clazz Class in the State Store.
+ * @return Version of the data.
+ */
+ private <R extends BaseRecord, S extends RecordStore<R>>
+ long getStateStoreVersion(final Class<S> clazz) {
+ long version = -1;
+ try {
+ StateStoreService stateStore = router.getStateStore();
+ S recordStore = stateStore.getRegisteredRecordStore(clazz);
+ if (recordStore != null) {
+ if (recordStore instanceof CachedRecordStore) {
+ CachedRecordStore<R> cachedRecordStore =
+ (CachedRecordStore<R>) recordStore;
+ List<R> records = cachedRecordStore.getCachedRecords();
+ for (BaseRecord record : records) {
+ if (record.getDateModified() > version) {
+ version = record.getDateModified();
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ LOG.error("Cannot get version for {}: {}", clazz, e.getMessage());
+ }
+ return version;
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+
+ long interval = conf.getTimeDuration(
+ RBFConfigKeys.DFS_ROUTER_HEARTBEAT_STATE_INTERVAL_MS,
+ RBFConfigKeys.DFS_ROUTER_HEARTBEAT_STATE_INTERVAL_MS_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ this.setIntervalMs(interval);
+
+ super.serviceInit(conf);
+ }
+
+ @Override
+ public void periodicInvoke() {
+ updateStateStore();
+ }
+
+ private boolean isStoreAvailable() {
+ if (router.getRouterStateManager() == null) {
+ return false;
+ }
+ if (router.getStateStore() == null) {
+ return false;
+ }
+ return router.getStateStore().isDriverReady();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java
new file mode 100644
index 0000000..93a9cff
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.server.common.JspHelper;
+import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.service.AbstractService;
+
+/**
+ * Web interface for the {@link Router}. It exposes the Web UI and the WebHDFS
+ * methods from {@link RouterWebHdfsMethods}.
+ */
+public class RouterHttpServer extends AbstractService {
+
+ protected static final String NAMENODE_ATTRIBUTE_KEY = "name.node";
+
+
+ /** Configuration for the Router HTTP server. */
+ private Configuration conf;
+
+ /** Router using this HTTP server. */
+ private final Router router;
+
+ /** HTTP server. */
+ private HttpServer2 httpServer;
+
+ /** HTTP addresses. */
+ private InetSocketAddress httpAddress;
+ private InetSocketAddress httpsAddress;
+
+
+ public RouterHttpServer(Router router) {
+ super(RouterHttpServer.class.getName());
+ this.router = router;
+ }
+
+ @Override
+ protected void serviceInit(Configuration configuration) throws Exception {
+ this.conf = configuration;
+
+ // Get HTTP address
+ this.httpAddress = conf.getSocketAddr(
+ RBFConfigKeys.DFS_ROUTER_HTTP_BIND_HOST_KEY,
+ RBFConfigKeys.DFS_ROUTER_HTTP_ADDRESS_KEY,
+ RBFConfigKeys.DFS_ROUTER_HTTP_ADDRESS_DEFAULT,
+ RBFConfigKeys.DFS_ROUTER_HTTP_PORT_DEFAULT);
+
+ // Get HTTPs address
+ this.httpsAddress = conf.getSocketAddr(
+ RBFConfigKeys.DFS_ROUTER_HTTPS_BIND_HOST_KEY,
+ RBFConfigKeys.DFS_ROUTER_HTTPS_ADDRESS_KEY,
+ RBFConfigKeys.DFS_ROUTER_HTTPS_ADDRESS_DEFAULT,
+ RBFConfigKeys.DFS_ROUTER_HTTPS_PORT_DEFAULT);
+
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ // Build and start server
+ String webApp = "router";
+ HttpServer2.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(
+ this.conf, this.httpAddress, this.httpsAddress, webApp,
+ DFSConfigKeys.DFS_NAMENODE_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY,
+ DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY);
+
+ this.httpServer = builder.build();
+
+ this.httpServer.setAttribute(NAMENODE_ATTRIBUTE_KEY, this.router);
+ this.httpServer.setAttribute(JspHelper.CURRENT_CONF, this.conf);
+ setupServlets(this.httpServer, this.conf);
+
+ this.httpServer.start();
+
+ // The server port can be ephemeral... ensure we have the correct info
+ InetSocketAddress listenAddress = this.httpServer.getConnectorAddress(0);
+ if (listenAddress != null) {
+ this.httpAddress = new InetSocketAddress(this.httpAddress.getHostName(),
+ listenAddress.getPort());
+ }
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if(this.httpServer != null) {
+ this.httpServer.stop();
+ }
+ super.serviceStop();
+ }
+
+ private static void setupServlets(
+ HttpServer2 httpServer, Configuration conf) {
+ // TODO Add servlets for FSCK, etc
+ }
+
+ public InetSocketAddress getHttpAddress() {
+ return this.httpAddress;
+ }
+
+ public InetSocketAddress getHttpsAddress() {
+ return this.httpsAddress;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetrics.java
new file mode 100644
index 0000000..851538a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetrics.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.apache.hadoop.metrics2.impl.MsInfo.ProcessName;
+import static org.apache.hadoop.metrics2.impl.MsInfo.SessionId;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
+import org.apache.hadoop.metrics2.source.JvmMetrics;
+
+/**
+ * This class is for maintaining the various Router activity statistics
+ * and publishing them through the metrics interfaces.
+ */
+@Metrics(name="RouterActivity", about="Router metrics", context="dfs")
+public class RouterMetrics {
+
+ private final MetricsRegistry registry = new MetricsRegistry("router");
+
+ @Metric("Duration in SafeMode at startup in msec")
+ private MutableGaugeInt safeModeTime;
+
+ private JvmMetrics jvmMetrics = null;
+
+ RouterMetrics(
+ String processName, String sessionId, final JvmMetrics jvmMetrics) {
+ this.jvmMetrics = jvmMetrics;
+ registry.tag(ProcessName, processName).tag(SessionId, sessionId);
+ }
+
+ public static RouterMetrics create(Configuration conf) {
+ String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY);
+ String processName = "Router";
+ MetricsSystem ms = DefaultMetricsSystem.instance();
+ JvmMetrics jm = JvmMetrics.create(processName, sessionId, ms);
+
+ return ms.register(new RouterMetrics(processName, sessionId, jm));
+ }
+
+ public JvmMetrics getJvmMetrics() {
+ return jvmMetrics;
+ }
+
+ public void shutdown() {
+ DefaultMetricsSystem.shutdown();
+ }
+
+ public void setSafeModeTime(long elapsed) {
+ safeModeTime.set((int) elapsed);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetricsService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetricsService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetricsService.java
new file mode 100644
index 0000000..f4debce
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetricsService.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.metrics.FederationMetrics;
+import org.apache.hadoop.hdfs.server.federation.metrics.NamenodeBeanMetrics;
+import org.apache.hadoop.metrics2.source.JvmMetrics;
+import org.apache.hadoop.service.AbstractService;
+
+/**
+ * Service to manage the metrics of the Router.
+ */
+public class RouterMetricsService extends AbstractService {
+
+ /** Router for this metrics. */
+ private final Router router;
+
+ /** Router metrics. */
+ private RouterMetrics routerMetrics;
+ /** Federation metrics. */
+ private FederationMetrics federationMetrics;
+ /** Namenode mock metrics. */
+ private NamenodeBeanMetrics nnMetrics;
+
+
+ public RouterMetricsService(final Router router) {
+ super(RouterMetricsService.class.getName());
+ this.router = router;
+ }
+
+ @Override
+ protected void serviceInit(Configuration configuration) throws Exception {
+ this.routerMetrics = RouterMetrics.create(configuration);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ // Wrapper for all the FSNamesystem JMX interfaces
+ this.nnMetrics = new NamenodeBeanMetrics(this.router);
+
+ // Federation MBean JMX interface
+ this.federationMetrics = new FederationMetrics(this.router);
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ // Remove JMX interfaces
+ if (this.federationMetrics != null) {
+ this.federationMetrics.close();
+ }
+
+ // Remove Namenode JMX interfaces
+ if (this.nnMetrics != null) {
+ this.nnMetrics.close();
+ }
+
+ // Shutdown metrics
+ if (this.routerMetrics != null) {
+ this.routerMetrics.shutdown();
+ }
+ }
+
+ /**
+ * Get the metrics system for the Router.
+ *
+ * @return Router metrics.
+ */
+ public RouterMetrics getRouterMetrics() {
+ return this.routerMetrics;
+ }
+
+ /**
+ * Get the federation metrics.
+ *
+ * @return Federation metrics.
+ */
+ public FederationMetrics getFederationMetrics() {
+ return this.federationMetrics;
+ }
+
+ /**
+ * Get the JVM metrics for the Router.
+ *
+ * @return JVM metrics.
+ */
+ public JvmMetrics getJvmMetrics() {
+ if (this.routerMetrics == null) {
+ return null;
+ }
+ return this.routerMetrics.getJvmMetrics();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterPermissionChecker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterPermissionChecker.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterPermissionChecker.java
new file mode 100644
index 0000000..9d81dce
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterPermissionChecker.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
+
+/**
+ * Class that helps in checking permissions in Router-based federation.
+ */
+public class RouterPermissionChecker extends FSPermissionChecker {
+ static final Log LOG = LogFactory.getLog(RouterPermissionChecker.class);
+
+ /** Mount table default permission. */
+ public static final short MOUNT_TABLE_PERMISSION_DEFAULT = 00755;
+
+ public RouterPermissionChecker(String routerOwner, String supergroup,
+ UserGroupInformation callerUgi) {
+ super(routerOwner, supergroup, callerUgi, null);
+ }
+
+ /**
+ * Whether a mount table entry can be accessed by the current context.
+ *
+ * @param mountTable
+ * MountTable being accessed
+ * @param access
+ * type of action being performed on the cache pool
+ * @throws AccessControlException
+ * if mount table cannot be accessed
+ */
+ public void checkPermission(MountTable mountTable, FsAction access)
+ throws AccessControlException {
+ if (isSuperUser()) {
+ return;
+ }
+
+ FsPermission mode = mountTable.getMode();
+ if (getUser().equals(mountTable.getOwnerName())
+ && mode.getUserAction().implies(access)) {
+ return;
+ }
+
+ if (isMemberOfGroup(mountTable.getGroupName())
+ && mode.getGroupAction().implies(access)) {
+ return;
+ }
+
+ if (!getUser().equals(mountTable.getOwnerName())
+ && !isMemberOfGroup(mountTable.getGroupName())
+ && mode.getOtherAction().implies(access)) {
+ return;
+ }
+
+ throw new AccessControlException(
+ "Permission denied while accessing mount table "
+ + mountTable.getSourcePath()
+ + ": user " + getUser() + " does not have " + access.toString()
+ + " permissions.");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaManager.java
new file mode 100644
index 0000000..0df34fc
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaManager.java
@@ -0,0 +1,172 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.apache.hadoop.hdfs.server.federation.router.FederationUtil.isParentEntry;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+
+/**
+ * Router quota manager in Router. The manager maintains
+ * {@link RouterQuotaUsage} cache of mount tables and do management
+ * for the quota caches.
+ */
+public class RouterQuotaManager {
+ /** Quota usage <MountTable Path, Aggregated QuotaUsage> cache. */
+ private TreeMap<String, RouterQuotaUsage> cache;
+
+ /** Lock to access the quota cache. */
+ private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ private final Lock readLock = readWriteLock.readLock();
+ private final Lock writeLock = readWriteLock.writeLock();
+
+ public RouterQuotaManager() {
+ this.cache = new TreeMap<>();
+ }
+
+ /**
+ * Get all the mount quota paths.
+ */
+ public Set<String> getAll() {
+ readLock.lock();
+ try {
+ return this.cache.keySet();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ /**
+ * Get the nearest ancestor's quota usage, and meanwhile its quota was set.
+ * @param path The path being written.
+ * @return RouterQuotaUsage Quota usage.
+ */
+ public RouterQuotaUsage getQuotaUsage(String path) {
+ readLock.lock();
+ try {
+ RouterQuotaUsage quotaUsage = this.cache.get(path);
+ if (quotaUsage != null && isQuotaSet(quotaUsage)) {
+ return quotaUsage;
+ }
+
+ // If not found, look for its parent path usage value.
+ int pos = path.lastIndexOf(Path.SEPARATOR);
+ if (pos != -1) {
+ String parentPath = path.substring(0, pos);
+ return getQuotaUsage(parentPath);
+ }
+ } finally {
+ readLock.unlock();
+ }
+
+ return null;
+ }
+
+ /**
+ * Get children paths (can including itself) under specified federation path.
+ * @param parentPath
+ * @return Set<String> Children path set.
+ */
+ public Set<String> getPaths(String parentPath) {
+ readLock.lock();
+ try {
+ String from = parentPath;
+ String to = parentPath + Character.MAX_VALUE;
+ SortedMap<String, RouterQuotaUsage> subMap = this.cache.subMap(from, to);
+
+ Set<String> validPaths = new HashSet<>();
+ if (subMap != null) {
+ for (String path : subMap.keySet()) {
+ if (isParentEntry(path, parentPath)) {
+ validPaths.add(path);
+ }
+ }
+ }
+ return validPaths;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ /**
+ * Put new entity into cache.
+ * @param path Mount table path.
+ * @param quotaUsage Corresponding cache value.
+ */
+ public void put(String path, RouterQuotaUsage quotaUsage) {
+ writeLock.lock();
+ try {
+ this.cache.put(path, quotaUsage);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /**
+ * Remove the entity from cache.
+ * @param path Mount table path.
+ */
+ public void remove(String path) {
+ writeLock.lock();
+ try {
+ this.cache.remove(path);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /**
+ * Clean up the cache.
+ */
+ public void clear() {
+ writeLock.lock();
+ try {
+ this.cache.clear();
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /**
+ * Check if the quota was set.
+ * @param quota RouterQuotaUsage set in mount table.
+ */
+ public boolean isQuotaSet(RouterQuotaUsage quota) {
+ if (quota != null) {
+ long nsQuota = quota.getQuota();
+ long ssQuota = quota.getSpaceQuota();
+
+ // once nsQuota or ssQuota was set, this mount table is quota set
+ if (nsQuota != HdfsConstants.QUOTA_DONT_SET
+ || ssQuota != HdfsConstants.QUOTA_DONT_SET) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java
new file mode 100644
index 0000000..9fc93c1
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java
@@ -0,0 +1,227 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.QuotaUsage;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.federation.store.MountTableStore;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Service to periodically update the {@link RouterQuotaUsage}
+ * cached information in the {@link Router} and update corresponding
+ * mount table in State Store.
+ */
+public class RouterQuotaUpdateService extends PeriodicService {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RouterQuotaUpdateService.class);
+
+ private MountTableStore mountTableStore;
+ private RouterRpcServer rpcServer;
+ /** Router using this Service. */
+ private final Router router;
+ /** Router Quota manager. */
+ private RouterQuotaManager quotaManager;
+
+ public RouterQuotaUpdateService(final Router router) throws IOException {
+ super(RouterQuotaUpdateService.class.getName());
+ this.router = router;
+ this.rpcServer = router.getRpcServer();
+ this.quotaManager = router.getQuotaManager();
+
+ if (this.quotaManager == null) {
+ throw new IOException("Router quota manager is not initialized.");
+ }
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ this.setIntervalMs(conf.getTimeDuration(
+ RBFConfigKeys.DFS_ROUTER_QUOTA_CACHE_UPATE_INTERVAL,
+ RBFConfigKeys.DFS_ROUTER_QUOTA_CACHE_UPATE_INTERVAL_DEFAULT,
+ TimeUnit.MILLISECONDS));
+
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void periodicInvoke() {
+ LOG.debug("Start to update quota cache.");
+ try {
+ List<MountTable> updateMountTables = new LinkedList<>();
+ List<MountTable> mountTables = getQuotaSetMountTables();
+ for (MountTable entry : mountTables) {
+ String src = entry.getSourcePath();
+ RouterQuotaUsage oldQuota = entry.getQuota();
+ long nsQuota = oldQuota.getQuota();
+ long ssQuota = oldQuota.getSpaceQuota();
+ // Call RouterRpcServer#getQuotaUsage for getting current quota usage.
+ QuotaUsage currentQuotaUsage = this.rpcServer.getQuotaModule()
+ .getQuotaUsage(src);
+ // If quota is not set in some subclusters under federation path,
+ // set quota for this path.
+ if (currentQuotaUsage.getQuota() == HdfsConstants.QUOTA_DONT_SET) {
+ this.rpcServer.setQuota(src, nsQuota, ssQuota, null);
+ }
+
+ RouterQuotaUsage newQuota = generateNewQuota(oldQuota,
+ currentQuotaUsage);
+ this.quotaManager.put(src, newQuota);
+ entry.setQuota(newQuota);
+
+ // only update mount tables which quota was changed
+ if (!oldQuota.equals(newQuota)) {
+ updateMountTables.add(entry);
+
+ LOG.debug(
+ "Update quota usage entity of path: {}, nsCount: {},"
+ + " nsQuota: {}, ssCount: {}, ssQuota: {}.",
+ src, newQuota.getFileAndDirectoryCount(),
+ newQuota.getQuota(), newQuota.getSpaceConsumed(),
+ newQuota.getSpaceQuota());
+ }
+ }
+
+ updateMountTableEntries(updateMountTables);
+ } catch (IOException e) {
+ LOG.error("Quota cache updated error.", e);
+ }
+ }
+
+ /**
+ * Get mount table store management interface.
+ * @return MountTableStore instance.
+ * @throws IOException
+ */
+ private MountTableStore getMountTableStore() throws IOException {
+ if (this.mountTableStore == null) {
+ this.mountTableStore = router.getStateStore().getRegisteredRecordStore(
+ MountTableStore.class);
+ if (this.mountTableStore == null) {
+ throw new IOException("Mount table state store is not available.");
+ }
+ }
+ return this.mountTableStore;
+ }
+
+ /**
+ * Get all the existing mount tables.
+ * @return List of mount tables.
+ * @throws IOException
+ */
+ private List<MountTable> getMountTableEntries() throws IOException {
+ // scan mount tables from root path
+ GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest
+ .newInstance("/");
+ GetMountTableEntriesResponse getResponse = getMountTableStore()
+ .getMountTableEntries(getRequest);
+ return getResponse.getEntries();
+ }
+
+ /**
+ * Get mount tables which quota was set.
+ * During this time, the quota usage cache will also be updated by
+ * quota manager:
+ * 1. Stale paths (entries) will be removed.
+ * 2. Existing entries will be override and updated.
+ * @return List of mount tables which quota was set.
+ * @throws IOException
+ */
+ private List<MountTable> getQuotaSetMountTables() throws IOException {
+ List<MountTable> mountTables = getMountTableEntries();
+ Set<String> stalePaths = new HashSet<>();
+ for (String path : this.quotaManager.getAll()) {
+ stalePaths.add(path);
+ }
+
+ List<MountTable> neededMountTables = new LinkedList<>();
+ for (MountTable entry : mountTables) {
+ // select mount tables which is quota set
+ if (isQuotaSet(entry)) {
+ neededMountTables.add(entry);
+ }
+
+ // update mount table entries info in quota cache
+ String src = entry.getSourcePath();
+ this.quotaManager.put(src, entry.getQuota());
+ stalePaths.remove(src);
+ }
+
+ // remove stale paths that currently cached
+ for (String stalePath : stalePaths) {
+ this.quotaManager.remove(stalePath);
+ }
+
+ return neededMountTables;
+ }
+
+ /**
+ * Check if the quota was set in given MountTable.
+ * @param mountTable Mount table entry.
+ */
+ private boolean isQuotaSet(MountTable mountTable) {
+ if (mountTable != null) {
+ return this.quotaManager.isQuotaSet(mountTable.getQuota());
+ }
+ return false;
+ }
+
+ /**
+ * Generate a new quota based on old quota and current quota usage value.
+ * @param oldQuota Old quota stored in State Store.
+ * @param currentQuotaUsage Current quota usage value queried from
+ * subcluster.
+ * @return A new RouterQuotaUsage.
+ */
+ private RouterQuotaUsage generateNewQuota(RouterQuotaUsage oldQuota,
+ QuotaUsage currentQuotaUsage) {
+ RouterQuotaUsage newQuota = new RouterQuotaUsage.Builder()
+ .fileAndDirectoryCount(currentQuotaUsage.getFileAndDirectoryCount())
+ .quota(oldQuota.getQuota())
+ .spaceConsumed(currentQuotaUsage.getSpaceConsumed())
+ .spaceQuota(oldQuota.getSpaceQuota()).build();
+ return newQuota;
+ }
+
+ /**
+ * Write out updated mount table entries into State Store.
+ * @param updateMountTables Mount tables to be updated.
+ * @throws IOException
+ */
+ private void updateMountTableEntries(List<MountTable> updateMountTables)
+ throws IOException {
+ for (MountTable entry : updateMountTables) {
+ UpdateMountTableEntryRequest updateRequest = UpdateMountTableEntryRequest
+ .newInstance(entry);
+ getMountTableStore().updateMountTableEntry(updateRequest);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUsage.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUsage.java
new file mode 100644
index 0000000..eedd80f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUsage.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.hadoop.fs.QuotaUsage;
+import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
+import org.apache.hadoop.hdfs.server.namenode.DirectoryWithQuotaFeature;
+import org.apache.hadoop.hdfs.server.namenode.Quota;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * The subclass of {@link QuotaUsage} used in Router-based federation.
+ */
+public final class RouterQuotaUsage extends QuotaUsage {
+
+ /** Default quota usage count. */
+ public static final long QUOTA_USAGE_COUNT_DEFAULT = 0;
+
+ private RouterQuotaUsage(Builder builder) {
+ super(builder);
+ }
+
+ /** Build the instance based on the builder. */
+ public static class Builder extends QuotaUsage.Builder {
+
+ public RouterQuotaUsage build() {
+ return new RouterQuotaUsage(this);
+ }
+
+ @Override
+ public Builder fileAndDirectoryCount(long count) {
+ super.fileAndDirectoryCount(count);
+ return this;
+ }
+
+ @Override
+ public Builder quota(long quota) {
+ super.quota(quota);
+ return this;
+ }
+
+ @Override
+ public Builder spaceConsumed(long spaceConsumed) {
+ super.spaceConsumed(spaceConsumed);
+ return this;
+ }
+
+ @Override
+ public Builder spaceQuota(long spaceQuota) {
+ super.spaceQuota(spaceQuota);
+ return this;
+ }
+ }
+
+ /**
+ * Verify if namespace quota is violated once quota is set. Relevant
+ * method {@link DirectoryWithQuotaFeature#verifyNamespaceQuota}.
+ * @throws NSQuotaExceededException
+ */
+ public void verifyNamespaceQuota() throws NSQuotaExceededException {
+ if (Quota.isViolated(getQuota(), getFileAndDirectoryCount())) {
+ throw new NSQuotaExceededException(getQuota(),
+ getFileAndDirectoryCount());
+ }
+ }
+
+ /**
+ * Verify if storage space quota is violated once quota is set. Relevant
+ * method {@link DirectoryWithQuotaFeature#verifyStoragespaceQuota}.
+ * @throws DSQuotaExceededException
+ */
+ public void verifyStoragespaceQuota() throws DSQuotaExceededException {
+ if (Quota.isViolated(getSpaceQuota(), getSpaceConsumed())) {
+ throw new DSQuotaExceededException(getSpaceQuota(), getSpaceConsumed());
+ }
+ }
+
+ @Override
+ public String toString() {
+ String nsQuota = String.valueOf(getQuota());
+ String nsCount = String.valueOf(getFileAndDirectoryCount());
+ if (getQuota() == HdfsConstants.QUOTA_DONT_SET) {
+ nsQuota = "-";
+ nsCount = "-";
+ }
+
+ String ssQuota = StringUtils.byteDesc(getSpaceQuota());
+ String ssCount = StringUtils.byteDesc(getSpaceConsumed());
+ if (getSpaceQuota() == HdfsConstants.QUOTA_DONT_SET) {
+ ssQuota = "-";
+ ssCount = "-";
+ }
+
+ StringBuilder str = new StringBuilder();
+ str.append("[NsQuota: ").append(nsQuota).append("/")
+ .append(nsCount);
+ str.append(", SsQuota: ").append(ssQuota)
+ .append("/").append(ssCount)
+ .append("]");
+ return str.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
new file mode 100644
index 0000000..0d298ac
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
@@ -0,0 +1,1021 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.net.InetSocketAddress;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.NameNodeProxiesClient.ProxyAndInfo;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.io.retry.RetryPolicies;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.io.retry.RetryPolicy.RetryAction.RetryDecision;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.ipc.StandbyException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+/**
+ * A client proxy for Router -> NN communication using the NN ClientProtocol.
+ * <p>
+ * Provides routers to invoke remote ClientProtocol methods and handle
+ * retries/failover.
+ * <ul>
+ * <li>invokeSingle Make a single request to a single namespace
+ * <li>invokeSequential Make a sequential series of requests to multiple
+ * ordered namespaces until a condition is met.
+ * <li>invokeConcurrent Make concurrent requests to multiple namespaces and
+ * return all of the results.
+ * </ul>
+ * Also maintains a cached pool of connections to NNs. Connections are managed
+ * by the ConnectionManager and are unique to each user + NN. The size of the
+ * connection pool can be configured. Larger pools allow for more simultaneous
+ * requests to a single NN from a single user.
+ */
+public class RouterRpcClient {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RouterRpcClient.class);
+
+
+ /** Router identifier. */
+ private final String routerId;
+
+ /** Interface to identify the active NN for a nameservice or blockpool ID. */
+ private final ActiveNamenodeResolver namenodeResolver;
+
+ /** Connection pool to the Namenodes per user for performance. */
+ private final ConnectionManager connectionManager;
+ /** Service to run asynchronous calls. */
+ private final ExecutorService executorService;
+ /** Retry policy for router -> NN communication. */
+ private final RetryPolicy retryPolicy;
+ /** Optional perf monitor. */
+ private final RouterRpcMonitor rpcMonitor;
+
+ /** Pattern to parse a stack trace line. */
+ private static final Pattern STACK_TRACE_PATTERN =
+ Pattern.compile("\\tat (.*)\\.(.*)\\((.*):(\\d*)\\)");
+
+
+ /**
+ * Create a router RPC client to manage remote procedure calls to NNs.
+ *
+ * @param conf Hdfs Configuation.
+ * @param resolver A NN resolver to determine the currently active NN in HA.
+ * @param monitor Optional performance monitor.
+ */
+ public RouterRpcClient(Configuration conf, String identifier,
+ ActiveNamenodeResolver resolver, RouterRpcMonitor monitor) {
+ this.routerId = identifier;
+
+ this.namenodeResolver = resolver;
+
+ this.connectionManager = new ConnectionManager(conf);
+ this.connectionManager.start();
+
+ int numThreads = conf.getInt(
+ RBFConfigKeys.DFS_ROUTER_CLIENT_THREADS_SIZE,
+ RBFConfigKeys.DFS_ROUTER_CLIENT_THREADS_SIZE_DEFAULT);
+ ThreadFactory threadFactory = new ThreadFactoryBuilder()
+ .setNameFormat("RPC Router Client-%d")
+ .build();
+ this.executorService = Executors.newFixedThreadPool(
+ numThreads, threadFactory);
+
+ this.rpcMonitor = monitor;
+
+ int maxFailoverAttempts = conf.getInt(
+ HdfsClientConfigKeys.Failover.MAX_ATTEMPTS_KEY,
+ HdfsClientConfigKeys.Failover.MAX_ATTEMPTS_DEFAULT);
+ int maxRetryAttempts = conf.getInt(
+ RBFConfigKeys.DFS_ROUTER_CLIENT_MAX_ATTEMPTS,
+ RBFConfigKeys.DFS_ROUTER_CLIENT_MAX_ATTEMPTS_DEFAULT);
+ int failoverSleepBaseMillis = conf.getInt(
+ HdfsClientConfigKeys.Failover.SLEEPTIME_BASE_KEY,
+ HdfsClientConfigKeys.Failover.SLEEPTIME_BASE_DEFAULT);
+ int failoverSleepMaxMillis = conf.getInt(
+ HdfsClientConfigKeys.Failover.SLEEPTIME_MAX_KEY,
+ HdfsClientConfigKeys.Failover.SLEEPTIME_MAX_DEFAULT);
+ this.retryPolicy = RetryPolicies.failoverOnNetworkException(
+ RetryPolicies.TRY_ONCE_THEN_FAIL, maxFailoverAttempts, maxRetryAttempts,
+ failoverSleepBaseMillis, failoverSleepMaxMillis);
+ }
+
+ /**
+ * Get the active namenode resolver used by this client.
+ * @return Active namenode resolver.
+ */
+ public ActiveNamenodeResolver getNamenodeResolver() {
+ return this.namenodeResolver;
+ }
+
+ /**
+ * Shutdown the client.
+ */
+ public void shutdown() {
+ if (this.connectionManager != null) {
+ this.connectionManager.close();
+ }
+ if (this.executorService != null) {
+ this.executorService.shutdownNow();
+ }
+ }
+
+ /**
+ * Total number of available sockets between the router and NNs.
+ *
+ * @return Number of namenode clients.
+ */
+ public int getNumConnections() {
+ return this.connectionManager.getNumConnections();
+ }
+
+ /**
+ * Total number of available sockets between the router and NNs.
+ *
+ * @return Number of namenode clients.
+ */
+ public int getNumActiveConnections() {
+ return this.connectionManager.getNumActiveConnections();
+ }
+
+ /**
+ * Total number of open connection pools to a NN. Each connection pool.
+ * represents one user + one NN.
+ *
+ * @return Number of connection pools.
+ */
+ public int getNumConnectionPools() {
+ return this.connectionManager.getNumConnectionPools();
+ }
+
+ /**
+ * Number of connections between the router and NNs being created sockets.
+ *
+ * @return Number of connections waiting to be created.
+ */
+ public int getNumCreatingConnections() {
+ return this.connectionManager.getNumCreatingConnections();
+ }
+
+ /**
+ * JSON representation of the connection pool.
+ *
+ * @return String representation of the JSON.
+ */
+ public String getJSON() {
+ return this.connectionManager.getJSON();
+ }
+
+ /**
+ * Get ClientProtocol proxy client for a NameNode. Each combination of user +
+ * NN must use a unique proxy client. Previously created clients are cached
+ * and stored in a connection pool by the ConnectionManager.
+ *
+ * @param ugi User group information.
+ * @param nsId Nameservice identifier.
+ * @param rpcAddress ClientProtocol RPC server address of the NN.
+ * @return ConnectionContext containing a ClientProtocol proxy client for the
+ * NN + current user.
+ * @throws IOException If we cannot get a connection to the NameNode.
+ */
+ private ConnectionContext getConnection(
+ UserGroupInformation ugi, String nsId, String rpcAddress)
+ throws IOException {
+ ConnectionContext connection = null;
+ try {
+ // Each proxy holds the UGI info for the current user when it is created.
+ // This cache does not scale very well, one entry per user per namenode,
+ // and may need to be adjusted and/or selectively pruned. The cache is
+ // important due to the excessive overhead of creating a new proxy wrapper
+ // for each individual request.
+
+ // TODO Add tokens from the federated UGI
+ connection = this.connectionManager.getConnection(ugi, rpcAddress);
+ LOG.debug("User {} NN {} is using connection {}",
+ ugi.getUserName(), rpcAddress, connection);
+ } catch (Exception ex) {
+ LOG.error("Cannot open NN client to address: {}", rpcAddress, ex);
+ }
+
+ if (connection == null) {
+ throw new IOException("Cannot get a connection to " + rpcAddress);
+ }
+ return connection;
+ }
+
+ /**
+ * Convert an exception to an IOException.
+ *
+ * For a non-IOException, wrap it with IOException. For a RemoteException,
+ * unwrap it. For an IOException which is not a RemoteException, return it.
+ *
+ * @param e Exception to convert into an exception.
+ * @return Created IO exception.
+ */
+ private static IOException toIOException(Exception e) {
+ if (e instanceof RemoteException) {
+ return ((RemoteException) e).unwrapRemoteException();
+ }
+ if (e instanceof IOException) {
+ return (IOException)e;
+ }
+ return new IOException(e);
+ }
+
+ /**
+ * If we should retry the RPC call.
+ *
+ * @param ioe IOException reported.
+ * @param retryCount Number of retries.
+ * @param nsId Nameservice ID.
+ * @return Retry decision.
+ * @throws IOException Original exception if the retry policy generates one
+ * or IOException for no available namenodes.
+ */
+ private RetryDecision shouldRetry(final IOException ioe, final int retryCount,
+ final String nsId) throws IOException {
+ // check for the case of cluster unavailable state
+ if (isClusterUnAvailable(nsId)) {
+ // we allow to retry once if cluster is unavailable
+ if (retryCount == 0) {
+ return RetryDecision.RETRY;
+ } else {
+ throw new IOException("No namenode available under nameservice " + nsId,
+ ioe);
+ }
+ }
+
+ try {
+ final RetryPolicy.RetryAction a =
+ this.retryPolicy.shouldRetry(ioe, retryCount, 0, true);
+ return a.action;
+ } catch (Exception ex) {
+ LOG.error("Re-throwing API exception, no more retries", ex);
+ throw toIOException(ex);
+ }
+ }
+
+ /**
+ * Invokes a method against the ClientProtocol proxy server. If a standby
+ * exception is generated by the call to the client, retries using the
+ * alternate server.
+ *
+ * Re-throws exceptions generated by the remote RPC call as either
+ * RemoteException or IOException.
+ *
+ * @param ugi User group information.
+ * @param namenodes A prioritized list of namenodes within the same
+ * nameservice.
+ * @param method Remote ClientProtcol method to invoke.
+ * @param params Variable list of parameters matching the method.
+ * @return The result of invoking the method.
+ * @throws IOException
+ */
+ private Object invokeMethod(
+ final UserGroupInformation ugi,
+ final List<? extends FederationNamenodeContext> namenodes,
+ final Method method, final Object... params) throws IOException {
+
+ if (namenodes == null || namenodes.isEmpty()) {
+ throw new IOException("No namenodes to invoke " + method.getName() +
+ " with params " + Arrays.toString(params) + " from " + this.routerId);
+ }
+
+ Object ret = null;
+ if (rpcMonitor != null) {
+ rpcMonitor.proxyOp();
+ }
+ boolean failover = false;
+ Map<FederationNamenodeContext, IOException> ioes = new LinkedHashMap<>();
+ for (FederationNamenodeContext namenode : namenodes) {
+ ConnectionContext connection = null;
+ try {
+ String nsId = namenode.getNameserviceId();
+ String rpcAddress = namenode.getRpcAddress();
+ connection = this.getConnection(ugi, nsId, rpcAddress);
+ ProxyAndInfo<ClientProtocol> client = connection.getClient();
+ ClientProtocol proxy = client.getProxy();
+ ret = invoke(nsId, 0, method, proxy, params);
+ if (failover) {
+ // Success on alternate server, update
+ InetSocketAddress address = client.getAddress();
+ namenodeResolver.updateActiveNamenode(nsId, address);
+ }
+ if (this.rpcMonitor != null) {
+ this.rpcMonitor.proxyOpComplete(true);
+ }
+ return ret;
+ } catch (IOException ioe) {
+ ioes.put(namenode, ioe);
+ if (ioe instanceof StandbyException) {
+ // Fail over indicated by retry policy and/or NN
+ if (this.rpcMonitor != null) {
+ this.rpcMonitor.proxyOpFailureStandby();
+ }
+ failover = true;
+ } else if (ioe instanceof RemoteException) {
+ if (this.rpcMonitor != null) {
+ this.rpcMonitor.proxyOpComplete(true);
+ }
+ // RemoteException returned by NN
+ throw (RemoteException) ioe;
+ } else {
+ // Other communication error, this is a failure
+ // Communication retries are handled by the retry policy
+ if (this.rpcMonitor != null) {
+ this.rpcMonitor.proxyOpFailureCommunicate();
+ this.rpcMonitor.proxyOpComplete(false);
+ }
+ throw ioe;
+ }
+ } finally {
+ if (connection != null) {
+ connection.release();
+ }
+ }
+ }
+ if (this.rpcMonitor != null) {
+ this.rpcMonitor.proxyOpComplete(false);
+ }
+
+ // All namenodes were unavailable or in standby
+ String msg = "No namenode available to invoke " + method.getName() + " " +
+ Arrays.toString(params);
+ LOG.error(msg);
+ for (Entry<FederationNamenodeContext, IOException> entry :
+ ioes.entrySet()) {
+ FederationNamenodeContext namenode = entry.getKey();
+ String nsId = namenode.getNameserviceId();
+ String nnId = namenode.getNamenodeId();
+ String addr = namenode.getRpcAddress();
+ IOException ioe = entry.getValue();
+ if (ioe instanceof StandbyException) {
+ LOG.error("{} {} at {} is in Standby", nsId, nnId, addr);
+ } else {
+ LOG.error("{} {} at {} error: \"{}\"",
+ nsId, nnId, addr, ioe.getMessage());
+ }
+ }
+ throw new StandbyException(msg);
+ }
+
+ /**
+ * Invokes a method on the designated object. Catches exceptions specific to
+ * the invocation.
+ *
+ * Re-throws exceptions generated by the remote RPC call as either
+ * RemoteException or IOException.
+ *
+ * @param nsId Identifier for the namespace
+ * @param retryCount Current retry times
+ * @param method Method to invoke
+ * @param obj Target object for the method
+ * @param params Variable parameters
+ * @return Response from the remote server
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ private Object invoke(String nsId, int retryCount, final Method method,
+ final Object obj, final Object... params) throws IOException {
+ try {
+ return method.invoke(obj, params);
+ } catch (IllegalAccessException e) {
+ LOG.error("Unexpected exception while proxying API", e);
+ return null;
+ } catch (IllegalArgumentException e) {
+ LOG.error("Unexpected exception while proxying API", e);
+ return null;
+ } catch (InvocationTargetException e) {
+ Throwable cause = e.getCause();
+ if (cause instanceof IOException) {
+ IOException ioe = (IOException) cause;
+
+ // Check if we should retry.
+ RetryDecision decision = shouldRetry(ioe, retryCount, nsId);
+ if (decision == RetryDecision.RETRY) {
+ if (this.rpcMonitor != null) {
+ this.rpcMonitor.proxyOpRetries();
+ }
+
+ // retry
+ return invoke(nsId, ++retryCount, method, obj, params);
+ } else if (decision == RetryDecision.FAILOVER_AND_RETRY) {
+ // failover, invoker looks for standby exceptions for failover.
+ if (ioe instanceof StandbyException) {
+ throw ioe;
+ } else {
+ throw new StandbyException(ioe.getMessage());
+ }
+ } else {
+ if (ioe instanceof RemoteException) {
+ RemoteException re = (RemoteException) ioe;
+ ioe = re.unwrapRemoteException();
+ ioe = getCleanException(ioe);
+ }
+ throw ioe;
+ }
+ } else {
+ throw new IOException(e);
+ }
+ }
+ }
+
+ /**
+ * Check if the cluster of given nameservice id is available.
+ * @param nsId nameservice ID.
+ * @return
+ * @throws IOException
+ */
+ private boolean isClusterUnAvailable(String nsId) throws IOException {
+ List<? extends FederationNamenodeContext> nnState = this.namenodeResolver
+ .getNamenodesForNameserviceId(nsId);
+
+ if (nnState != null) {
+ for (FederationNamenodeContext nnContext : nnState) {
+ // Once we find one NN is in active state, we assume this
+ // cluster is available.
+ if (nnContext.getState() == FederationNamenodeServiceState.ACTIVE) {
+ return false;
+ }
+ }
+ }
+
+ return true;
+ }
+
+ /**
+ * Get a clean copy of the exception. Sometimes the exceptions returned by the
+ * server contain the full stack trace in the message.
+ *
+ * @param ioe Exception to clean up.
+ * @return Copy of the original exception with a clean message.
+ */
+ private static IOException getCleanException(IOException ioe) {
+ IOException ret = null;
+
+ String msg = ioe.getMessage();
+ Throwable cause = ioe.getCause();
+ StackTraceElement[] stackTrace = ioe.getStackTrace();
+
+ // Clean the message by removing the stack trace
+ int index = msg.indexOf("\n");
+ if (index > 0) {
+ String[] msgSplit = msg.split("\n");
+ msg = msgSplit[0];
+
+ // Parse stack trace from the message
+ List<StackTraceElement> elements = new LinkedList<>();
+ for (int i=1; i<msgSplit.length; i++) {
+ String line = msgSplit[i];
+ Matcher matcher = STACK_TRACE_PATTERN.matcher(line);
+ if (matcher.find()) {
+ String declaringClass = matcher.group(1);
+ String methodName = matcher.group(2);
+ String fileName = matcher.group(3);
+ int lineNumber = Integer.parseInt(matcher.group(4));
+ StackTraceElement element = new StackTraceElement(
+ declaringClass, methodName, fileName, lineNumber);
+ elements.add(element);
+ }
+ }
+ stackTrace = elements.toArray(new StackTraceElement[elements.size()]);
+ }
+
+ // Create the new output exception
+ if (ioe instanceof RemoteException) {
+ RemoteException re = (RemoteException)ioe;
+ ret = new RemoteException(re.getClassName(), msg);
+ } else {
+ // Try the simple constructor and initialize the fields
+ Class<? extends IOException> ioeClass = ioe.getClass();
+ try {
+ Constructor<? extends IOException> constructor =
+ ioeClass.getDeclaredConstructor(String.class);
+ ret = constructor.newInstance(msg);
+ } catch (ReflectiveOperationException e) {
+ // If there are errors, just use the input one
+ LOG.error("Could not create exception {}", ioeClass.getSimpleName(), e);
+ ret = ioe;
+ }
+ }
+ if (ret != null) {
+ ret.initCause(cause);
+ ret.setStackTrace(stackTrace);
+ }
+
+ return ret;
+ }
+
+ /**
+ * Invokes a ClientProtocol method. Determines the target nameservice via a
+ * provided block.
+ *
+ * Re-throws exceptions generated by the remote RPC call as either
+ * RemoteException or IOException.
+ *
+ * @param block Block used to determine appropriate nameservice.
+ * @param method The remote method and parameters to invoke.
+ * @return The result of invoking the method.
+ * @throws IOException
+ */
+ public Object invokeSingle(final ExtendedBlock block, RemoteMethod method)
+ throws IOException {
+ String bpId = block.getBlockPoolId();
+ return invokeSingleBlockPool(bpId, method);
+ }
+
+ /**
+ * Invokes a ClientProtocol method. Determines the target nameservice using
+ * the block pool id.
+ *
+ * Re-throws exceptions generated by the remote RPC call as either
+ * RemoteException or IOException.
+ *
+ * @param bpId Block pool identifier.
+ * @param method The remote method and parameters to invoke.
+ * @return The result of invoking the method.
+ * @throws IOException
+ */
+ public Object invokeSingleBlockPool(final String bpId, RemoteMethod method)
+ throws IOException {
+ String nsId = getNameserviceForBlockPoolId(bpId);
+ return invokeSingle(nsId, method);
+ }
+
+ /**
+ * Invokes a ClientProtocol method against the specified namespace.
+ *
+ * Re-throws exceptions generated by the remote RPC call as either
+ * RemoteException or IOException.
+ *
+ * @param nsId Target namespace for the method.
+ * @param method The remote method and parameters to invoke.
+ * @return The result of invoking the method.
+ * @throws IOException
+ */
+ public Object invokeSingle(final String nsId, RemoteMethod method)
+ throws IOException {
+ UserGroupInformation ugi = RouterRpcServer.getRemoteUser();
+ List<? extends FederationNamenodeContext> nns =
+ getNamenodesForNameservice(nsId);
+ RemoteLocationContext loc = new RemoteLocation(nsId, "/");
+ return invokeMethod(ugi, nns, method.getMethod(), method.getParams(loc));
+ }
+
+ /**
+ * Invokes a single proxy call for a single location.
+ *
+ * Re-throws exceptions generated by the remote RPC call as either
+ * RemoteException or IOException.
+ *
+ * @param location RemoteLocation to invoke.
+ * @param remoteMethod The remote method and parameters to invoke.
+ * @return The result of invoking the method if successful.
+ * @throws IOException
+ */
+ public Object invokeSingle(final RemoteLocationContext location,
+ RemoteMethod remoteMethod) throws IOException {
+ List<RemoteLocationContext> locations = Collections.singletonList(location);
+ return invokeSequential(locations, remoteMethod);
+ }
+
+ /**
+ * Invokes sequential proxy calls to different locations. Continues to invoke
+ * calls until a call returns without throwing a remote exception.
+ *
+ * @param locations List of locations/nameservices to call concurrently.
+ * @param remoteMethod The remote method and parameters to invoke.
+ * @return The result of the first successful call, or if no calls are
+ * successful, the result of the last RPC call executed.
+ * @throws IOException if the success condition is not met and one of the RPC
+ * calls generated a remote exception.
+ */
+ public Object invokeSequential(
+ final List<? extends RemoteLocationContext> locations,
+ final RemoteMethod remoteMethod) throws IOException {
+ return invokeSequential(locations, remoteMethod, null, null);
+ }
+
+ /**
+ * Invokes sequential proxy calls to different locations. Continues to invoke
+ * calls until the success condition is met, or until all locations have been
+ * attempted.
+ *
+ * The success condition may be specified by:
+ * <ul>
+ * <li>An expected result class
+ * <li>An expected result value
+ * </ul>
+ *
+ * If no expected result class/values are specified, the success condition is
+ * a call that does not throw a remote exception.
+ *
+ * @param locations List of locations/nameservices to call concurrently.
+ * @param remoteMethod The remote method and parameters to invoke.
+ * @param expectedResultClass In order to be considered a positive result, the
+ * return type must be of this class.
+ * @param expectedResultValue In order to be considered a positive result, the
+ * return value must equal the value of this object.
+ * @return The result of the first successful call, or if no calls are
+ * successful, the result of the first RPC call executed.
+ * @throws IOException if the success condition is not met, return the first
+ * remote exception generated.
+ */
+ public <T> T invokeSequential(
+ final List<? extends RemoteLocationContext> locations,
+ final RemoteMethod remoteMethod, Class<T> expectedResultClass,
+ Object expectedResultValue) throws IOException {
+
+ final UserGroupInformation ugi = RouterRpcServer.getRemoteUser();
+ final Method m = remoteMethod.getMethod();
+ IOException firstThrownException = null;
+ IOException lastThrownException = null;
+ Object firstResult = null;
+ // Invoke in priority order
+ for (final RemoteLocationContext loc : locations) {
+ String ns = loc.getNameserviceId();
+ List<? extends FederationNamenodeContext> namenodes =
+ getNamenodesForNameservice(ns);
+ try {
+ Object[] params = remoteMethod.getParams(loc);
+ Object result = invokeMethod(ugi, namenodes, m, params);
+ // Check if the result is what we expected
+ if (isExpectedClass(expectedResultClass, result) &&
+ isExpectedValue(expectedResultValue, result)) {
+ // Valid result, stop here
+ @SuppressWarnings("unchecked")
+ T ret = (T)result;
+ return ret;
+ }
+ if (firstResult == null) {
+ firstResult = result;
+ }
+ } catch (IOException ioe) {
+ // Record it and move on
+ lastThrownException = (IOException) ioe;
+ if (firstThrownException == null) {
+ firstThrownException = lastThrownException;
+ }
+ } catch (Exception e) {
+ // Unusual error, ClientProtocol calls always use IOException (or
+ // RemoteException). Re-wrap in IOException for compatibility with
+ // ClientProtcol.
+ LOG.error("Unexpected exception {} proxying {} to {}",
+ e.getClass(), m.getName(), ns, e);
+ lastThrownException = new IOException(
+ "Unexpected exception proxying API " + e.getMessage(), e);
+ if (firstThrownException == null) {
+ firstThrownException = lastThrownException;
+ }
+ }
+ }
+
+ if (firstThrownException != null) {
+ // re-throw the last exception thrown for compatibility
+ throw firstThrownException;
+ }
+ // Return the last result, whether it is the value we are looking for or a
+ @SuppressWarnings("unchecked")
+ T ret = (T)firstResult;
+ return ret;
+ }
+
+ /**
+ * Checks if a result matches the required result class.
+ *
+ * @param expectedClass Required result class, null to skip the check.
+ * @param clazz The result to check.
+ * @return True if the result is an instance of the required class or if the
+ * expected class is null.
+ */
+ private static boolean isExpectedClass(Class<?> expectedClass, Object clazz) {
+ if (expectedClass == null) {
+ return true;
+ } else if (clazz == null) {
+ return false;
+ } else {
+ return expectedClass.isInstance(clazz);
+ }
+ }
+
+ /**
+ * Checks if a result matches the expected value.
+ *
+ * @param expectedValue The expected value, null to skip the check.
+ * @param value The result to check.
+ * @return True if the result is equals to the expected value or if the
+ * expected value is null.
+ */
+ private static boolean isExpectedValue(Object expectedValue, Object value) {
+ if (expectedValue == null) {
+ return true;
+ } else if (value == null) {
+ return false;
+ } else {
+ return value.equals(expectedValue);
+ }
+ }
+
+ /**
+ * Invoke multiple concurrent proxy calls to different clients. Returns an
+ * array of results.
+ *
+ * Re-throws exceptions generated by the remote RPC call as either
+ * RemoteException or IOException.
+ *
+ * @param <T> The type of the remote location.
+ * @param locations List of remote locations to call concurrently.
+ * @param method The remote method and parameters to invoke.
+ * @param requireResponse If true an exception will be thrown if all calls do
+ * not complete. If false exceptions are ignored and all data results
+ * successfully received are returned.
+ * @param standby If the requests should go to the standby namenodes too.
+ * @throws IOException If all the calls throw an exception.
+ */
+ public <T extends RemoteLocationContext, R> void invokeConcurrent(
+ final Collection<T> locations, final RemoteMethod method,
+ boolean requireResponse, boolean standby) throws IOException {
+ invokeConcurrent(locations, method, requireResponse, standby, void.class);
+ }
+
+ /**
+ * Invokes multiple concurrent proxy calls to different clients. Returns an
+ * array of results.
+ *
+ * Re-throws exceptions generated by the remote RPC call as either
+ * RemoteException or IOException.
+ *
+ * @param <T> The type of the remote location.
+ * @param <R> The type of the remote method return.
+ * @param locations List of remote locations to call concurrently.
+ * @param method The remote method and parameters to invoke.
+ * @param requireResponse If true an exception will be thrown if all calls do
+ * not complete. If false exceptions are ignored and all data results
+ * successfully received are returned.
+ * @param standby If the requests should go to the standby namenodes too.
+ * @param clazz Type of the remote return type.
+ * @return Result of invoking the method per subcluster: nsId -> result.
+ * @throws IOException If requiredResponse=true and any of the calls throw an
+ * exception.
+ */
+ public <T extends RemoteLocationContext, R> Map<T, R> invokeConcurrent(
+ final Collection<T> locations, final RemoteMethod method,
+ boolean requireResponse, boolean standby, Class<R> clazz)
+ throws IOException {
+ return invokeConcurrent(
+ locations, method, requireResponse, standby, -1, clazz);
+ }
+
+ /**
+ * Invokes multiple concurrent proxy calls to different clients. Returns an
+ * array of results.
+ *
+ * Re-throws exceptions generated by the remote RPC call as either
+ * RemoteException or IOException.
+ *
+ * @param <T> The type of the remote location.
+ * @param <R> The type of the remote method return.
+ * @param locations List of remote locations to call concurrently.
+ * @param method The remote method and parameters to invoke.
+ * @param requireResponse If true an exception will be thrown if all calls do
+ * not complete. If false exceptions are ignored and all data results
+ * successfully received are returned.
+ * @param standby If the requests should go to the standby namenodes too.
+ * @param timeOutMs Timeout for each individual call.
+ * @param clazz Type of the remote return type.
+ * @return Result of invoking the method per subcluster: nsId -> result.
+ * @throws IOException If requiredResponse=true and any of the calls throw an
+ * exception.
+ */
+ @SuppressWarnings("unchecked")
+ public <T extends RemoteLocationContext, R> Map<T, R> invokeConcurrent(
+ final Collection<T> locations, final RemoteMethod method,
+ boolean requireResponse, boolean standby, long timeOutMs, Class<R> clazz)
+ throws IOException {
+
+ final UserGroupInformation ugi = RouterRpcServer.getRemoteUser();
+ final Method m = method.getMethod();
+
+ if (locations.size() == 1) {
+ // Shortcut, just one call
+ T location = locations.iterator().next();
+ String ns = location.getNameserviceId();
+ final List<? extends FederationNamenodeContext> namenodes =
+ getNamenodesForNameservice(ns);
+ Object[] paramList = method.getParams(location);
+ Object result = invokeMethod(ugi, namenodes, m, paramList);
+ return Collections.singletonMap(location, clazz.cast(result));
+ }
+
+ List<T> orderedLocations = new LinkedList<>();
+ Set<Callable<Object>> callables = new HashSet<>();
+ for (final T location : locations) {
+ String nsId = location.getNameserviceId();
+ final List<? extends FederationNamenodeContext> namenodes =
+ getNamenodesForNameservice(nsId);
+ final Object[] paramList = method.getParams(location);
+ if (standby) {
+ // Call the objectGetter to all NNs (including standby)
+ for (final FederationNamenodeContext nn : namenodes) {
+ String nnId = nn.getNamenodeId();
+ final List<FederationNamenodeContext> nnList =
+ Collections.singletonList(nn);
+ T nnLocation = location;
+ if (location instanceof RemoteLocation) {
+ nnLocation = (T)new RemoteLocation(nsId, nnId, location.getDest());
+ }
+ orderedLocations.add(nnLocation);
+ callables.add(new Callable<Object>() {
+ public Object call() throws Exception {
+ return invokeMethod(ugi, nnList, m, paramList);
+ }
+ });
+ }
+ } else {
+ // Call the objectGetter in order of nameservices in the NS list
+ orderedLocations.add(location);
+ callables.add(new Callable<Object>() {
+ public Object call() throws Exception {
+ return invokeMethod(ugi, namenodes, m, paramList);
+ }
+ });
+ }
+ }
+
+ if (rpcMonitor != null) {
+ rpcMonitor.proxyOp();
+ }
+
+ try {
+ List<Future<Object>> futures = null;
+ if (timeOutMs > 0) {
+ futures = executorService.invokeAll(
+ callables, timeOutMs, TimeUnit.MILLISECONDS);
+ } else {
+ futures = executorService.invokeAll(callables);
+ }
+ Map<T, R> results = new TreeMap<>();
+ Map<T, IOException> exceptions = new TreeMap<>();
+ for (int i=0; i<futures.size(); i++) {
+ T location = orderedLocations.get(i);
+ try {
+ Future<Object> future = futures.get(i);
+ Object result = future.get();
+ results.put(location, clazz.cast(result));
+ } catch (CancellationException ce) {
+ T loc = orderedLocations.get(i);
+ String msg =
+ "Invocation to \"" + loc + "\" for \"" + method + "\" timed out";
+ LOG.error(msg);
+ IOException ioe = new IOException(msg);
+ exceptions.put(location, ioe);
+ } catch (ExecutionException ex) {
+ Throwable cause = ex.getCause();
+ LOG.debug("Canot execute {} in {}: {}",
+ m.getName(), location, cause.getMessage());
+
+ // Convert into IOException if needed
+ IOException ioe = null;
+ if (cause instanceof IOException) {
+ ioe = (IOException) cause;
+ } else {
+ ioe = new IOException("Unhandled exception while proxying API " +
+ m.getName() + ": " + cause.getMessage(), cause);
+ }
+
+ // Response from all servers required, use this error.
+ if (requireResponse) {
+ throw ioe;
+ }
+
+ // Store the exceptions
+ exceptions.put(location, ioe);
+ }
+ }
+
+ // Throw the exception for the first location if there are no results
+ if (results.isEmpty()) {
+ T location = orderedLocations.get(0);
+ IOException ioe = exceptions.get(location);
+ if (ioe != null) {
+ throw ioe;
+ }
+ }
+
+ return results;
+ } catch (InterruptedException ex) {
+ LOG.error("Unexpected error while invoking API: {}", ex.getMessage());
+ throw new IOException(
+ "Unexpected error while invoking API " + ex.getMessage(), ex);
+ }
+ }
+
+ /**
+ * Get a prioritized list of NNs that share the same nameservice ID (in the
+ * same namespace). NNs that are reported as ACTIVE will be first in the list.
+ *
+ * @param nsId The nameservice ID for the namespace.
+ * @return A prioritized list of NNs to use for communication.
+ * @throws IOException If a NN cannot be located for the nameservice ID.
+ */
+ private List<? extends FederationNamenodeContext> getNamenodesForNameservice(
+ final String nsId) throws IOException {
+
+ final List<? extends FederationNamenodeContext> namenodes =
+ namenodeResolver.getNamenodesForNameserviceId(nsId);
+
+ if (namenodes == null || namenodes.isEmpty()) {
+ throw new IOException("Cannot locate a registered namenode for " + nsId +
+ " from " + this.routerId);
+ }
+ return namenodes;
+ }
+
+ /**
+ * Get a prioritized list of NNs that share the same block pool ID (in the
+ * same namespace). NNs that are reported as ACTIVE will be first in the list.
+ *
+ * @param bpId The blockpool ID for the namespace.
+ * @return A prioritized list of NNs to use for communication.
+ * @throws IOException If a NN cannot be located for the block pool ID.
+ */
+ private List<? extends FederationNamenodeContext> getNamenodesForBlockPoolId(
+ final String bpId) throws IOException {
+
+ List<? extends FederationNamenodeContext> namenodes =
+ namenodeResolver.getNamenodesForBlockPoolId(bpId);
+
+ if (namenodes == null || namenodes.isEmpty()) {
+ throw new IOException("Cannot locate a registered namenode for " + bpId +
+ " from " + this.routerId);
+ }
+ return namenodes;
+ }
+
+ /**
+ * Get the nameservice identifier for a block pool.
+ *
+ * @param bpId Identifier of the block pool.
+ * @return Nameservice identifier.
+ * @throws IOException If a NN cannot be located for the block pool ID.
+ */
+ private String getNameserviceForBlockPoolId(final String bpId)
+ throws IOException {
+ List<? extends FederationNamenodeContext> namenodes =
+ getNamenodesForBlockPoolId(bpId);
+ FederationNamenodeContext namenode = namenodes.get(0);
+ return namenode.getNameserviceId();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcMonitor.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcMonitor.java
new file mode 100644
index 0000000..df9aa11
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcMonitor.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCMetrics;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+
+/**
+ * Metrics and monitoring interface for the router RPC server. Allows pluggable
+ * diagnostics and monitoring services to be attached.
+ */
+public interface RouterRpcMonitor {
+
+ /**
+ * Initialize the monitor.
+ * @param conf Configuration for the monitor.
+ * @param server RPC server.
+ * @param store State Store.
+ */
+ void init(
+ Configuration conf, RouterRpcServer server, StateStoreService store);
+
+ /**
+ * Get Router RPC metrics info.
+ * @return The instance of FederationRPCMetrics.
+ */
+ FederationRPCMetrics getRPCMetrics();
+
+ /**
+ * Close the monitor.
+ */
+ void close();
+
+ /**
+ * Start processing an operation on the Router.
+ */
+ void startOp();
+
+ /**
+ * Start proxying an operation to the Namenode.
+ * @return Id of the thread doing the proxying.
+ */
+ long proxyOp();
+
+ /**
+ * Mark a proxy operation as completed.
+ * @param success If the operation was successful.
+ */
+ void proxyOpComplete(boolean success);
+
+ /**
+ * Failed to proxy an operation to a Namenode because it was in standby.
+ */
+ void proxyOpFailureStandby();
+
+ /**
+ * Failed to proxy an operation to a Namenode because of an unexpected
+ * exception.
+ */
+ void proxyOpFailureCommunicate();
+
+ /**
+ * Failed to proxy an operation because it is not implemented.
+ */
+ void proxyOpNotImplemented();
+
+ /**
+ * Retry to proxy an operation to a Namenode because of an unexpected
+ * exception.
+ */
+ void proxyOpRetries();
+
+ /**
+ * If the Router cannot contact the State Store in an operation.
+ */
+ void routerFailureStateStore();
+
+ /**
+ * If the Router is in safe mode.
+ */
+ void routerFailureSafemode();
+
+ /**
+ * If a path is locked.
+ */
+ void routerFailureLocked();
+
+ /**
+ * If a path is in a read only mount point.
+ */
+ void routerFailureReadOnly();
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntryResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntryResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntryResponse.java
deleted file mode 100644
index 9bc7f92..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntryResponse.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-
-/**
- * API response for adding a mount table entry to the state store.
- */
-public abstract class AddMountTableEntryResponse {
-
- public static AddMountTableEntryResponse newInstance() throws IOException {
- return StateStoreSerializer.newRecord(AddMountTableEntryResponse.class);
- }
-
- @Public
- @Unstable
- public abstract boolean getStatus();
-
- @Public
- @Unstable
- public abstract void setStatus(boolean result);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/EnterSafeModeRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/EnterSafeModeRequest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/EnterSafeModeRequest.java
deleted file mode 100644
index 89df1bb..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/EnterSafeModeRequest.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-
-/**
- * API request for the Router entering safe mode state and updating
- * its state in State Store.
- */
-public abstract class EnterSafeModeRequest {
- public static EnterSafeModeRequest newInstance() throws IOException {
- return StateStoreSerializer.newRecord(EnterSafeModeRequest.class);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/EnterSafeModeResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/EnterSafeModeResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/EnterSafeModeResponse.java
deleted file mode 100644
index 4c994f6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/EnterSafeModeResponse.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-
-/**
- * API response for the Router entering safe mode state and updating
- * its state in State Store.
- */
-public abstract class EnterSafeModeResponse {
-
- public static EnterSafeModeResponse newInstance() throws IOException {
- return StateStoreSerializer.newRecord(EnterSafeModeResponse.class);
- }
-
- public static EnterSafeModeResponse newInstance(boolean status)
- throws IOException {
- EnterSafeModeResponse response = newInstance();
- response.setStatus(status);
- return response;
- }
-
- @Public
- @Unstable
- public abstract boolean getStatus();
-
- @Public
- @Unstable
- public abstract void setStatus(boolean result);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetMountTableEntriesRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetMountTableEntriesRequest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetMountTableEntriesRequest.java
deleted file mode 100644
index cd6c278..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetMountTableEntriesRequest.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-
-/**
- * API request for listing mount table entries present in the state store.
- */
-public abstract class GetMountTableEntriesRequest {
-
- public static GetMountTableEntriesRequest newInstance() throws IOException {
- return StateStoreSerializer.newRecord(GetMountTableEntriesRequest.class);
- }
-
- public static GetMountTableEntriesRequest newInstance(String srcPath)
- throws IOException {
- GetMountTableEntriesRequest request = newInstance();
- request.setSrcPath(srcPath);
- return request;
- }
-
- @Public
- @Unstable
- public abstract String getSrcPath();
-
- @Public
- @Unstable
- public abstract void setSrcPath(String path);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetMountTableEntriesResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetMountTableEntriesResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetMountTableEntriesResponse.java
deleted file mode 100644
index cebc3f6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetMountTableEntriesResponse.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-
-/**
- * API response for listing mount table entries present in the state store.
- */
-public abstract class GetMountTableEntriesResponse {
-
- public static GetMountTableEntriesResponse newInstance() throws IOException {
- return StateStoreSerializer.newRecord(GetMountTableEntriesResponse.class);
- }
-
- @Public
- @Unstable
- public abstract List<MountTable> getEntries() throws IOException;
-
- @Public
- @Unstable
- public abstract void setEntries(List<MountTable> entries)
- throws IOException;
-
- @Public
- @Unstable
- public abstract long getTimestamp();
-
- @Public
- @Unstable
- public abstract void setTimestamp(long time);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamenodeRegistrationsRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamenodeRegistrationsRequest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamenodeRegistrationsRequest.java
deleted file mode 100644
index 568feaf..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamenodeRegistrationsRequest.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
-
-/**
- * API request for listing namenode registrations present in the state store.
- */
-public abstract class GetNamenodeRegistrationsRequest {
-
- public static GetNamenodeRegistrationsRequest newInstance()
- throws IOException {
- return StateStoreSerializer.newRecord(
- GetNamenodeRegistrationsRequest.class);
- }
-
- public static GetNamenodeRegistrationsRequest newInstance(
- MembershipState member) throws IOException {
- GetNamenodeRegistrationsRequest request = newInstance();
- request.setPartialMembership(member);
- return request;
- }
-
- @Public
- @Unstable
- public abstract MembershipState getPartialMembership();
-
- @Public
- @Unstable
- public abstract void setPartialMembership(MembershipState member);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamenodeRegistrationsResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamenodeRegistrationsResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamenodeRegistrationsResponse.java
deleted file mode 100644
index 0d60c90..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamenodeRegistrationsResponse.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
-
-/**
- * API response for listing namenode registrations present in the state store.
- */
-public abstract class GetNamenodeRegistrationsResponse {
-
- public static GetNamenodeRegistrationsResponse newInstance()
- throws IOException {
- return StateStoreSerializer.newRecord(
- GetNamenodeRegistrationsResponse.class);
- }
-
- public static GetNamenodeRegistrationsResponse newInstance(
- List<MembershipState> records) throws IOException {
- GetNamenodeRegistrationsResponse response = newInstance();
- response.setNamenodeMemberships(records);
- return response;
- }
-
- @Public
- @Unstable
- public abstract List<MembershipState> getNamenodeMemberships()
- throws IOException;
-
- @Public
- @Unstable
- public abstract void setNamenodeMemberships(
- List<MembershipState> records) throws IOException;
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamespaceInfoRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamespaceInfoRequest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamespaceInfoRequest.java
deleted file mode 100644
index b5cc01b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamespaceInfoRequest.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-
-/**
- * API response for listing HDFS namespaces present in the state store.
- */
-public abstract class GetNamespaceInfoRequest {
-
- public static GetNamespaceInfoRequest newInstance() {
- return StateStoreSerializer.newRecord(GetNamespaceInfoRequest.class);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamespaceInfoResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamespaceInfoResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamespaceInfoResponse.java
deleted file mode 100644
index f541453..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamespaceInfoResponse.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import java.io.IOException;
-import java.util.Set;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-
-/**
- * API response for listing HDFS namespaces present in the state store.
- */
-public abstract class GetNamespaceInfoResponse {
-
- public static GetNamespaceInfoResponse newInstance() {
- return StateStoreSerializer.newRecord(GetNamespaceInfoResponse.class);
- }
-
- public static GetNamespaceInfoResponse newInstance(
- Set<FederationNamespaceInfo> namespaces) throws IOException {
- GetNamespaceInfoResponse response = newInstance();
- response.setNamespaceInfo(namespaces);
- return response;
- }
-
- @Public
- @Unstable
- public abstract Set<FederationNamespaceInfo> getNamespaceInfo();
-
- @Public
- @Unstable
- public abstract void setNamespaceInfo(
- Set<FederationNamespaceInfo> namespaceInfo);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationRequest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationRequest.java
deleted file mode 100644
index 9ba5788..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationRequest.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-
-/**
- * API request for retrieving a single router registration present in the state
- * store.
- */
-public abstract class GetRouterRegistrationRequest {
-
- public static GetRouterRegistrationRequest newInstance() {
- return StateStoreSerializer.newRecord(GetRouterRegistrationRequest.class);
- }
-
- public static GetRouterRegistrationRequest newInstance(String routerId) {
- GetRouterRegistrationRequest request = newInstance();
- request.setRouterId(routerId);
- return request;
- }
-
- @Public
- @Unstable
- public abstract String getRouterId();
-
- @Public
- @Unstable
- public abstract void setRouterId(String routerId);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationResponse.java
deleted file mode 100644
index b569376..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationResponse.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
-
-/**
- * API response for retrieving a single router registration present in the state
- * store.
- */
-public abstract class GetRouterRegistrationResponse {
-
- public static GetRouterRegistrationResponse newInstance() throws IOException {
- return StateStoreSerializer.newRecord(GetRouterRegistrationResponse.class);
- }
-
- @Public
- @Unstable
- public abstract RouterState getRouter() throws IOException;
-
- @Public
- @Unstable
- public abstract void setRouter(RouterState router) throws IOException;
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsRequest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsRequest.java
deleted file mode 100644
index b70cccf..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsRequest.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-
-/**
- * API request for retrieving a all non-expired router registrations present in
- * the state store.
- */
-public abstract class GetRouterRegistrationsRequest {
-
- public static GetRouterRegistrationsRequest newInstance() throws IOException {
- return StateStoreSerializer.newRecord(GetRouterRegistrationsRequest.class);
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsResponse.java
deleted file mode 100644
index 94e35c6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsResponse.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
-
-/**
- * API response for retrieving a all non-expired router registrations present in
- * the state store.
- */
-public abstract class GetRouterRegistrationsResponse {
-
- public static GetRouterRegistrationsResponse newInstance()
- throws IOException {
- return StateStoreSerializer.newRecord(GetRouterRegistrationsResponse.class);
- }
-
- @Public
- @Unstable
- public abstract List<RouterState> getRouters() throws IOException;
-
- @Public
- @Unstable
- public abstract void setRouters(List<RouterState> routers)
- throws IOException;
-
- @Public
- @Unstable
- public abstract long getTimestamp();
-
- @Public
- @Unstable
- public abstract void setTimestamp(long time);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetSafeModeRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetSafeModeRequest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetSafeModeRequest.java
deleted file mode 100644
index 00a8986..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetSafeModeRequest.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-
-/**
- * API request for verifying if current Router state is safe mode.
- */
-public abstract class GetSafeModeRequest {
- public static GetSafeModeRequest newInstance() throws IOException {
- return StateStoreSerializer.newRecord(GetSafeModeRequest.class);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetSafeModeResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetSafeModeResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetSafeModeResponse.java
deleted file mode 100644
index 0d7e229..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetSafeModeResponse.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-
-/**
- * API response for verifying if current Router state is safe mode.
- */
-public abstract class GetSafeModeResponse {
-
- public static GetSafeModeResponse newInstance() throws IOException {
- return StateStoreSerializer.newRecord(GetSafeModeResponse.class);
- }
-
- public static GetSafeModeResponse newInstance(boolean isInSafeMode)
- throws IOException {
- GetSafeModeResponse response = newInstance();
- response.setSafeMode(isInSafeMode);
- return response;
- }
-
- @Public
- @Unstable
- public abstract boolean isInSafeMode();
-
- @Public
- @Unstable
- public abstract void setSafeMode(boolean isInSafeMode);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/LeaveSafeModeRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/LeaveSafeModeRequest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/LeaveSafeModeRequest.java
deleted file mode 100644
index 2d64a1e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/LeaveSafeModeRequest.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-
-/**
- * API request for the Router leaving safe mode state and updating
- * its state in State Store.
- */
-public abstract class LeaveSafeModeRequest {
- public static LeaveSafeModeRequest newInstance() throws IOException {
- return StateStoreSerializer.newRecord(LeaveSafeModeRequest.class);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/LeaveSafeModeResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/LeaveSafeModeResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/LeaveSafeModeResponse.java
deleted file mode 100644
index 555f4fd..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/LeaveSafeModeResponse.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-
-/**
- * API response for the Router leaving safe mode state and updating
- * its state in State Store.
- */
-public abstract class LeaveSafeModeResponse {
-
- public static LeaveSafeModeResponse newInstance() throws IOException {
- return StateStoreSerializer.newRecord(LeaveSafeModeResponse.class);
- }
-
- public static LeaveSafeModeResponse newInstance(boolean status)
- throws IOException {
- LeaveSafeModeResponse response = newInstance();
- response.setStatus(status);
- return response;
- }
-
- @Public
- @Unstable
- public abstract boolean getStatus();
-
- @Public
- @Unstable
- public abstract void setStatus(boolean result);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/NamenodeHeartbeatRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/NamenodeHeartbeatRequest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/NamenodeHeartbeatRequest.java
deleted file mode 100644
index 9506026..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/NamenodeHeartbeatRequest.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
-
-/**
- * API request for registering a namenode with the state store.
- */
-public abstract class NamenodeHeartbeatRequest {
-
- public static NamenodeHeartbeatRequest newInstance() throws IOException {
- return StateStoreSerializer.newRecord(NamenodeHeartbeatRequest.class);
- }
-
- public static NamenodeHeartbeatRequest newInstance(MembershipState namenode)
- throws IOException {
- NamenodeHeartbeatRequest request = newInstance();
- request.setNamenodeMembership(namenode);
- return request;
- }
-
- @Private
- @Unstable
- public abstract MembershipState getNamenodeMembership()
- throws IOException;
-
- @Private
- @Unstable
- public abstract void setNamenodeMembership(MembershipState report)
- throws IOException;
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/NamenodeHeartbeatResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/NamenodeHeartbeatResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/NamenodeHeartbeatResponse.java
deleted file mode 100644
index acb7a6f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/NamenodeHeartbeatResponse.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-
-/**
- * API response for registering a namenode with the state store.
- */
-public abstract class NamenodeHeartbeatResponse {
-
- public static NamenodeHeartbeatResponse newInstance() throws IOException {
- return StateStoreSerializer.newRecord(NamenodeHeartbeatResponse.class);
- }
-
- public static NamenodeHeartbeatResponse newInstance(boolean status)
- throws IOException {
- NamenodeHeartbeatResponse response = newInstance();
- response.setResult(status);
- return response;
- }
-
- @Private
- @Unstable
- public abstract boolean getResult();
-
- @Private
- @Unstable
- public abstract void setResult(boolean result);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntryRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntryRequest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntryRequest.java
deleted file mode 100644
index 642ee0d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntryRequest.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-
-/**
- * API request for removing a mount table path present in the state store.
- */
-public abstract class RemoveMountTableEntryRequest {
-
- public static RemoveMountTableEntryRequest newInstance() throws IOException {
- return StateStoreSerializer.newRecord(RemoveMountTableEntryRequest.class);
- }
-
- public static RemoveMountTableEntryRequest newInstance(String path)
- throws IOException {
- RemoveMountTableEntryRequest request = newInstance();
- request.setSrcPath(path);
- return request;
- }
-
- @Public
- @Unstable
- public abstract String getSrcPath();
-
- @Public
- @Unstable
- public abstract void setSrcPath(String path);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntryResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntryResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntryResponse.java
deleted file mode 100644
index 70f117d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntryResponse.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-
-/**
- * API response for removing a mount table path present in the state store.
- */
-public abstract class RemoveMountTableEntryResponse {
-
- public static RemoveMountTableEntryResponse newInstance() throws IOException {
- return StateStoreSerializer.newRecord(RemoveMountTableEntryResponse.class);
- }
-
- @Public
- @Unstable
- public abstract boolean getStatus();
-
- @Public
- @Unstable
- public abstract void setStatus(boolean result);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RouterHeartbeatRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RouterHeartbeatRequest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RouterHeartbeatRequest.java
deleted file mode 100644
index 97d637b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RouterHeartbeatRequest.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
-
-/**
- * API request for registering a router with the state store.
- */
-public abstract class RouterHeartbeatRequest {
-
- public static RouterHeartbeatRequest newInstance() throws IOException {
- return StateStoreSerializer.newRecord(RouterHeartbeatRequest.class);
- }
-
- public static RouterHeartbeatRequest newInstance(RouterState router)
- throws IOException {
- RouterHeartbeatRequest request = newInstance();
- request.setRouter(router);
- return request;
- }
-
- @Public
- @Unstable
- public abstract RouterState getRouter() throws IOException;
-
- @Public
- @Unstable
- public abstract void setRouter(RouterState routerState);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RouterHeartbeatResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RouterHeartbeatResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RouterHeartbeatResponse.java
deleted file mode 100644
index 5912e9f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RouterHeartbeatResponse.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-
-/**
- * API response for registering a router with the state store.
- */
-public abstract class RouterHeartbeatResponse {
-
- public static RouterHeartbeatResponse newInstance() throws IOException {
- return StateStoreSerializer.newRecord(RouterHeartbeatResponse.class);
- }
-
- public static RouterHeartbeatResponse newInstance(boolean status)
- throws IOException {
- RouterHeartbeatResponse response = newInstance();
- response.setStatus(status);
- return response;
- }
-
- @Public
- @Unstable
- public abstract boolean getStatus();
-
- @Public
- @Unstable
- public abstract void setStatus(boolean result);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateMountTableEntryRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateMountTableEntryRequest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateMountTableEntryRequest.java
deleted file mode 100644
index afd5128..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateMountTableEntryRequest.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-
-/**
- * API request for updating the destination of an existing mount point in the
- * state store.
- */
-public abstract class UpdateMountTableEntryRequest {
-
- public static UpdateMountTableEntryRequest newInstance() throws IOException {
- return StateStoreSerializer.newRecord(UpdateMountTableEntryRequest.class);
- }
-
- public static UpdateMountTableEntryRequest newInstance(MountTable entry)
- throws IOException {
- UpdateMountTableEntryRequest request = newInstance();
- request.setEntry(entry);
- return request;
- }
-
- @Public
- @Unstable
- public abstract MountTable getEntry() throws IOException;
-
- @Public
- @Unstable
- public abstract void setEntry(MountTable mount) throws IOException;
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateMountTableEntryResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateMountTableEntryResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateMountTableEntryResponse.java
deleted file mode 100644
index 7097e10..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateMountTableEntryResponse.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-
-/**
- * API response for updating the destination of an existing mount point in the
- * state store.
- */
-public abstract class UpdateMountTableEntryResponse {
-
- public static UpdateMountTableEntryResponse newInstance() throws IOException {
- return StateStoreSerializer.newRecord(UpdateMountTableEntryResponse.class);
- }
-
- @Public
- @Unstable
- public abstract boolean getStatus();
-
- @Public
- @Unstable
- public abstract void setStatus(boolean result);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateNamenodeRegistrationRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateNamenodeRegistrationRequest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateNamenodeRegistrationRequest.java
deleted file mode 100644
index 4459e33..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateNamenodeRegistrationRequest.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-
-/**
- * API request for overriding an existing namenode registration in the state
- * store.
- */
-public abstract class UpdateNamenodeRegistrationRequest {
-
- public static UpdateNamenodeRegistrationRequest newInstance()
- throws IOException {
- return StateStoreSerializer.newRecord(
- UpdateNamenodeRegistrationRequest.class);
- }
-
- public static UpdateNamenodeRegistrationRequest newInstance(
- String nameserviceId, String namenodeId,
- FederationNamenodeServiceState state) throws IOException {
- UpdateNamenodeRegistrationRequest request = newInstance();
- request.setNameserviceId(nameserviceId);
- request.setNamenodeId(namenodeId);
- request.setState(state);
- return request;
- }
-
- @Private
- @Unstable
- public abstract String getNameserviceId();
-
- @Private
- @Unstable
- public abstract String getNamenodeId();
-
- @Private
- @Unstable
- public abstract FederationNamenodeServiceState getState();
-
- @Private
- @Unstable
- public abstract void setNameserviceId(String nsId);
-
- @Private
- @Unstable
- public abstract void setNamenodeId(String nnId);
-
- @Private
- @Unstable
- public abstract void setState(FederationNamenodeServiceState state);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateNamenodeRegistrationResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateNamenodeRegistrationResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateNamenodeRegistrationResponse.java
deleted file mode 100644
index 1f0d556..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateNamenodeRegistrationResponse.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-
-/**
- * API response for overriding an existing namenode registration in the state
- * store.
- */
-public abstract class UpdateNamenodeRegistrationResponse {
-
- public static UpdateNamenodeRegistrationResponse newInstance() {
- return StateStoreSerializer.newRecord(
- UpdateNamenodeRegistrationResponse.class);
- }
-
- public static UpdateNamenodeRegistrationResponse newInstance(boolean status)
- throws IOException {
- UpdateNamenodeRegistrationResponse response = newInstance();
- response.setResult(status);
- return response;
- }
-
- @Private
- @Unstable
- public abstract boolean getResult();
-
- @Private
- @Unstable
- public abstract void setResult(boolean result);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntryRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntryRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntryRequestPBImpl.java
deleted file mode 100644
index 35455d2..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntryRequestPBImpl.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryRequestProtoOrBuilder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProto;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.MountTablePBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * AddMountTableEntryRequest.
- */
-public class AddMountTableEntryRequestPBImpl
- extends AddMountTableEntryRequest implements PBRecord {
-
- private FederationProtocolPBTranslator<AddMountTableEntryRequestProto,
- AddMountTableEntryRequestProto.Builder,
- AddMountTableEntryRequestProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<AddMountTableEntryRequestProto,
- AddMountTableEntryRequestProto.Builder,
- AddMountTableEntryRequestProtoOrBuilder>(
- AddMountTableEntryRequestProto.class);
-
- public AddMountTableEntryRequestPBImpl() {
- }
-
- public AddMountTableEntryRequestPBImpl(AddMountTableEntryRequestProto proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public AddMountTableEntryRequestProto getProto() {
- return this.translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public MountTable getEntry() {
- MountTableRecordProto entryProto =
- this.translator.getProtoOrBuilder().getEntry();
- return new MountTablePBImpl(entryProto);
- }
-
- @Override
- public void setEntry(MountTable mount) {
- if (mount instanceof MountTablePBImpl) {
- MountTablePBImpl mountPB = (MountTablePBImpl)mount;
- MountTableRecordProto mountProto = mountPB.getProto();
- translator.getBuilder().setEntry(mountProto);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntryResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntryResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntryResponsePBImpl.java
deleted file mode 100644
index c1d9a65..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntryResponsePBImpl.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryResponseProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryResponseProtoOrBuilder;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * AddMountTableEntryResponse.
- */
-public class AddMountTableEntryResponsePBImpl
- extends AddMountTableEntryResponse implements PBRecord {
-
- private FederationProtocolPBTranslator<AddMountTableEntryResponseProto,
- AddMountTableEntryResponseProto.Builder,
- AddMountTableEntryResponseProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<AddMountTableEntryResponseProto,
- AddMountTableEntryResponseProto.Builder,
- AddMountTableEntryResponseProtoOrBuilder>(
- AddMountTableEntryResponseProto.class);
-
- public AddMountTableEntryResponsePBImpl() {
- }
-
- public AddMountTableEntryResponsePBImpl(
- AddMountTableEntryResponseProto proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public AddMountTableEntryResponseProto getProto() {
- return this.translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public boolean getStatus() {
- return this.translator.getProtoOrBuilder().getStatus();
- }
-
- @Override
- public void setStatus(boolean result) {
- this.translator.getBuilder().setStatus(result);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/EnterSafeModeRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/EnterSafeModeRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/EnterSafeModeRequestPBImpl.java
deleted file mode 100644
index 50e14af..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/EnterSafeModeRequestPBImpl.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.EnterSafeModeRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.EnterSafeModeRequestProtoOrBuilder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationResponseProto.Builder;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeRequest;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * EnterSafeModeRequest.
- */
-public class EnterSafeModeRequestPBImpl extends EnterSafeModeRequest
- implements PBRecord {
-
- private FederationProtocolPBTranslator<EnterSafeModeRequestProto,
- Builder, EnterSafeModeRequestProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<>(EnterSafeModeRequestProto.class);
-
- public EnterSafeModeRequestPBImpl() {
- }
-
- public EnterSafeModeRequestPBImpl(EnterSafeModeRequestProto proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public EnterSafeModeRequestProto getProto() {
- return translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/EnterSafeModeResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/EnterSafeModeResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/EnterSafeModeResponsePBImpl.java
deleted file mode 100644
index 9805821..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/EnterSafeModeResponsePBImpl.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.EnterSafeModeResponseProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.EnterSafeModeResponseProto.Builder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.EnterSafeModeResponseProtoOrBuilder;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * EnterSafeModeResponse.
- */
-public class EnterSafeModeResponsePBImpl extends EnterSafeModeResponse
- implements PBRecord {
-
- private FederationProtocolPBTranslator<EnterSafeModeResponseProto,
- Builder, EnterSafeModeResponseProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<>(
- EnterSafeModeResponseProto.class);
-
- public EnterSafeModeResponsePBImpl() {
- }
-
- public EnterSafeModeResponsePBImpl(EnterSafeModeResponseProto proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public EnterSafeModeResponseProto getProto() {
- return translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public boolean getStatus() {
- return this.translator.getProtoOrBuilder().getStatus();
- }
-
- @Override
- public void setStatus(boolean result) {
- this.translator.getBuilder().setStatus(result);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/FederationProtocolPBTranslator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/FederationProtocolPBTranslator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/FederationProtocolPBTranslator.java
deleted file mode 100644
index baad113..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/FederationProtocolPBTranslator.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-import java.lang.reflect.Method;
-
-import org.apache.commons.codec.binary.Base64;
-
-import com.google.protobuf.GeneratedMessage;
-import com.google.protobuf.Message;
-import com.google.protobuf.Message.Builder;
-import com.google.protobuf.MessageOrBuilder;
-
-/**
- * Helper class for setting/getting data elements in an object backed by a
- * protobuf implementation.
- */
-public class FederationProtocolPBTranslator<P extends GeneratedMessage,
- B extends Builder, T extends MessageOrBuilder> {
-
- /** Optional proto byte stream used to create this object. */
- private P proto;
- /** The class of the proto handler for this translator. */
- private Class<P> protoClass;
- /** Internal builder, used to store data that has been set. */
- private B builder;
-
- public FederationProtocolPBTranslator(Class<P> protoType) {
- this.protoClass = protoType;
- }
-
- /**
- * Called if this translator is to be created from an existing protobuf byte
- * stream.
- *
- * @param p The existing proto object to use to initialize the translator.
- * @throws IllegalArgumentException
- */
- @SuppressWarnings("unchecked")
- public void setProto(Message p) {
- if (protoClass.isInstance(p)) {
- if (this.builder != null) {
- // Merge with builder
- this.builder.mergeFrom((P) p);
- } else {
- // Store proto
- this.proto = (P) p;
- }
- } else {
- throw new IllegalArgumentException(
- "Cannot decode proto type " + p.getClass().getName());
- }
- }
-
- /**
- * Create or return the cached protobuf builder for this translator.
- *
- * @return cached Builder instance
- */
- @SuppressWarnings("unchecked")
- public B getBuilder() {
- if (this.builder == null) {
- try {
- Method method = protoClass.getMethod("newBuilder");
- this.builder = (B) method.invoke(null);
- if (this.proto != null) {
- // Merge in existing immutable proto
- this.builder.mergeFrom(this.proto);
- }
- } catch (ReflectiveOperationException e) {
- this.builder = null;
- }
- }
- return this.builder;
- }
-
- /**
- * Get the serialized proto object. If the translator was created from a byte
- * stream, returns the intitial byte stream. Otherwise creates a new byte
- * stream from the cached builder.
- *
- * @return Protobuf message object
- */
- @SuppressWarnings("unchecked")
- public P build() {
- if (this.builder != null) {
- // serialize from builder (mutable) first
- Message m = this.builder.build();
- return (P) m;
- } else if (this.proto != null) {
- // Use immutable message source, message is unchanged
- return this.proto;
- }
- return null;
- }
-
- /**
- * Returns an interface to access data stored within this object. The object
- * may have been initialized either via a builder or by an existing protobuf
- * byte stream.
- *
- * @return MessageOrBuilder protobuf interface for the requested class.
- */
- @SuppressWarnings("unchecked")
- public T getProtoOrBuilder() {
- if (this.builder != null) {
- // Use mutable builder if it exists
- return (T) this.builder;
- } else if (this.proto != null) {
- // Use immutable message source
- return (T) this.proto;
- } else {
- // Construct empty builder
- return (T) this.getBuilder();
- }
- }
-
- /**
- * Read instance from base64 data.
- * @param base64String
- * @throws IOException
- */
- @SuppressWarnings("unchecked")
- public void readInstance(String base64String) throws IOException {
- byte[] bytes = Base64.decodeBase64(base64String);
- Message msg = getBuilder().mergeFrom(bytes).build();
- this.proto = (P) msg;
- }
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreBaseImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreBaseImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreBaseImpl.java
new file mode 100644
index 0000000..1bd35f2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreBaseImpl.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
+
+import static org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils.filterMultiple;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.Query;
+import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
+
+/**
+ * Base implementation of a State Store driver. It contains default
+ * implementations for the optional functions. These implementations use an
+ * uncached read/write all algorithm for all changes. In most cases it is
+ * recommended to override the optional functions.
+ * <p>
+ * Drivers may optionally override additional routines for performance
+ * optimization, such as custom get/put/remove queries, depending on the
+ * capabilities of the data store.
+ * <p>
+ */
+public abstract class StateStoreBaseImpl extends StateStoreDriver {
+
+ @Override
+ public <T extends BaseRecord> T get(
+ Class<T> clazz, Query<T> query) throws IOException {
+ List<T> records = getMultiple(clazz, query);
+ if (records.size() > 1) {
+ throw new IOException("Found more than one object in collection");
+ } else if (records.size() == 1) {
+ return records.get(0);
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public <T extends BaseRecord> List<T> getMultiple(
+ Class<T> clazz, Query<T> query) throws IOException {
+ QueryResult<T> result = get(clazz);
+ List<T> records = result.getRecords();
+ List<T> ret = filterMultiple(query, records);
+ if (ret == null) {
+ throw new IOException("Cannot fetch records from the store");
+ }
+ return ret;
+ }
+
+ @Override
+ public <T extends BaseRecord> boolean put(
+ T record, boolean allowUpdate, boolean errorIfExists) throws IOException {
+ List<T> singletonList = new ArrayList<>();
+ singletonList.add(record);
+ return putAll(singletonList, allowUpdate, errorIfExists);
+ }
+
+ @Override
+ public <T extends BaseRecord> boolean remove(T record) throws IOException {
+ final Query<T> query = new Query<T>(record);
+ Class<? extends BaseRecord> clazz = record.getClass();
+ @SuppressWarnings("unchecked")
+ Class<T> recordClass = (Class<T>)StateStoreUtils.getRecordClass(clazz);
+ return remove(recordClass, query) == 1;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
new file mode 100644
index 0000000..6638d1c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
@@ -0,0 +1,462 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
+
+import static org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils.filterMultiple;
+import static org.apache.hadoop.util.Time.monotonicNow;
+import static org.apache.hadoop.util.Time.now;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.hdfs.server.federation.metrics.StateStoreMetrics;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreUnavailableException;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.Query;
+import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * {@link StateStoreDriver} implementation based on files. In this approach, we
+ * use temporary files for the writes and renaming "atomically" to the final
+ * value. Instead of writing to the final location, it will go to a temporary
+ * one and then rename to the final destination.
+ */
+public abstract class StateStoreFileBaseImpl
+ extends StateStoreSerializableImpl {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StateStoreFileBaseImpl.class);
+
+ /** File extension for temporary files. */
+ private static final String TMP_MARK = ".tmp";
+ /** We remove temporary files older than 10 seconds. */
+ private static final long OLD_TMP_RECORD_MS = TimeUnit.SECONDS.toMillis(10);
+ /** File pattern for temporary records: file.XYZ.tmp. */
+ private static final Pattern OLD_TMP_RECORD_PATTERN =
+ Pattern.compile(".+\\.(\\d+)\\.tmp");
+
+ /** If it is initialized. */
+ private boolean initialized = false;
+
+
+ /**
+ * Get the reader of a record for the file system.
+ *
+ * @param path Path of the record to read.
+ * @return Reader for the record.
+ */
+ protected abstract <T extends BaseRecord> BufferedReader getReader(
+ String path);
+
+ /**
+ * Get the writer of a record for the file system.
+ *
+ * @param path Path of the record to write.
+ * @return Writer for the record.
+ */
+ protected abstract <T extends BaseRecord> BufferedWriter getWriter(
+ String path);
+
+ /**
+ * Check if a path exists.
+ *
+ * @param path Path to check.
+ * @return If the path exists.
+ */
+ protected abstract boolean exists(String path);
+
+ /**
+ * Make a directory.
+ *
+ * @param path Path of the directory to create.
+ * @return If the directory was created.
+ */
+ protected abstract boolean mkdir(String path);
+
+ /**
+ * Rename a file. This should be atomic.
+ *
+ * @param src Source name.
+ * @param dst Destination name.
+ * @return If the rename was successful.
+ */
+ protected abstract boolean rename(String src, String dst);
+
+ /**
+ * Remove a file.
+ *
+ * @param path Path for the file to remove
+ * @return If the file was removed.
+ */
+ protected abstract boolean remove(String path);
+
+ /**
+ * Get the children for a path.
+ *
+ * @param path Path to check.
+ * @return List of children.
+ */
+ protected abstract List<String> getChildren(String path);
+
+ /**
+ * Get root directory.
+ *
+ * @return Root directory.
+ */
+ protected abstract String getRootDir();
+
+ /**
+ * Set the driver as initialized.
+ *
+ * @param ini If the driver is initialized.
+ */
+ public void setInitialized(boolean ini) {
+ this.initialized = ini;
+ }
+
+ @Override
+ public boolean initDriver() {
+ String rootDir = getRootDir();
+ try {
+ if (rootDir == null) {
+ LOG.error("Invalid root directory, unable to initialize driver.");
+ return false;
+ }
+
+ // Check root path
+ if (!exists(rootDir)) {
+ if (!mkdir(rootDir)) {
+ LOG.error("Cannot create State Store root directory {}", rootDir);
+ return false;
+ }
+ }
+ } catch (Exception ex) {
+ LOG.error(
+ "Cannot initialize filesystem using root directory {}", rootDir, ex);
+ return false;
+ }
+ setInitialized(true);
+ return true;
+ }
+
+ @Override
+ public <T extends BaseRecord> boolean initRecordStorage(
+ String className, Class<T> recordClass) {
+
+ String dataDirPath = getRootDir() + "/" + className;
+ try {
+ // Create data directories for files
+ if (!exists(dataDirPath)) {
+ LOG.info("{} data directory doesn't exist, creating it", dataDirPath);
+ if (!mkdir(dataDirPath)) {
+ LOG.error("Cannot create data directory {}", dataDirPath);
+ return false;
+ }
+ }
+ } catch (Exception ex) {
+ LOG.error("Cannot create data directory {}", dataDirPath, ex);
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public <T extends BaseRecord> QueryResult<T> get(Class<T> clazz)
+ throws IOException {
+ verifyDriverReady();
+ long start = monotonicNow();
+ StateStoreMetrics metrics = getMetrics();
+ List<T> ret = new ArrayList<>();
+ try {
+ String path = getPathForClass(clazz);
+ List<String> children = getChildren(path);
+ for (String child : children) {
+ String pathRecord = path + "/" + child;
+ if (child.endsWith(TMP_MARK)) {
+ LOG.debug("There is a temporary file {} in {}", child, path);
+ if (isOldTempRecord(child)) {
+ LOG.warn("Removing {} as it's an old temporary record", child);
+ remove(pathRecord);
+ }
+ } else {
+ T record = getRecord(pathRecord, clazz);
+ ret.add(record);
+ }
+ }
+ } catch (Exception e) {
+ if (metrics != null) {
+ metrics.addFailure(monotonicNow() - start);
+ }
+ String msg = "Cannot fetch records for " + clazz.getSimpleName();
+ LOG.error(msg, e);
+ throw new IOException(msg, e);
+ }
+
+ if (metrics != null) {
+ metrics.addRead(monotonicNow() - start);
+ }
+ return new QueryResult<T>(ret, getTime());
+ }
+
+ /**
+ * Check if a record is temporary and old.
+ *
+ * @param pathRecord Path for the record to check.
+ * @return If the record is temporary and old.
+ */
+ @VisibleForTesting
+ public static boolean isOldTempRecord(final String pathRecord) {
+ if (!pathRecord.endsWith(TMP_MARK)) {
+ return false;
+ }
+ // Extract temporary record creation time
+ Matcher m = OLD_TMP_RECORD_PATTERN.matcher(pathRecord);
+ if (m.find()) {
+ long time = Long.parseLong(m.group(1));
+ return now() - time > OLD_TMP_RECORD_MS;
+ }
+ return false;
+ }
+
+ /**
+ * Read a record from a file.
+ *
+ * @param path Path to the file containing the record.
+ * @param clazz Class of the record.
+ * @return Record read from the file.
+ * @throws IOException If the file cannot be read.
+ */
+ private <T extends BaseRecord> T getRecord(
+ final String path, final Class<T> clazz) throws IOException {
+ BufferedReader reader = getReader(path);
+ try {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ if (!line.startsWith("#") && line.length() > 0) {
+ try {
+ T record = newRecord(line, clazz, false);
+ return record;
+ } catch (Exception ex) {
+ LOG.error("Cannot parse line {} in file {}", line, path, ex);
+ }
+ }
+ }
+ } finally {
+ if (reader != null) {
+ reader.close();
+ }
+ }
+ throw new IOException("Cannot read " + path + " for record " +
+ clazz.getSimpleName());
+ }
+
+ /**
+ * Get the path for a record class.
+ * @param clazz Class of the record.
+ * @return Path for this record class.
+ */
+ private <T extends BaseRecord> String getPathForClass(final Class<T> clazz) {
+ String className = StateStoreUtils.getRecordName(clazz);
+ StringBuilder sb = new StringBuilder();
+ sb.append(getRootDir());
+ if (sb.charAt(sb.length() - 1) != '/') {
+ sb.append("/");
+ }
+ sb.append(className);
+ return sb.toString();
+ }
+
+ @Override
+ public boolean isDriverReady() {
+ return this.initialized;
+ }
+
+ @Override
+ public <T extends BaseRecord> boolean putAll(
+ List<T> records, boolean allowUpdate, boolean errorIfExists)
+ throws StateStoreUnavailableException {
+ verifyDriverReady();
+ if (records.isEmpty()) {
+ return true;
+ }
+
+ long start = monotonicNow();
+ StateStoreMetrics metrics = getMetrics();
+
+ // Check if any record exists
+ Map<String, T> toWrite = new HashMap<>();
+ for (T record : records) {
+ Class<? extends BaseRecord> recordClass = record.getClass();
+ String path = getPathForClass(recordClass);
+ String primaryKey = getPrimaryKey(record);
+ String recordPath = path + "/" + primaryKey;
+
+ if (exists(recordPath)) {
+ if (allowUpdate) {
+ // Update the mod time stamp. Many backends will use their
+ // own timestamp for the mod time.
+ record.setDateModified(this.getTime());
+ toWrite.put(recordPath, record);
+ } else if (errorIfExists) {
+ LOG.error("Attempt to insert record {} that already exists",
+ recordPath);
+ if (metrics != null) {
+ metrics.addFailure(monotonicNow() - start);
+ }
+ return false;
+ } else {
+ LOG.debug("Not updating {}", record);
+ }
+ } else {
+ toWrite.put(recordPath, record);
+ }
+ }
+
+ // Write the records
+ boolean success = true;
+ for (Entry<String, T> entry : toWrite.entrySet()) {
+ String recordPath = entry.getKey();
+ String recordPathTemp = recordPath + "." + now() + TMP_MARK;
+ BufferedWriter writer = getWriter(recordPathTemp);
+ try {
+ T record = entry.getValue();
+ String line = serializeString(record);
+ writer.write(line);
+ } catch (IOException e) {
+ LOG.error("Cannot write {}", recordPathTemp, e);
+ success = false;
+ } finally {
+ if (writer != null) {
+ try {
+ writer.close();
+ } catch (IOException e) {
+ LOG.error("Cannot close the writer for {}", recordPathTemp);
+ }
+ }
+ }
+ // Commit
+ if (!rename(recordPathTemp, recordPath)) {
+ LOG.error("Failed committing record into {}", recordPath);
+ success = false;
+ }
+ }
+
+ long end = monotonicNow();
+ if (metrics != null) {
+ if (success) {
+ metrics.addWrite(end - start);
+ } else {
+ metrics.addFailure(end - start);
+ }
+ }
+ return success;
+ }
+
+ @Override
+ public <T extends BaseRecord> int remove(Class<T> clazz, Query<T> query)
+ throws StateStoreUnavailableException {
+ verifyDriverReady();
+
+ if (query == null) {
+ return 0;
+ }
+
+ long start = Time.monotonicNow();
+ StateStoreMetrics metrics = getMetrics();
+ int removed = 0;
+ // Get the current records
+ try {
+ final QueryResult<T> result = get(clazz);
+ final List<T> existingRecords = result.getRecords();
+ // Write all of the existing records except those to be removed
+ final List<T> recordsToRemove = filterMultiple(query, existingRecords);
+ boolean success = true;
+ for (T recordToRemove : recordsToRemove) {
+ String path = getPathForClass(clazz);
+ String primaryKey = getPrimaryKey(recordToRemove);
+ String recordToRemovePath = path + "/" + primaryKey;
+ if (remove(recordToRemovePath)) {
+ removed++;
+ } else {
+ LOG.error("Cannot remove record {}", recordToRemovePath);
+ success = false;
+ }
+ }
+ if (!success) {
+ LOG.error("Cannot remove records {} query {}", clazz, query);
+ if (metrics != null) {
+ metrics.addFailure(monotonicNow() - start);
+ }
+ }
+ } catch (IOException e) {
+ LOG.error("Cannot remove records {} query {}", clazz, query, e);
+ if (metrics != null) {
+ metrics.addFailure(monotonicNow() - start);
+ }
+ }
+
+ if (removed > 0 && metrics != null) {
+ metrics.addRemove(monotonicNow() - start);
+ }
+ return removed;
+ }
+
+ @Override
+ public <T extends BaseRecord> boolean removeAll(Class<T> clazz)
+ throws StateStoreUnavailableException {
+ verifyDriverReady();
+ long start = Time.monotonicNow();
+ StateStoreMetrics metrics = getMetrics();
+
+ boolean success = true;
+ String path = getPathForClass(clazz);
+ List<String> children = getChildren(path);
+ for (String child : children) {
+ String pathRecord = path + "/" + child;
+ if (!remove(pathRecord)) {
+ success = false;
+ }
+ }
+
+ if (metrics != null) {
+ long time = Time.monotonicNow() - start;
+ if (success) {
+ metrics.addRemove(time);
+ } else {
+ metrics.addFailure(time);
+ }
+ }
+ return success;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java
new file mode 100644
index 0000000..7d9ddc6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.io.Files;
+
+/**
+ * StateStoreDriver implementation based on a local file.
+ */
+public class StateStoreFileImpl extends StateStoreFileBaseImpl {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StateStoreFileImpl.class);
+
+ /** Configuration keys. */
+ public static final String FEDERATION_STORE_FILE_DIRECTORY =
+ RBFConfigKeys.FEDERATION_STORE_PREFIX + "driver.file.directory";
+
+ /** Root directory for the state store. */
+ private String rootDirectory;
+
+
+ @Override
+ protected boolean exists(String path) {
+ File test = new File(path);
+ return test.exists();
+ }
+
+ @Override
+ protected boolean mkdir(String path) {
+ File dir = new File(path);
+ return dir.mkdirs();
+ }
+
+ @Override
+ protected boolean rename(String src, String dst) {
+ try {
+ Files.move(new File(src), new File(dst));
+ return true;
+ } catch (IOException e) {
+ LOG.error("Cannot rename {} to {}", src, dst, e);
+ return false;
+ }
+ }
+
+ @Override
+ protected boolean remove(String path) {
+ File file = new File(path);
+ return file.delete();
+ }
+
+ @Override
+ protected String getRootDir() {
+ if (this.rootDirectory == null) {
+ String dir = getConf().get(FEDERATION_STORE_FILE_DIRECTORY);
+ if (dir == null) {
+ File tempDir = Files.createTempDir();
+ dir = tempDir.getAbsolutePath();
+ LOG.warn("The root directory is not available, using {}", dir);
+ }
+ this.rootDirectory = dir;
+ }
+ return this.rootDirectory;
+ }
+
+ @Override
+ protected <T extends BaseRecord> BufferedReader getReader(String filename) {
+ BufferedReader reader = null;
+ try {
+ LOG.debug("Loading file: {}", filename);
+ File file = new File(filename);
+ FileInputStream fis = new FileInputStream(file);
+ InputStreamReader isr =
+ new InputStreamReader(fis, StandardCharsets.UTF_8);
+ reader = new BufferedReader(isr);
+ } catch (Exception ex) {
+ LOG.error("Cannot open read stream for record {}", filename, ex);
+ }
+ return reader;
+ }
+
+ @Override
+ protected <T extends BaseRecord> BufferedWriter getWriter(String filename) {
+ BufferedWriter writer = null;
+ try {
+ LOG.debug("Writing file: {}", filename);
+ File file = new File(filename);
+ FileOutputStream fos = new FileOutputStream(file, false);
+ OutputStreamWriter osw =
+ new OutputStreamWriter(fos, StandardCharsets.UTF_8);
+ writer = new BufferedWriter(osw);
+ } catch (IOException e) {
+ LOG.error("Cannot open write stream for record {}", filename, e);
+ }
+ return writer;
+ }
+
+ @Override
+ public void close() throws Exception {
+ setInitialized(false);
+ }
+
+ @Override
+ protected List<String> getChildren(String path) {
+ List<String> ret = new LinkedList<>();
+ File dir = new File(path);
+ File[] files = dir.listFiles();
+ if (files != null) {
+ for (File file : files) {
+ String filename = file.getName();
+ ret.add(filename);
+ }
+ }
+ return ret;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileSystemImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileSystemImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileSystemImpl.java
new file mode 100644
index 0000000..ad822fb
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileSystemImpl.java
@@ -0,0 +1,178 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Options;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * StateStoreDriver} implementation based on a filesystem. The most common uses
+ * HDFS as a backend.
+ */
+public class StateStoreFileSystemImpl extends StateStoreFileBaseImpl {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StateStoreFileSystemImpl.class);
+
+
+ /** Configuration keys. */
+ public static final String FEDERATION_STORE_FS_PATH =
+ RBFConfigKeys.FEDERATION_STORE_PREFIX + "driver.fs.path";
+
+ /** File system to back the State Store. */
+ private FileSystem fs;
+ /** Working path in the filesystem. */
+ private String workPath;
+
+ @Override
+ protected boolean exists(String path) {
+ try {
+ return fs.exists(new Path(path));
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ @Override
+ protected boolean mkdir(String path) {
+ try {
+ return fs.mkdirs(new Path(path));
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ @Override
+ protected boolean rename(String src, String dst) {
+ try {
+ if (fs instanceof DistributedFileSystem) {
+ DistributedFileSystem dfs = (DistributedFileSystem)fs;
+ dfs.rename(new Path(src), new Path(dst), Options.Rename.OVERWRITE);
+ return true;
+ } else {
+ // Replace should be atomic but not available
+ if (fs.exists(new Path(dst))) {
+ fs.delete(new Path(dst), true);
+ }
+ return fs.rename(new Path(src), new Path(dst));
+ }
+ } catch (Exception e) {
+ LOG.error("Cannot rename {} to {}", src, dst, e);
+ return false;
+ }
+ }
+
+ @Override
+ protected boolean remove(String path) {
+ try {
+ return fs.delete(new Path(path), true);
+ } catch (Exception e) {
+ LOG.error("Cannot remove {}", path, e);
+ return false;
+ }
+ }
+
+ @Override
+ protected String getRootDir() {
+ if (this.workPath == null) {
+ String rootPath = getConf().get(FEDERATION_STORE_FS_PATH);
+ URI workUri;
+ try {
+ workUri = new URI(rootPath);
+ fs = FileSystem.get(workUri, getConf());
+ } catch (Exception ex) {
+ return null;
+ }
+ this.workPath = rootPath;
+ }
+ return this.workPath;
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (fs != null) {
+ fs.close();
+ }
+ }
+
+ @Override
+ protected <T extends BaseRecord> BufferedReader getReader(String pathName) {
+ BufferedReader reader = null;
+ Path path = new Path(pathName);
+ try {
+ FSDataInputStream fdis = fs.open(path);
+ InputStreamReader isr =
+ new InputStreamReader(fdis, StandardCharsets.UTF_8);
+ reader = new BufferedReader(isr);
+ } catch (IOException ex) {
+ LOG.error("Cannot open read stream for {}", path);
+ }
+ return reader;
+ }
+
+ @Override
+ protected <T extends BaseRecord> BufferedWriter getWriter(String pathName) {
+ BufferedWriter writer = null;
+ Path path = new Path(pathName);
+ try {
+ FSDataOutputStream fdos = fs.create(path, true);
+ OutputStreamWriter osw =
+ new OutputStreamWriter(fdos, StandardCharsets.UTF_8);
+ writer = new BufferedWriter(osw);
+ } catch (IOException ex) {
+ LOG.error("Cannot open write stream for {}", path);
+ }
+ return writer;
+ }
+
+ @Override
+ protected List<String> getChildren(String pathName) {
+ List<String> ret = new LinkedList<>();
+ Path path = new Path(workPath, pathName);
+ try {
+ FileStatus[] files = fs.listStatus(path);
+ for (FileStatus file : files) {
+ Path filePath = file.getPath();
+ String fileName = filePath.getName();
+ ret.add(fileName);
+ }
+ } catch (Exception e) {
+ LOG.error("Cannot get children for {}", pathName, e);
+ }
+ return ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializableImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializableImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializableImpl.java
new file mode 100644
index 0000000..7bc93de
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializableImpl.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.metrics.StateStoreMetrics;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+
+/**
+ * State Store driver that stores a serialization of the records. The serializer
+ * is pluggable.
+ */
+public abstract class StateStoreSerializableImpl extends StateStoreBaseImpl {
+
+ /** Mark for slashes in path names. */
+ protected static final String SLASH_MARK = "0SLASH0";
+ /** Mark for colon in path names. */
+ protected static final String COLON_MARK = "_";
+
+ /** Default serializer for this driver. */
+ private StateStoreSerializer serializer;
+
+
+ @Override
+ public boolean init(final Configuration config, final String id,
+ final Collection<Class<? extends BaseRecord>> records,
+ final StateStoreMetrics metrics) {
+ boolean ret = super.init(config, id, records, metrics);
+
+ this.serializer = StateStoreSerializer.getSerializer(config);
+
+ return ret;
+ }
+
+ /**
+ * Serialize a record using the serializer.
+ * @param record Record to serialize.
+ * @return Byte array with the serialization of the record.
+ */
+ protected <T extends BaseRecord> byte[] serialize(T record) {
+ return serializer.serialize(record);
+ }
+
+ /**
+ * Serialize a record using the serializer.
+ * @param record Record to serialize.
+ * @return String with the serialization of the record.
+ */
+ protected <T extends BaseRecord> String serializeString(T record) {
+ return serializer.serializeString(record);
+ }
+
+ /**
+ * Creates a record from an input data string.
+ * @param data Serialized text of the record.
+ * @param clazz Record class.
+ * @param includeDates If dateModified and dateCreated are serialized.
+ * @return The created record.
+ * @throws IOException
+ */
+ protected <T extends BaseRecord> T newRecord(
+ String data, Class<T> clazz, boolean includeDates) throws IOException {
+ return serializer.deserialize(data, clazz);
+ }
+
+ /**
+ * Get the primary key for a record. If we don't want to store in folders, we
+ * need to remove / from the name.
+ *
+ * @param record Record to get the primary key for.
+ * @return Primary key for the record.
+ */
+ protected static String getPrimaryKey(BaseRecord record) {
+ String primaryKey = record.getPrimaryKey();
+ primaryKey = primaryKey.replaceAll("/", SLASH_MARK);
+ primaryKey = primaryKey.replaceAll(":", COLON_MARK);
+ return primaryKey;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializerPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializerPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializerPBImpl.java
new file mode 100644
index 0000000..45c5dd6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreSerializerPBImpl.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
+
+import java.io.IOException;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.codec.binary.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+import org.apache.hadoop.util.ReflectionUtils;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the State Store serializer.
+ */
+public final class StateStoreSerializerPBImpl extends StateStoreSerializer {
+
+ private static final String PB_IMPL_PACKAGE_SUFFIX = "impl.pb";
+ private static final String PB_IMPL_CLASS_SUFFIX = "PBImpl";
+
+ private Configuration localConf = new Configuration();
+
+
+ private StateStoreSerializerPBImpl() {
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <T> T newRecordInstance(Class<T> clazz) {
+ try {
+ String clazzPBImpl = getPBImplClassName(clazz);
+ Class<?> pbClazz = localConf.getClassByName(clazzPBImpl);
+ Object retObject = ReflectionUtils.newInstance(pbClazz, localConf);
+ return (T)retObject;
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private String getPBImplClassName(Class<?> clazz) {
+ String srcPackagePart = getPackageName(clazz);
+ String srcClassName = getClassName(clazz);
+ String destPackagePart = srcPackagePart + "." + PB_IMPL_PACKAGE_SUFFIX;
+ String destClassPart = srcClassName + PB_IMPL_CLASS_SUFFIX;
+ return destPackagePart + "." + destClassPart;
+ }
+
+ private String getClassName(Class<?> clazz) {
+ String fqName = clazz.getName();
+ return (fqName.substring(fqName.lastIndexOf(".") + 1, fqName.length()));
+ }
+
+ private String getPackageName(Class<?> clazz) {
+ return clazz.getPackage().getName();
+ }
+
+ @Override
+ public byte[] serialize(BaseRecord record) {
+ byte[] byteArray64 = null;
+ if (record instanceof PBRecord) {
+ PBRecord recordPB = (PBRecord) record;
+ Message msg = recordPB.getProto();
+ byte[] byteArray = msg.toByteArray();
+ byteArray64 = Base64.encodeBase64(byteArray, false);
+ }
+ return byteArray64;
+ }
+
+ @Override
+ public String serializeString(BaseRecord record) {
+ byte[] byteArray64 = serialize(record);
+ String base64Encoded = StringUtils.newStringUtf8(byteArray64);
+ return base64Encoded;
+ }
+
+ @Override
+ public <T extends BaseRecord> T deserialize(
+ byte[] byteArray, Class<T> clazz) throws IOException {
+
+ T record = newRecord(clazz);
+ if (record instanceof PBRecord) {
+ PBRecord pbRecord = (PBRecord) record;
+ byte[] byteArray64 = Base64.encodeBase64(byteArray, false);
+ String base64Encoded = StringUtils.newStringUtf8(byteArray64);
+ pbRecord.readInstance(base64Encoded);
+ }
+ return record;
+ }
+
+ @Override
+ public <T extends BaseRecord> T deserialize(String data, Class<T> clazz)
+ throws IOException {
+ byte[] byteArray64 = Base64.decodeBase64(data);
+ return deserialize(byteArray64, clazz);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreZooKeeperImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreZooKeeperImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreZooKeeperImpl.java
new file mode 100644
index 0000000..cd5372d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreZooKeeperImpl.java
@@ -0,0 +1,328 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
+
+import static org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils.filterMultiple;
+import static org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils.getRecordName;
+import static org.apache.hadoop.util.curator.ZKCuratorManager.getNodePath;
+import static org.apache.hadoop.util.Time.monotonicNow;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.Query;
+import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
+import org.apache.hadoop.util.curator.ZKCuratorManager;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * {@link StateStoreDriver} driver implementation that uses ZooKeeper as a
+ * backend.
+ * <p>
+ * The structure of the znodes in the ensemble is:
+ * PARENT_PATH
+ * |--- MOUNT
+ * |--- MEMBERSHIP
+ * |--- REBALANCER
+ * |--- ROUTERS
+ */
+public class StateStoreZooKeeperImpl extends StateStoreSerializableImpl {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StateStoreZooKeeperImpl.class);
+
+
+ /** Configuration keys. */
+ public static final String FEDERATION_STORE_ZK_DRIVER_PREFIX =
+ RBFConfigKeys.FEDERATION_STORE_PREFIX + "driver.zk.";
+ public static final String FEDERATION_STORE_ZK_PARENT_PATH =
+ FEDERATION_STORE_ZK_DRIVER_PREFIX + "parent-path";
+ public static final String FEDERATION_STORE_ZK_PARENT_PATH_DEFAULT =
+ "/hdfs-federation";
+
+
+ /** Directory to store the state store data. */
+ private String baseZNode;
+
+ /** Interface to ZooKeeper. */
+ private ZKCuratorManager zkManager;
+
+
+ @Override
+ public boolean initDriver() {
+ LOG.info("Initializing ZooKeeper connection");
+
+ Configuration conf = getConf();
+ baseZNode = conf.get(
+ FEDERATION_STORE_ZK_PARENT_PATH,
+ FEDERATION_STORE_ZK_PARENT_PATH_DEFAULT);
+ try {
+ this.zkManager = new ZKCuratorManager(conf);
+ this.zkManager.start();
+ } catch (IOException e) {
+ LOG.error("Cannot initialize the ZK connection", e);
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public <T extends BaseRecord> boolean initRecordStorage(
+ String className, Class<T> clazz) {
+ try {
+ String checkPath = getNodePath(baseZNode, className);
+ zkManager.createRootDirRecursively(checkPath);
+ return true;
+ } catch (Exception e) {
+ LOG.error("Cannot initialize ZK node for {}: {}",
+ className, e.getMessage());
+ return false;
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (zkManager != null) {
+ zkManager.close();
+ }
+ }
+
+ @Override
+ public boolean isDriverReady() {
+ if (zkManager == null) {
+ return false;
+ }
+ CuratorFramework curator = zkManager.getCurator();
+ if (curator == null) {
+ return false;
+ }
+ return curator.getState() == CuratorFrameworkState.STARTED;
+ }
+
+ @Override
+ public <T extends BaseRecord> QueryResult<T> get(Class<T> clazz)
+ throws IOException {
+ verifyDriverReady();
+ long start = monotonicNow();
+ List<T> ret = new ArrayList<>();
+ String znode = getZNodeForClass(clazz);
+ try {
+ List<String> children = zkManager.getChildren(znode);
+ for (String child : children) {
+ try {
+ String path = getNodePath(znode, child);
+ Stat stat = new Stat();
+ String data = zkManager.getStringData(path, stat);
+ boolean corrupted = false;
+ if (data == null || data.equals("")) {
+ // All records should have data, otherwise this is corrupted
+ corrupted = true;
+ } else {
+ try {
+ T record = createRecord(data, stat, clazz);
+ ret.add(record);
+ } catch (IOException e) {
+ LOG.error("Cannot create record type \"{}\" from \"{}\": {}",
+ clazz.getSimpleName(), data, e.getMessage());
+ corrupted = true;
+ }
+ }
+
+ if (corrupted) {
+ LOG.error("Cannot get data for {} at {}, cleaning corrupted data",
+ child, path);
+ zkManager.delete(path);
+ }
+ } catch (Exception e) {
+ LOG.error("Cannot get data for {}: {}", child, e.getMessage());
+ }
+ }
+ } catch (Exception e) {
+ getMetrics().addFailure(monotonicNow() - start);
+ String msg = "Cannot get children for \"" + znode + "\": " +
+ e.getMessage();
+ LOG.error(msg);
+ throw new IOException(msg);
+ }
+ long end = monotonicNow();
+ getMetrics().addRead(end - start);
+ return new QueryResult<T>(ret, getTime());
+ }
+
+ @Override
+ public <T extends BaseRecord> boolean putAll(
+ List<T> records, boolean update, boolean error) throws IOException {
+ verifyDriverReady();
+ if (records.isEmpty()) {
+ return true;
+ }
+
+ // All records should be the same
+ T record0 = records.get(0);
+ Class<? extends BaseRecord> recordClass = record0.getClass();
+ String znode = getZNodeForClass(recordClass);
+
+ long start = monotonicNow();
+ boolean status = true;
+ for (T record : records) {
+ String primaryKey = getPrimaryKey(record);
+ String recordZNode = getNodePath(znode, primaryKey);
+ byte[] data = serialize(record);
+ if (!writeNode(recordZNode, data, update, error)){
+ status = false;
+ }
+ }
+ long end = monotonicNow();
+ if (status) {
+ getMetrics().addWrite(end - start);
+ } else {
+ getMetrics().addFailure(end - start);
+ }
+ return status;
+ }
+
+ @Override
+ public <T extends BaseRecord> int remove(
+ Class<T> clazz, Query<T> query) throws IOException {
+ verifyDriverReady();
+ if (query == null) {
+ return 0;
+ }
+
+ // Read the current data
+ long start = monotonicNow();
+ List<T> records = null;
+ try {
+ QueryResult<T> result = get(clazz);
+ records = result.getRecords();
+ } catch (IOException ex) {
+ LOG.error("Cannot get existing records", ex);
+ getMetrics().addFailure(monotonicNow() - start);
+ return 0;
+ }
+
+ // Check the records to remove
+ String znode = getZNodeForClass(clazz);
+ List<T> recordsToRemove = filterMultiple(query, records);
+
+ // Remove the records
+ int removed = 0;
+ for (T existingRecord : recordsToRemove) {
+ LOG.info("Removing \"{}\"", existingRecord);
+ try {
+ String primaryKey = getPrimaryKey(existingRecord);
+ String path = getNodePath(znode, primaryKey);
+ if (zkManager.delete(path)) {
+ removed++;
+ } else {
+ LOG.error("Did not remove \"{}\"", existingRecord);
+ }
+ } catch (Exception e) {
+ LOG.error("Cannot remove \"{}\"", existingRecord, e);
+ getMetrics().addFailure(monotonicNow() - start);
+ }
+ }
+ long end = monotonicNow();
+ if (removed > 0) {
+ getMetrics().addRemove(end - start);
+ }
+ return removed;
+ }
+
+ @Override
+ public <T extends BaseRecord> boolean removeAll(Class<T> clazz)
+ throws IOException {
+ long start = monotonicNow();
+ boolean status = true;
+ String znode = getZNodeForClass(clazz);
+ LOG.info("Deleting all children under {}", znode);
+ try {
+ List<String> children = zkManager.getChildren(znode);
+ for (String child : children) {
+ String path = getNodePath(znode, child);
+ LOG.info("Deleting {}", path);
+ zkManager.delete(path);
+ }
+ } catch (Exception e) {
+ LOG.error("Cannot remove {}: {}", znode, e.getMessage());
+ status = false;
+ }
+ long time = monotonicNow() - start;
+ if (status) {
+ getMetrics().addRemove(time);
+ } else {
+ getMetrics().addFailure(time);
+ }
+ return status;
+ }
+
+ private boolean writeNode(
+ String znode, byte[] bytes, boolean update, boolean error) {
+ try {
+ boolean created = zkManager.create(znode);
+ if (!update && !created && error) {
+ LOG.info("Cannot write record \"{}\", it already exists", znode);
+ return false;
+ }
+
+ // Write data
+ zkManager.setData(znode, bytes, -1);
+ return true;
+ } catch (Exception e) {
+ LOG.error("Cannot write record \"{}\": {}", znode, e.getMessage());
+ }
+ return false;
+ }
+
+ /**
+ * Get the ZNode for a class.
+ *
+ * @param clazz Record class to evaluate.
+ * @return The ZNode for the class.
+ */
+ private <T extends BaseRecord> String getZNodeForClass(Class<T> clazz) {
+ String className = getRecordName(clazz);
+ return getNodePath(baseZNode, className);
+ }
+
+ /**
+ * Creates a record from a string returned by ZooKeeper.
+ *
+ * @param data The data to write.
+ * @param stat Stat of the data record to create.
+ * @param clazz The data record type to create.
+ * @return The created record.
+ * @throws IOException
+ */
+ private <T extends BaseRecord> T createRecord(
+ String data, Stat stat, Class<T> clazz) throws IOException {
+ T record = newRecord(data, clazz, false);
+ record.setDateCreated(stat.getCtime());
+ record.setDateModified(stat.getMtime());
+ return record;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/package-info.java
new file mode 100644
index 0000000..a18433e
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/package-info.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Implementations of state store data providers/drivers. Each driver is
+ * responsible for maintaining, querying, updating and deleting persistent data
+ * records. Data records are defined as subclasses of
+ * {@link org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord}.
+ * Each driver implements the
+ * {@link org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver
+ * StateStoreDriver} interface.
+ * <p>
+ * Currently supported drivers:
+ * <ul>
+ * <li>{@link StateStoreFileImpl} A file-based data storage backend.
+ * <li>{@link StateStoreZooKeeperImpl} A zookeeper based data storage backend.
+ * </ul>
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+package org.apache.hadoop.hdfs.server.federation.store.driver.impl;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/package-info.java
new file mode 100644
index 0000000..da998b5
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/package-info.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The state store uses modular data storage classes derived from
+ * StateStoreDriver to handle querying, updating and deleting data records. The
+ * data storage driver is initialized and maintained by the StateStoreService.
+ * The state store supports fetching all records of a type, filtering by column
+ * values or fetching a single record by its primary key.
+ * <p>
+ * Each data storage backend is required to implement the methods contained in
+ * the StateStoreDriver interface. These methods allow the querying, updating,
+ * inserting and deleting of data records into the state store.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+
+package org.apache.hadoop.hdfs.server.federation.store.driver;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MembershipStoreImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MembershipStoreImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MembershipStoreImpl.java
new file mode 100644
index 0000000..57b7b61
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MembershipStoreImpl.java
@@ -0,0 +1,310 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.impl;
+
+import static org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils.filterMultiple;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
+import org.apache.hadoop.hdfs.server.federation.store.MembershipStore;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreCache;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateNamenodeRegistrationRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateNamenodeRegistrationResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.hdfs.server.federation.store.records.Query;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Implementation of the {@link MembershipStore} State Store API.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class MembershipStoreImpl
+ extends MembershipStore implements StateStoreCache {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(MembershipStoreImpl.class);
+
+
+ /** Reported namespaces that are not decommissioned. */
+ private final Set<FederationNamespaceInfo> activeNamespaces;
+
+ /** Namenodes (after evaluating the quorum) that are active in the cluster. */
+ private final Map<String, MembershipState> activeRegistrations;
+ /** Namenode status reports (raw) that were discarded for being too old. */
+ private final Map<String, MembershipState> expiredRegistrations;
+
+ /** Lock to access the local memory cache. */
+ private final ReadWriteLock cacheReadWriteLock =
+ new ReentrantReadWriteLock();
+ private final Lock cacheReadLock = cacheReadWriteLock.readLock();
+ private final Lock cacheWriteLock = cacheReadWriteLock.writeLock();
+
+
+ public MembershipStoreImpl(StateStoreDriver driver) {
+ super(driver);
+
+ this.activeRegistrations = new HashMap<>();
+ this.expiredRegistrations = new HashMap<>();
+ this.activeNamespaces = new TreeSet<>();
+ }
+
+ @Override
+ public GetNamenodeRegistrationsResponse getExpiredNamenodeRegistrations(
+ GetNamenodeRegistrationsRequest request) throws IOException {
+
+ GetNamenodeRegistrationsResponse response =
+ GetNamenodeRegistrationsResponse.newInstance();
+ cacheReadLock.lock();
+ try {
+ Collection<MembershipState> vals = this.expiredRegistrations.values();
+ List<MembershipState> copyVals = new ArrayList<>(vals);
+ response.setNamenodeMemberships(copyVals);
+ } finally {
+ cacheReadLock.unlock();
+ }
+ return response;
+ }
+
+ @Override
+ public GetNamespaceInfoResponse getNamespaceInfo(
+ GetNamespaceInfoRequest request) throws IOException {
+
+ Set<FederationNamespaceInfo> namespaces = new HashSet<>();
+ try {
+ cacheReadLock.lock();
+ namespaces.addAll(activeNamespaces);
+ } finally {
+ cacheReadLock.unlock();
+ }
+
+ GetNamespaceInfoResponse response =
+ GetNamespaceInfoResponse.newInstance(namespaces);
+ return response;
+ }
+
+ @Override
+ public GetNamenodeRegistrationsResponse getNamenodeRegistrations(
+ final GetNamenodeRegistrationsRequest request) throws IOException {
+
+ // TODO Cache some common queries and sorts
+ List<MembershipState> ret = null;
+
+ cacheReadLock.lock();
+ try {
+ Collection<MembershipState> registrations = activeRegistrations.values();
+ MembershipState partialMembership = request.getPartialMembership();
+ if (partialMembership == null) {
+ ret = new ArrayList<>(registrations);
+ } else {
+ Query<MembershipState> query = new Query<>(partialMembership);
+ ret = filterMultiple(query, registrations);
+ }
+ } finally {
+ cacheReadLock.unlock();
+ }
+ // Sort in ascending update date order
+ Collections.sort(ret);
+
+ GetNamenodeRegistrationsResponse response =
+ GetNamenodeRegistrationsResponse.newInstance(ret);
+ return response;
+ }
+
+ @Override
+ public NamenodeHeartbeatResponse namenodeHeartbeat(
+ NamenodeHeartbeatRequest request) throws IOException {
+
+ MembershipState record = request.getNamenodeMembership();
+ String nnId = record.getNamenodeKey();
+ MembershipState existingEntry = null;
+ cacheReadLock.lock();
+ try {
+ existingEntry = this.activeRegistrations.get(nnId);
+ } finally {
+ cacheReadLock.unlock();
+ }
+
+ if (existingEntry != null) {
+ if (existingEntry.getState() != record.getState()) {
+ LOG.info("NN registration state has changed: {} -> {}",
+ existingEntry, record);
+ } else {
+ LOG.debug("Updating NN registration: {} -> {}", existingEntry, record);
+ }
+ } else {
+ LOG.info("Inserting new NN registration: {}", record);
+ }
+
+ boolean status = getDriver().put(record, true, false);
+
+ NamenodeHeartbeatResponse response =
+ NamenodeHeartbeatResponse.newInstance(status);
+ return response;
+ }
+
+ @Override
+ public boolean loadCache(boolean force) throws IOException {
+ super.loadCache(force);
+
+ // Update local cache atomically
+ cacheWriteLock.lock();
+ try {
+ this.activeRegistrations.clear();
+ this.expiredRegistrations.clear();
+ this.activeNamespaces.clear();
+
+ // Build list of NN registrations: nnId -> registration list
+ Map<String, List<MembershipState>> nnRegistrations = new HashMap<>();
+ List<MembershipState> cachedRecords = getCachedRecords();
+ for (MembershipState membership : cachedRecords) {
+ String nnId = membership.getNamenodeKey();
+ if (membership.getState() == FederationNamenodeServiceState.EXPIRED) {
+ // Expired, RPC service does not use these
+ String key = membership.getPrimaryKey();
+ this.expiredRegistrations.put(key, membership);
+ } else {
+ // This is a valid NN registration, build a list of all registrations
+ // using the NN id to use for the quorum calculation.
+ List<MembershipState> nnRegistration =
+ nnRegistrations.get(nnId);
+ if (nnRegistration == null) {
+ nnRegistration = new LinkedList<>();
+ nnRegistrations.put(nnId, nnRegistration);
+ }
+ nnRegistration.add(membership);
+ String bpId = membership.getBlockPoolId();
+ String cId = membership.getClusterId();
+ String nsId = membership.getNameserviceId();
+ FederationNamespaceInfo nsInfo =
+ new FederationNamespaceInfo(bpId, cId, nsId);
+ this.activeNamespaces.add(nsInfo);
+ }
+ }
+
+ // Calculate most representative entry for each active NN id
+ for (List<MembershipState> nnRegistration : nnRegistrations.values()) {
+ // Run quorum based on NN state
+ MembershipState representativeRecord =
+ getRepresentativeQuorum(nnRegistration);
+ String nnKey = representativeRecord.getNamenodeKey();
+ this.activeRegistrations.put(nnKey, representativeRecord);
+ }
+ LOG.debug("Refreshed {} NN registrations from State Store",
+ cachedRecords.size());
+ } finally {
+ cacheWriteLock.unlock();
+ }
+ return true;
+ }
+
+ @Override
+ public UpdateNamenodeRegistrationResponse updateNamenodeRegistration(
+ UpdateNamenodeRegistrationRequest request) throws IOException {
+
+ boolean status = false;
+ cacheWriteLock.lock();
+ try {
+ String namenode = MembershipState.getNamenodeKey(
+ request.getNameserviceId(), request.getNamenodeId());
+ MembershipState member = this.activeRegistrations.get(namenode);
+ if (member != null) {
+ member.setState(request.getState());
+ status = true;
+ }
+ } finally {
+ cacheWriteLock.unlock();
+ }
+ UpdateNamenodeRegistrationResponse response =
+ UpdateNamenodeRegistrationResponse.newInstance(status);
+ return response;
+ }
+
+ /**
+ * Picks the most recent entry in the subset that is most agreeable on the
+ * specified field. 1) If a majority of the collection has the same value for
+ * the field, the first sorted entry within the subset the matches the
+ * majority value 2) Otherwise the first sorted entry in the set of all
+ * entries
+ *
+ * @param records - Collection of state store record objects of the same type
+ * @return record that is most representative of the field name
+ */
+ private MembershipState getRepresentativeQuorum(
+ Collection<MembershipState> records) {
+
+ // Collate objects by field value: field value -> order set of records
+ Map<FederationNamenodeServiceState, TreeSet<MembershipState>> occurenceMap =
+ new HashMap<>();
+ for (MembershipState record : records) {
+ FederationNamenodeServiceState state = record.getState();
+ TreeSet<MembershipState> matchingSet = occurenceMap.get(state);
+ if (matchingSet == null) {
+ // TreeSet orders elements by descending date via comparators
+ matchingSet = new TreeSet<>();
+ occurenceMap.put(state, matchingSet);
+ }
+ matchingSet.add(record);
+ }
+
+ // Select largest group
+ TreeSet<MembershipState> largestSet = new TreeSet<>();
+ for (TreeSet<MembershipState> matchingSet : occurenceMap.values()) {
+ if (largestSet.size() < matchingSet.size()) {
+ largestSet = matchingSet;
+ }
+ }
+
+ // If quorum, use the newest element here
+ if (largestSet.size() > records.size() / 2) {
+ return largestSet.first();
+ // Otherwise, return most recent by class comparator
+ } else if (records.size() > 0) {
+ TreeSet<MembershipState> sortedList = new TreeSet<>(records);
+ LOG.debug("Quorum failed, using most recent: {}", sortedList.first());
+ return sortedList.first();
+ } else {
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MountTableStoreImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MountTableStoreImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MountTableStoreImpl.java
new file mode 100644
index 0000000..eb117d6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MountTableStoreImpl.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.impl;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.hdfs.server.federation.router.RouterAdminServer;
+import org.apache.hadoop.hdfs.server.federation.router.RouterPermissionChecker;
+import org.apache.hadoop.hdfs.server.federation.store.MountTableStore;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.hdfs.server.federation.store.records.Query;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.util.Time;
+
+/**
+ * Implementation of the {@link MountTableStore} state store API.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class MountTableStoreImpl extends MountTableStore {
+
+ public MountTableStoreImpl(StateStoreDriver driver) {
+ super(driver);
+ }
+
+ @Override
+ public AddMountTableEntryResponse addMountTableEntry(
+ AddMountTableEntryRequest request) throws IOException {
+ MountTable mountTable = request.getEntry();
+ if (mountTable != null) {
+ RouterPermissionChecker pc = RouterAdminServer.getPermissionChecker();
+ if (pc != null) {
+ pc.checkPermission(mountTable, FsAction.WRITE);
+ }
+ }
+
+ boolean status = getDriver().put(mountTable, false, true);
+ AddMountTableEntryResponse response =
+ AddMountTableEntryResponse.newInstance();
+ response.setStatus(status);
+ return response;
+ }
+
+ @Override
+ public UpdateMountTableEntryResponse updateMountTableEntry(
+ UpdateMountTableEntryRequest request) throws IOException {
+ MountTable mountTable = request.getEntry();
+ if (mountTable != null) {
+ RouterPermissionChecker pc = RouterAdminServer.getPermissionChecker();
+ if (pc != null) {
+ pc.checkPermission(mountTable, FsAction.WRITE);
+ }
+ }
+
+ boolean status = getDriver().put(mountTable, true, true);
+ UpdateMountTableEntryResponse response =
+ UpdateMountTableEntryResponse.newInstance();
+ response.setStatus(status);
+ return response;
+ }
+
+ @Override
+ public RemoveMountTableEntryResponse removeMountTableEntry(
+ RemoveMountTableEntryRequest request) throws IOException {
+ final String srcPath = request.getSrcPath();
+ final MountTable partial = MountTable.newInstance();
+ partial.setSourcePath(srcPath);
+ final Query<MountTable> query = new Query<>(partial);
+ final MountTable deleteEntry = getDriver().get(getRecordClass(), query);
+
+ boolean status = false;
+ if (deleteEntry != null) {
+ RouterPermissionChecker pc = RouterAdminServer.getPermissionChecker();
+ if (pc != null) {
+ pc.checkPermission(deleteEntry, FsAction.WRITE);
+ }
+ status = getDriver().remove(deleteEntry);
+ }
+
+ RemoveMountTableEntryResponse response =
+ RemoveMountTableEntryResponse.newInstance();
+ response.setStatus(status);
+ return response;
+ }
+
+ @Override
+ public GetMountTableEntriesResponse getMountTableEntries(
+ GetMountTableEntriesRequest request) throws IOException {
+ RouterPermissionChecker pc =
+ RouterAdminServer.getPermissionChecker();
+ // Get all values from the cache
+ List<MountTable> records = getCachedRecords();
+
+ // Sort and filter
+ Collections.sort(records, MountTable.SOURCE_COMPARATOR);
+ String reqSrcPath = request.getSrcPath();
+ if (reqSrcPath != null && !reqSrcPath.isEmpty()) {
+ // Return only entries beneath this path
+ Iterator<MountTable> it = records.iterator();
+ while (it.hasNext()) {
+ MountTable record = it.next();
+ String srcPath = record.getSourcePath();
+ if (!srcPath.startsWith(reqSrcPath)) {
+ it.remove();
+ } else if (pc != null) {
+ // do the READ permission check
+ try {
+ pc.checkPermission(record, FsAction.READ);
+ } catch (AccessControlException ignored) {
+ // Remove this mount table entry if it cannot
+ // be accessed by current user.
+ it.remove();
+ }
+ }
+ }
+ }
+
+ GetMountTableEntriesResponse response =
+ GetMountTableEntriesResponse.newInstance();
+ response.setEntries(records);
+ response.setTimestamp(Time.now());
+ return response;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/RouterStoreImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/RouterStoreImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/RouterStoreImpl.java
new file mode 100644
index 0000000..d58c288
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/RouterStoreImpl.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.impl;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.server.federation.store.RouterStore;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationsRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationsResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.Query;
+import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+
+/**
+ * Implementation of the {@link RouterStore} state store API.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class RouterStoreImpl extends RouterStore {
+
+ public RouterStoreImpl(StateStoreDriver driver) {
+ super(driver);
+ }
+
+ @Override
+ public GetRouterRegistrationResponse getRouterRegistration(
+ GetRouterRegistrationRequest request) throws IOException {
+
+ final RouterState partial = RouterState.newInstance();
+ partial.setAddress(request.getRouterId());
+ final Query<RouterState> query = new Query<RouterState>(partial);
+ RouterState record = getDriver().get(getRecordClass(), query);
+ if (record != null) {
+ overrideExpiredRecord(record);
+ }
+ GetRouterRegistrationResponse response =
+ GetRouterRegistrationResponse.newInstance();
+ response.setRouter(record);
+ return response;
+ }
+
+ @Override
+ public GetRouterRegistrationsResponse getRouterRegistrations(
+ GetRouterRegistrationsRequest request) throws IOException {
+
+ // Get all values from the cache
+ QueryResult<RouterState> recordsAndTimeStamp =
+ getCachedRecordsAndTimeStamp();
+ List<RouterState> records = recordsAndTimeStamp.getRecords();
+ long timestamp = recordsAndTimeStamp.getTimestamp();
+
+ // Generate response
+ GetRouterRegistrationsResponse response =
+ GetRouterRegistrationsResponse.newInstance();
+ response.setRouters(records);
+ response.setTimestamp(timestamp);
+ return response;
+ }
+
+ @Override
+ public RouterHeartbeatResponse routerHeartbeat(RouterHeartbeatRequest request)
+ throws IOException {
+
+ RouterState record = request.getRouter();
+ boolean status = getDriver().put(record, true, false);
+ RouterHeartbeatResponse response =
+ RouterHeartbeatResponse.newInstance(status);
+ return response;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/package-info.java
new file mode 100644
index 0000000..1a50d15
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/package-info.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Contains implementations of the state store API interfaces. All classes
+ * derive from {@link
+ * org.apache.hadoop.hdfs.server.federation.store.RecordStore}. The API
+ * definitions are contained in the
+ * org.apache.hadoop.hdfs.server.federation.store package.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+package org.apache.hadoop.hdfs.server.federation.store.impl;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/package-info.java
new file mode 100644
index 0000000..949ec7c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/package-info.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The federation state store tracks persistent values that are shared between
+ * multiple routers.
+ * <p>
+ * Data is stored in data records that inherit from a common class. Data records
+ * are serialized when written to the data store using a modular serialization
+ * implementation. The default is profobuf serialization. Data is stored as rows
+ * of records of the same type with each data member in a record representing a
+ * column.
+ * <p>
+ * The state store uses a modular data storage
+ * {@link org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver
+ * StateStoreDriver} to handle querying, updating and deleting data records. The
+ * data storage driver is initialized and maintained by the
+ * {@link org.apache.hadoop.hdfs.server.federation.store.
+ * StateStoreService FederationStateStoreService}. The state store
+ * supports fetching all records of a type, filtering by column values or
+ * fetching a single record by its primary key.
+ * <p>
+ * The state store contains several API interfaces, one for each data records
+ * type.
+ * <p>
+ * <ul>
+ * <li>FederationMembershipStateStore: state of all Namenodes in the federation.
+ * Uses the MembershipState record.
+ * <li>FederationMountTableStore: Mount table mapping paths in the global
+ * namespace to individual subcluster paths. Uses the MountTable record.
+ * <li>RouterStateStore: State of all routers in the federation. Uses the
+ * RouterState record.
+ * </ul>
+ * <p>
+ * Each API is defined in a separate interface. The implementations of these
+ * interfaces are responsible for accessing the
+ * {@link org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver
+ * StateStoreDriver} to query, update and delete data records.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntryRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntryRequest.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntryRequest.java
new file mode 100644
index 0000000..2d9f102
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntryRequest.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+
+/**
+ * API request for adding a mount table entry to the state store.
+ */
+public abstract class AddMountTableEntryRequest {
+
+ public static AddMountTableEntryRequest newInstance() {
+ return StateStoreSerializer.newRecord(AddMountTableEntryRequest.class);
+ }
+
+ public static AddMountTableEntryRequest newInstance(MountTable newEntry) {
+ AddMountTableEntryRequest request = newInstance();
+ request.setEntry(newEntry);
+ return request;
+ }
+
+ @Public
+ @Unstable
+ public abstract MountTable getEntry();
+
+ @Public
+ @Unstable
+ public abstract void setEntry(MountTable mount);
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
deleted file mode 100644
index a80c3be..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
+++ /dev/null
@@ -1,766 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.metrics;
-
-import static org.apache.hadoop.util.Time.now;
-
-import java.io.IOException;
-import java.lang.reflect.Method;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.UnknownHostException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.function.Function;
-import java.util.function.ToIntFunction;
-import java.util.function.ToLongFunction;
-import java.util.stream.Collectors;
-
-import javax.management.NotCompliantMBeanException;
-import javax.management.ObjectName;
-import javax.management.StandardMBean;
-
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
-import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
-import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
-import org.apache.hadoop.hdfs.server.federation.router.Router;
-import org.apache.hadoop.hdfs.server.federation.router.RouterRpcServer;
-import org.apache.hadoop.hdfs.server.federation.store.MembershipStore;
-import org.apache.hadoop.hdfs.server.federation.store.MountTableStore;
-import org.apache.hadoop.hdfs.server.federation.store.RouterStore;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationsRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationsResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
-import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
-import org.apache.hadoop.hdfs.server.federation.store.records.MembershipStats;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
-import org.apache.hadoop.hdfs.server.federation.store.records.StateStoreVersion;
-import org.apache.hadoop.metrics2.util.MBeans;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.VersionInfo;
-import org.codehaus.jettison.json.JSONObject;
-import org.eclipse.jetty.util.ajax.JSON;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.annotations.VisibleForTesting;
-
-/**
- * Implementation of the Router metrics collector.
- */
-public class FederationMetrics implements FederationMBean {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(FederationMetrics.class);
-
- /** Format for a date. */
- private static final String DATE_FORMAT = "yyyy/MM/dd HH:mm:ss";
-
- /** Prevent holding the page from load too long. */
- private static final long TIME_OUT = TimeUnit.SECONDS.toMillis(1);
-
-
- /** Router interface. */
- private final Router router;
-
- /** FederationState JMX bean. */
- private ObjectName beanName;
-
- /** Resolve the namenode for each namespace. */
- private final ActiveNamenodeResolver namenodeResolver;
-
- /** State store. */
- private final StateStoreService stateStore;
- /** Membership state store. */
- private MembershipStore membershipStore;
- /** Mount table store. */
- private MountTableStore mountTableStore;
- /** Router state store. */
- private RouterStore routerStore;
-
-
- public FederationMetrics(Router router) throws IOException {
- this.router = router;
-
- try {
- StandardMBean bean = new StandardMBean(this, FederationMBean.class);
- this.beanName = MBeans.register("Router", "FederationState", bean);
- LOG.info("Registered Router MBean: {}", this.beanName);
- } catch (NotCompliantMBeanException e) {
- throw new RuntimeException("Bad Router MBean setup", e);
- }
-
- // Resolve namenode for each nameservice
- this.namenodeResolver = this.router.getNamenodeResolver();
-
- // State store interfaces
- this.stateStore = this.router.getStateStore();
- if (this.stateStore == null) {
- LOG.error("State store not available");
- } else {
- this.membershipStore = stateStore.getRegisteredRecordStore(
- MembershipStore.class);
- this.mountTableStore = stateStore.getRegisteredRecordStore(
- MountTableStore.class);
- this.routerStore = stateStore.getRegisteredRecordStore(
- RouterStore.class);
- }
- }
-
- /**
- * Unregister the JMX beans.
- */
- public void close() {
- if (this.beanName != null) {
- MBeans.unregister(beanName);
- }
- }
-
- @Override
- public String getNamenodes() {
- final Map<String, Map<String, Object>> info = new LinkedHashMap<>();
- try {
- // Get the values from the store
- GetNamenodeRegistrationsRequest request =
- GetNamenodeRegistrationsRequest.newInstance();
- GetNamenodeRegistrationsResponse response =
- membershipStore.getNamenodeRegistrations(request);
-
- // Order the namenodes
- final List<MembershipState> namenodes = response.getNamenodeMemberships();
- if (namenodes == null || namenodes.size() == 0) {
- return JSON.toString(info);
- }
- List<MembershipState> namenodesOrder = new ArrayList<>(namenodes);
- Collections.sort(namenodesOrder, MembershipState.NAME_COMPARATOR);
-
- // Dump namenodes information into JSON
- for (MembershipState namenode : namenodesOrder) {
- Map<String, Object> innerInfo = new HashMap<>();
- Map<String, Object> map = getJson(namenode);
- innerInfo.putAll(map);
- long dateModified = namenode.getDateModified();
- long lastHeartbeat = getSecondsSince(dateModified);
- innerInfo.put("lastHeartbeat", lastHeartbeat);
- MembershipStats stats = namenode.getStats();
- long used = stats.getTotalSpace() - stats.getAvailableSpace();
- innerInfo.put("used", used);
- info.put(namenode.getNamenodeKey(),
- Collections.unmodifiableMap(innerInfo));
- }
- } catch (IOException e) {
- LOG.error("Enable to fetch json representation of namenodes {}",
- e.getMessage());
- return "{}";
- }
- return JSON.toString(info);
- }
-
- @Override
- public String getNameservices() {
- final Map<String, Map<String, Object>> info = new LinkedHashMap<>();
- try {
- final List<MembershipState> namenodes = getActiveNamenodeRegistrations();
- List<MembershipState> namenodesOrder = new ArrayList<>(namenodes);
- Collections.sort(namenodesOrder, MembershipState.NAME_COMPARATOR);
-
- // Dump namenodes information into JSON
- for (MembershipState namenode : namenodesOrder) {
- Map<String, Object> innerInfo = new HashMap<>();
- Map<String, Object> map = getJson(namenode);
- innerInfo.putAll(map);
- long dateModified = namenode.getDateModified();
- long lastHeartbeat = getSecondsSince(dateModified);
- innerInfo.put("lastHeartbeat", lastHeartbeat);
- MembershipStats stats = namenode.getStats();
- long used = stats.getTotalSpace() - stats.getAvailableSpace();
- innerInfo.put("used", used);
- info.put(namenode.getNamenodeKey(),
- Collections.unmodifiableMap(innerInfo));
- }
- } catch (IOException e) {
- LOG.error("Cannot retrieve nameservices for JMX: {}", e.getMessage());
- return "{}";
- }
- return JSON.toString(info);
- }
-
- @Override
- public String getMountTable() {
- final List<Map<String, Object>> info = new LinkedList<>();
-
- try {
- // Get all the mount points in order
- GetMountTableEntriesRequest request =
- GetMountTableEntriesRequest.newInstance("/");
- GetMountTableEntriesResponse response =
- mountTableStore.getMountTableEntries(request);
- final List<MountTable> mounts = response.getEntries();
- List<MountTable> orderedMounts = new ArrayList<>(mounts);
- Collections.sort(orderedMounts, MountTable.SOURCE_COMPARATOR);
-
- // Dump mount table entries information into JSON
- for (MountTable entry : orderedMounts) {
- // Sumarize destinations
- Set<String> nameservices = new LinkedHashSet<>();
- Set<String> paths = new LinkedHashSet<>();
- for (RemoteLocation location : entry.getDestinations()) {
- nameservices.add(location.getNameserviceId());
- paths.add(location.getDest());
- }
-
- Map<String, Object> map = getJson(entry);
- // We add some values with a cleaner format
- map.put("dateCreated", getDateString(entry.getDateCreated()));
- map.put("dateModified", getDateString(entry.getDateModified()));
-
- Map<String, Object> innerInfo = new HashMap<>();
- innerInfo.putAll(map);
- innerInfo.put("nameserviceId", StringUtils.join(",", nameservices));
- innerInfo.put("path", StringUtils.join(",", paths));
- if (nameservices.size() > 1) {
- innerInfo.put("order", entry.getDestOrder().toString());
- } else {
- innerInfo.put("order", "");
- }
- innerInfo.put("readonly", entry.isReadOnly());
- info.add(Collections.unmodifiableMap(innerInfo));
- }
- } catch (IOException e) {
- LOG.error(
- "Cannot generate JSON of mount table from store: {}", e.getMessage());
- return "[]";
- }
- return JSON.toString(info);
- }
-
- @Override
- public String getRouters() {
- final Map<String, Map<String, Object>> info = new LinkedHashMap<>();
- try {
- // Get all the routers in order
- GetRouterRegistrationsRequest request =
- GetRouterRegistrationsRequest.newInstance();
- GetRouterRegistrationsResponse response =
- routerStore.getRouterRegistrations(request);
- final List<RouterState> routers = response.getRouters();
- List<RouterState> routersOrder = new ArrayList<>(routers);
- Collections.sort(routersOrder);
-
- // Dump router information into JSON
- for (RouterState record : routersOrder) {
- Map<String, Object> innerInfo = new HashMap<>();
- Map<String, Object> map = getJson(record);
- innerInfo.putAll(map);
- long dateModified = record.getDateModified();
- long lastHeartbeat = getSecondsSince(dateModified);
- innerInfo.put("lastHeartbeat", lastHeartbeat);
-
- StateStoreVersion stateStoreVersion = record.getStateStoreVersion();
- if (stateStoreVersion == null) {
- LOG.error("Cannot get State Store versions");
- } else {
- setStateStoreVersions(innerInfo, stateStoreVersion);
- }
-
- info.put(record.getPrimaryKey(),
- Collections.unmodifiableMap(innerInfo));
- }
- } catch (IOException e) {
- LOG.error("Cannot get Routers JSON from the State Store", e);
- return "{}";
- }
- return JSON.toString(info);
- }
-
- /**
- * Populate the map with the State Store versions.
- *
- * @param innerInfo Map with the information.
- * @param version State Store versions.
- */
- private static void setStateStoreVersions(
- Map<String, Object> map, StateStoreVersion version) {
-
- long membershipVersion = version.getMembershipVersion();
- String lastMembershipUpdate = getDateString(membershipVersion);
- map.put("lastMembershipUpdate", lastMembershipUpdate);
-
- long mountTableVersion = version.getMountTableVersion();
- String lastMountTableDate = getDateString(mountTableVersion);
- map.put("lastMountTableUpdate", lastMountTableDate);
- }
-
- @Override
- public long getTotalCapacity() {
- return getNameserviceAggregatedLong(MembershipStats::getTotalSpace);
- }
-
- @Override
- public long getRemainingCapacity() {
- return getNameserviceAggregatedLong(MembershipStats::getAvailableSpace);
- }
-
- @Override
- public long getProvidedSpace() {
- return getNameserviceAggregatedLong(MembershipStats::getProvidedSpace);
- }
-
- @Override
- public long getUsedCapacity() {
- return getTotalCapacity() - getRemainingCapacity();
- }
-
- @Override
- public int getNumNameservices() {
- try {
- Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
- return nss.size();
- } catch (IOException e) {
- LOG.error(
- "Cannot fetch number of expired registrations from the store: {}",
- e.getMessage());
- return 0;
- }
- }
-
- @Override
- public int getNumNamenodes() {
- try {
- GetNamenodeRegistrationsRequest request =
- GetNamenodeRegistrationsRequest.newInstance();
- GetNamenodeRegistrationsResponse response =
- membershipStore.getNamenodeRegistrations(request);
- List<MembershipState> memberships = response.getNamenodeMemberships();
- return memberships.size();
- } catch (IOException e) {
- LOG.error("Cannot retrieve numNamenodes for JMX: {}", e.getMessage());
- return 0;
- }
- }
-
- @Override
- public int getNumExpiredNamenodes() {
- try {
- GetNamenodeRegistrationsRequest request =
- GetNamenodeRegistrationsRequest.newInstance();
- GetNamenodeRegistrationsResponse response =
- membershipStore.getExpiredNamenodeRegistrations(request);
- List<MembershipState> expiredMemberships =
- response.getNamenodeMemberships();
- return expiredMemberships.size();
- } catch (IOException e) {
- LOG.error(
- "Cannot retrieve numExpiredNamenodes for JMX: {}", e.getMessage());
- return 0;
- }
- }
-
- @Override
- public int getNumLiveNodes() {
- return getNameserviceAggregatedInt(
- MembershipStats::getNumOfActiveDatanodes);
- }
-
- @Override
- public int getNumDeadNodes() {
- return getNameserviceAggregatedInt(MembershipStats::getNumOfDeadDatanodes);
- }
-
- @Override
- public int getNumDecommissioningNodes() {
- return getNameserviceAggregatedInt(
- MembershipStats::getNumOfDecommissioningDatanodes);
- }
-
- @Override
- public int getNumDecomLiveNodes() {
- return getNameserviceAggregatedInt(
- MembershipStats::getNumOfDecomActiveDatanodes);
- }
-
- @Override
- public int getNumDecomDeadNodes() {
- return getNameserviceAggregatedInt(
- MembershipStats::getNumOfDecomDeadDatanodes);
- }
-
- @Override // NameNodeMXBean
- public String getNodeUsage() {
- float median = 0;
- float max = 0;
- float min = 0;
- float dev = 0;
-
- final Map<String, Map<String, Object>> info = new HashMap<>();
- try {
- RouterRpcServer rpcServer = this.router.getRpcServer();
- DatanodeInfo[] live = rpcServer.getDatanodeReport(
- DatanodeReportType.LIVE, TIME_OUT);
-
- if (live.length > 0) {
- float totalDfsUsed = 0;
- float[] usages = new float[live.length];
- int i = 0;
- for (DatanodeInfo dn : live) {
- usages[i++] = dn.getDfsUsedPercent();
- totalDfsUsed += dn.getDfsUsedPercent();
- }
- totalDfsUsed /= live.length;
- Arrays.sort(usages);
- median = usages[usages.length / 2];
- max = usages[usages.length - 1];
- min = usages[0];
-
- for (i = 0; i < usages.length; i++) {
- dev += (usages[i] - totalDfsUsed) * (usages[i] - totalDfsUsed);
- }
- dev = (float) Math.sqrt(dev / usages.length);
- }
- } catch (IOException e) {
- LOG.info("Cannot get the live nodes: {}", e.getMessage());
- }
-
- final Map<String, Object> innerInfo = new HashMap<>();
- innerInfo.put("min", StringUtils.format("%.2f%%", min));
- innerInfo.put("median", StringUtils.format("%.2f%%", median));
- innerInfo.put("max", StringUtils.format("%.2f%%", max));
- innerInfo.put("stdDev", StringUtils.format("%.2f%%", dev));
- info.put("nodeUsage", innerInfo);
-
- return JSON.toString(info);
- }
-
- @Override
- public long getNumBlocks() {
- return getNameserviceAggregatedLong(MembershipStats::getNumOfBlocks);
- }
-
- @Override
- public long getNumOfMissingBlocks() {
- return getNameserviceAggregatedLong(MembershipStats::getNumOfBlocksMissing);
- }
-
- @Override
- public long getNumOfBlocksPendingReplication() {
- return getNameserviceAggregatedLong(
- MembershipStats::getNumOfBlocksPendingReplication);
- }
-
- @Override
- public long getNumOfBlocksUnderReplicated() {
- return getNameserviceAggregatedLong(
- MembershipStats::getNumOfBlocksUnderReplicated);
- }
-
- @Override
- public long getNumOfBlocksPendingDeletion() {
- return getNameserviceAggregatedLong(
- MembershipStats::getNumOfBlocksPendingDeletion);
- }
-
- @Override
- public long getNumFiles() {
- return getNameserviceAggregatedLong(MembershipStats::getNumOfFiles);
- }
-
- @Override
- public String getRouterStarted() {
- long startTime = this.router.getStartTime();
- return new Date(startTime).toString();
- }
-
- @Override
- public String getVersion() {
- return VersionInfo.getVersion() + ", r" + VersionInfo.getRevision();
- }
-
- @Override
- public String getCompiledDate() {
- return VersionInfo.getDate();
- }
-
- @Override
- public String getCompileInfo() {
- return VersionInfo.getDate() + " by " + VersionInfo.getUser() + " from "
- + VersionInfo.getBranch();
- }
-
- @Override
- public String getHostAndPort() {
- InetSocketAddress address = this.router.getHttpServerAddress();
- if (address != null) {
- try {
- String hostname = InetAddress.getLocalHost().getHostName();
- int port = address.getPort();
- return hostname + ":" + port;
- } catch (UnknownHostException ignored) { }
- }
- return "Unknown";
- }
-
- @Override
- public String getRouterId() {
- return this.router.getRouterId();
- }
-
- @Override
- public String getClusterId() {
- try {
- Collection<String> clusterIds =
- getNamespaceInfo(FederationNamespaceInfo::getClusterId);
- return clusterIds.toString();
- } catch (IOException e) {
- LOG.error("Cannot fetch cluster ID metrics: {}", e.getMessage());
- return "";
- }
- }
-
- @Override
- public String getBlockPoolId() {
- try {
- Collection<String> blockpoolIds =
- getNamespaceInfo(FederationNamespaceInfo::getBlockPoolId);
- return blockpoolIds.toString();
- } catch (IOException e) {
- LOG.error("Cannot fetch block pool ID metrics: {}", e.getMessage());
- return "";
- }
- }
-
- @Override
- public String getRouterStatus() {
- return "RUNNING";
- }
-
- /**
- * Build a set of unique values found in all namespaces.
- *
- * @param f Method reference of the appropriate FederationNamespaceInfo
- * getter function
- * @return Set of unique string values found in all discovered namespaces.
- * @throws IOException if the query could not be executed.
- */
- private Collection<String> getNamespaceInfo(
- Function<FederationNamespaceInfo, String> f) throws IOException {
- GetNamespaceInfoRequest request = GetNamespaceInfoRequest.newInstance();
- GetNamespaceInfoResponse response =
- membershipStore.getNamespaceInfo(request);
- return response.getNamespaceInfo().stream()
- .map(f)
- .collect(Collectors.toSet());
- }
-
- /**
- * Get the aggregated value for a method for all nameservices.
- * @param f Method reference
- * @return Aggregated integer.
- */
- private int getNameserviceAggregatedInt(ToIntFunction<MembershipStats> f) {
- try {
- return getActiveNamenodeRegistrations().stream()
- .map(MembershipState::getStats)
- .collect(Collectors.summingInt(f));
- } catch (IOException e) {
- LOG.error("Unable to extract metrics: {}", e.getMessage());
- return 0;
- }
- }
-
- /**
- * Get the aggregated value for a method for all nameservices.
- * @param f Method reference
- * @return Aggregated long.
- */
- private long getNameserviceAggregatedLong(ToLongFunction<MembershipStats> f) {
- try {
- return getActiveNamenodeRegistrations().stream()
- .map(MembershipState::getStats)
- .collect(Collectors.summingLong(f));
- } catch (IOException e) {
- LOG.error("Unable to extract metrics: {}", e.getMessage());
- return 0;
- }
- }
-
- /**
- * Fetches the most active namenode memberships for all known nameservices.
- * The fetched membership may not or may not be active. Excludes expired
- * memberships.
- * @throws IOException if the query could not be performed.
- * @return List of the most active NNs from each known nameservice.
- */
- private List<MembershipState> getActiveNamenodeRegistrations()
- throws IOException {
-
- List<MembershipState> resultList = new ArrayList<>();
- GetNamespaceInfoRequest request = GetNamespaceInfoRequest.newInstance();
- GetNamespaceInfoResponse response =
- membershipStore.getNamespaceInfo(request);
- for (FederationNamespaceInfo nsInfo : response.getNamespaceInfo()) {
- // Fetch the most recent namenode registration
- String nsId = nsInfo.getNameserviceId();
- List<? extends FederationNamenodeContext> nns =
- namenodeResolver.getNamenodesForNameserviceId(nsId);
- if (nns != null) {
- FederationNamenodeContext nn = nns.get(0);
- if (nn != null && nn instanceof MembershipState) {
- resultList.add((MembershipState) nn);
- }
- }
- }
- return resultList;
- }
-
- /**
- * Get time as a date string.
- * @param time Seconds since 1970.
- * @return String representing the date.
- */
- @VisibleForTesting
- static String getDateString(long time) {
- if (time <= 0) {
- return "-";
- }
- Date date = new Date(time);
-
- SimpleDateFormat sdf = new SimpleDateFormat(DATE_FORMAT);
- return sdf.format(date);
- }
-
- /**
- * Get the number of seconds passed since a date.
- *
- * @param timeMs to use as a reference.
- * @return Seconds since the date.
- */
- private static long getSecondsSince(long timeMs) {
- if (timeMs < 0) {
- return -1;
- }
- return (now() - timeMs) / 1000;
- }
-
- /**
- * Get JSON for this record.
- *
- * @return Map representing the data for the JSON representation.
- */
- private static Map<String, Object> getJson(BaseRecord record) {
- Map<String, Object> json = new HashMap<>();
- Map<String, Class<?>> fields = getFields(record);
-
- for (String fieldName : fields.keySet()) {
- if (!fieldName.equalsIgnoreCase("proto")) {
- try {
- Object value = getField(record, fieldName);
- if (value instanceof BaseRecord) {
- BaseRecord recordField = (BaseRecord) value;
- json.putAll(getJson(recordField));
- } else {
- json.put(fieldName, value == null ? JSONObject.NULL : value);
- }
- } catch (Exception e) {
- throw new IllegalArgumentException(
- "Cannot serialize field " + fieldName + " into JSON");
- }
- }
- }
- return json;
- }
-
- /**
- * Returns all serializable fields in the object.
- *
- * @return Map with the fields.
- */
- private static Map<String, Class<?>> getFields(BaseRecord record) {
- Map<String, Class<?>> getters = new HashMap<>();
- for (Method m : record.getClass().getDeclaredMethods()) {
- if (m.getName().startsWith("get")) {
- try {
- Class<?> type = m.getReturnType();
- char[] c = m.getName().substring(3).toCharArray();
- c[0] = Character.toLowerCase(c[0]);
- String key = new String(c);
- getters.put(key, type);
- } catch (Exception e) {
- LOG.error("Cannot execute getter {} on {}", m.getName(), record);
- }
- }
- }
- return getters;
- }
-
- /**
- * Fetches the value for a field name.
- *
- * @param fieldName the legacy name of the field.
- * @return The field data or null if not found.
- */
- private static Object getField(BaseRecord record, String fieldName) {
- Object result = null;
- Method m = locateGetter(record, fieldName);
- if (m != null) {
- try {
- result = m.invoke(record);
- } catch (Exception e) {
- LOG.error("Cannot get field {} on {}", fieldName, record);
- }
- }
- return result;
- }
-
- /**
- * Finds the appropriate getter for a field name.
- *
- * @param fieldName The legacy name of the field.
- * @return The matching getter or null if not found.
- */
- private static Method locateGetter(BaseRecord record, String fieldName) {
- for (Method m : record.getClass().getMethods()) {
- if (m.getName().equalsIgnoreCase("get" + fieldName)) {
- return m;
- }
- }
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java
deleted file mode 100644
index 3e031fe..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.metrics;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * JMX interface for the RPC server.
- * TODO use the default RPC MBean.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public interface FederationRPCMBean {
-
- long getProxyOps();
-
- double getProxyAvg();
-
- long getProcessingOps();
-
- double getProcessingAvg();
-
- long getProxyOpFailureCommunicate();
-
- long getProxyOpFailureStandby();
-
- long getProxyOpNotImplemented();
-
- long getProxyOpRetries();
-
- long getRouterFailureStateStoreOps();
-
- long getRouterFailureReadOnlyOps();
-
- long getRouterFailureLockedOps();
-
- long getRouterFailureSafemodeOps();
-
- int getRpcServerCallQueue();
-
- /**
- * Get the number of RPC connections between the clients and the Router.
- * @return Number of RPC connections between the clients and the Router.
- */
- int getRpcServerNumOpenConnections();
-
- /**
- * Get the number of RPC connections between the Router and the NNs.
- * @return Number of RPC connections between the Router and the NNs.
- */
- int getRpcClientNumConnections();
-
- /**
- * Get the number of active RPC connections between the Router and the NNs.
- * @return Number of active RPC connections between the Router and the NNs.
- */
- int getRpcClientNumActiveConnections();
-
- /**
- * Get the number of RPC connections to be created.
- * @return Number of RPC connections to be created.
- */
- int getRpcClientNumCreatingConnections();
-
- /**
- * Get the number of connection pools between the Router and a NNs.
- * @return Number of connection pools between the Router and a NNs.
- */
- int getRpcClientNumConnectionPools();
-
- /**
- * JSON representation of the RPC connections from the Router to the NNs.
- * @return JSON string representation.
- */
- String getRpcClientConnections();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java
deleted file mode 100644
index 94d3383..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java
+++ /dev/null
@@ -1,250 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.metrics;
-
-import static org.apache.hadoop.metrics2.impl.MsInfo.ProcessName;
-import static org.apache.hadoop.metrics2.impl.MsInfo.SessionId;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.server.federation.router.RouterRpcServer;
-import org.apache.hadoop.metrics2.MetricsSystem;
-import org.apache.hadoop.metrics2.annotation.Metric;
-import org.apache.hadoop.metrics2.annotation.Metrics;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.lib.MetricsRegistry;
-import org.apache.hadoop.metrics2.lib.MutableCounterLong;
-import org.apache.hadoop.metrics2.lib.MutableRate;
-
-/**
- * Implementation of the RPC metrics collector.
- */
-@Metrics(name = "RouterRPCActivity", about = "Router RPC Activity",
- context = "dfs")
-public class FederationRPCMetrics implements FederationRPCMBean {
-
- private final MetricsRegistry registry = new MetricsRegistry("router");
-
- private RouterRpcServer rpcServer;
-
- @Metric("Time for the router to process an operation internally")
- private MutableRate processing;
- @Metric("Number of operations the Router processed internally")
- private MutableCounterLong processingOp;
- @Metric("Time for the Router to proxy an operation to the Namenodes")
- private MutableRate proxy;
- @Metric("Number of operations the Router proxied to a Namenode")
- private MutableCounterLong proxyOp;
-
- @Metric("Number of operations to fail to reach NN")
- private MutableCounterLong proxyOpFailureStandby;
- @Metric("Number of operations to hit a standby NN")
- private MutableCounterLong proxyOpFailureCommunicate;
- @Metric("Number of operations not implemented")
- private MutableCounterLong proxyOpNotImplemented;
- @Metric("Number of operation retries")
- private MutableCounterLong proxyOpRetries;
-
- @Metric("Failed requests due to State Store unavailable")
- private MutableCounterLong routerFailureStateStore;
- @Metric("Failed requests due to read only mount point")
- private MutableCounterLong routerFailureReadOnly;
- @Metric("Failed requests due to locked path")
- private MutableCounterLong routerFailureLocked;
- @Metric("Failed requests due to safe mode")
- private MutableCounterLong routerFailureSafemode;
-
- public FederationRPCMetrics(Configuration conf, RouterRpcServer rpcServer) {
- this.rpcServer = rpcServer;
-
- registry.tag(SessionId, "RouterRPCSession");
- registry.tag(ProcessName, "Router");
- }
-
- public static FederationRPCMetrics create(Configuration conf,
- RouterRpcServer rpcServer) {
- MetricsSystem ms = DefaultMetricsSystem.instance();
- return ms.register(FederationRPCMetrics.class.getName(),
- "HDFS Federation RPC Metrics",
- new FederationRPCMetrics(conf, rpcServer));
- }
-
- /**
- * Convert nanoseconds to milliseconds.
- * @param ns Time in nanoseconds.
- * @return Time in milliseconds.
- */
- private static double toMs(double ns) {
- return ns / 1000000;
- }
-
- /**
- * Reset the metrics system.
- */
- public static void reset() {
- MetricsSystem ms = DefaultMetricsSystem.instance();
- ms.unregisterSource(FederationRPCMetrics.class.getName());
- }
-
- public void incrProxyOpFailureStandby() {
- proxyOpFailureStandby.incr();
- }
-
- @Override
- public long getProxyOpFailureStandby() {
- return proxyOpFailureStandby.value();
- }
-
- public void incrProxyOpFailureCommunicate() {
- proxyOpFailureCommunicate.incr();
- }
-
- @Override
- public long getProxyOpFailureCommunicate() {
- return proxyOpFailureCommunicate.value();
- }
-
-
- public void incrProxyOpNotImplemented() {
- proxyOpNotImplemented.incr();
- }
-
- @Override
- public long getProxyOpNotImplemented() {
- return proxyOpNotImplemented.value();
- }
-
- public void incrProxyOpRetries() {
- proxyOpRetries.incr();
- }
-
- @Override
- public long getProxyOpRetries() {
- return proxyOpRetries.value();
- }
-
- public void incrRouterFailureStateStore() {
- routerFailureStateStore.incr();
- }
-
- @Override
- public long getRouterFailureStateStoreOps() {
- return routerFailureStateStore.value();
- }
-
- public void incrRouterFailureSafemode() {
- routerFailureSafemode.incr();
- }
-
- @Override
- public long getRouterFailureSafemodeOps() {
- return routerFailureSafemode.value();
- }
-
- public void incrRouterFailureReadOnly() {
- routerFailureReadOnly.incr();
- }
-
- @Override
- public long getRouterFailureReadOnlyOps() {
- return routerFailureReadOnly.value();
- }
-
- public void incrRouterFailureLocked() {
- routerFailureLocked.incr();
- }
-
- @Override
- public long getRouterFailureLockedOps() {
- return routerFailureLocked.value();
- }
-
- @Override
- public int getRpcServerCallQueue() {
- return rpcServer.getServer().getCallQueueLen();
- }
-
- @Override
- public int getRpcServerNumOpenConnections() {
- return rpcServer.getServer().getNumOpenConnections();
- }
-
- @Override
- public int getRpcClientNumConnections() {
- return rpcServer.getRPCClient().getNumConnections();
- }
-
- @Override
- public int getRpcClientNumActiveConnections() {
- return rpcServer.getRPCClient().getNumActiveConnections();
- }
-
- @Override
- public int getRpcClientNumCreatingConnections() {
- return rpcServer.getRPCClient().getNumCreatingConnections();
- }
-
- @Override
- public int getRpcClientNumConnectionPools() {
- return rpcServer.getRPCClient().getNumConnectionPools();
- }
-
- @Override
- public String getRpcClientConnections() {
- return rpcServer.getRPCClient().getJSON();
- }
-
- /**
- * Add the time to proxy an operation from the moment the Router sends it to
- * the Namenode until it replied.
- * @param time Proxy time of an operation in nanoseconds.
- */
- public void addProxyTime(long time) {
- proxy.add(time);
- proxyOp.incr();
- }
-
- @Override
- public double getProxyAvg() {
- return toMs(proxy.lastStat().mean());
- }
-
- @Override
- public long getProxyOps() {
- return proxyOp.value();
- }
-
- /**
- * Add the time to process a request in the Router from the time we receive
- * the call until we send it to the Namenode.
- * @param time Process time of an operation in nanoseconds.
- */
- public void addProcessingTime(long time) {
- processing.add(time);
- processingOp.incr();
- }
-
- @Override
- public double getProcessingAvg() {
- return toMs(processing.lastStat().mean());
- }
-
- @Override
- public long getProcessingOps() {
- return processingOp.value();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCPerformanceMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCPerformanceMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCPerformanceMonitor.java
deleted file mode 100644
index 547ebb5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCPerformanceMonitor.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.metrics;
-
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ThreadFactory;
-
-import javax.management.NotCompliantMBeanException;
-import javax.management.ObjectName;
-import javax.management.StandardMBean;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.server.federation.router.RouterRpcMonitor;
-import org.apache.hadoop.hdfs.server.federation.router.RouterRpcServer;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
-import org.apache.hadoop.metrics2.util.MBeans;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-
-/**
- * Customizable RPC performance monitor. Receives events from the RPC server
- * and aggregates them via JMX.
- */
-public class FederationRPCPerformanceMonitor implements RouterRpcMonitor {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(FederationRPCPerformanceMonitor.class);
-
-
- /** Time for an operation to be received in the Router. */
- private static final ThreadLocal<Long> START_TIME = new ThreadLocal<>();
- /** Time for an operation to be send to the Namenode. */
- private static final ThreadLocal<Long> PROXY_TIME = new ThreadLocal<>();
-
- /** Configuration for the performance monitor. */
- private Configuration conf;
- /** RPC server for the Router. */
- private RouterRpcServer server;
- /** State Store. */
- private StateStoreService store;
-
- /** JMX interface to monitor the RPC metrics. */
- private FederationRPCMetrics metrics;
- private ObjectName registeredBean;
-
- /** Thread pool for logging stats. */
- private ExecutorService executor;
-
-
- @Override
- public void init(Configuration configuration, RouterRpcServer rpcServer,
- StateStoreService stateStore) {
-
- this.conf = configuration;
- this.server = rpcServer;
- this.store = stateStore;
-
- // Create metrics
- this.metrics = FederationRPCMetrics.create(conf, server);
-
- // Create thread pool
- ThreadFactory threadFactory = new ThreadFactoryBuilder()
- .setNameFormat("Federation RPC Performance Monitor-%d").build();
- this.executor = Executors.newFixedThreadPool(1, threadFactory);
-
- // Adding JMX interface
- try {
- StandardMBean bean =
- new StandardMBean(this.metrics, FederationRPCMBean.class);
- registeredBean = MBeans.register("Router", "FederationRPC", bean);
- LOG.info("Registered FederationRPCMBean: {}", registeredBean);
- } catch (NotCompliantMBeanException e) {
- throw new RuntimeException("Bad FederationRPCMBean setup", e);
- }
- }
-
- @Override
- public void close() {
- if (registeredBean != null) {
- MBeans.unregister(registeredBean);
- registeredBean = null;
- }
- if (this.executor != null) {
- this.executor.shutdown();
- }
- }
-
- /**
- * Resets all RPC service performance counters to their defaults.
- */
- public void resetPerfCounters() {
- if (registeredBean != null) {
- MBeans.unregister(registeredBean);
- registeredBean = null;
- }
- if (metrics != null) {
- FederationRPCMetrics.reset();
- metrics = null;
- }
- init(conf, server, store);
- }
-
- @Override
- public void startOp() {
- START_TIME.set(this.getNow());
- }
-
- @Override
- public long proxyOp() {
- PROXY_TIME.set(this.getNow());
- long processingTime = getProcessingTime();
- if (processingTime >= 0) {
- metrics.addProcessingTime(processingTime);
- }
- return Thread.currentThread().getId();
- }
-
- @Override
- public void proxyOpComplete(boolean success) {
- if (success) {
- long proxyTime = getProxyTime();
- if (proxyTime >= 0) {
- metrics.addProxyTime(proxyTime);
- }
- }
- }
-
- @Override
- public void proxyOpFailureStandby() {
- metrics.incrProxyOpFailureStandby();
- }
-
- @Override
- public void proxyOpFailureCommunicate() {
- metrics.incrProxyOpFailureCommunicate();
- }
-
- @Override
- public void proxyOpNotImplemented() {
- metrics.incrProxyOpNotImplemented();
- }
-
- @Override
- public void proxyOpRetries() {
- metrics.incrProxyOpRetries();
- }
-
- @Override
- public void routerFailureStateStore() {
- metrics.incrRouterFailureStateStore();
- }
-
- @Override
- public void routerFailureSafemode() {
- metrics.incrRouterFailureSafemode();
- }
-
- @Override
- public void routerFailureReadOnly() {
- metrics.incrRouterFailureReadOnly();
- }
-
- @Override
- public void routerFailureLocked() {
- metrics.incrRouterFailureLocked();
- }
-
- /**
- * Get current time.
- * @return Current time in nanoseconds.
- */
- private long getNow() {
- return System.nanoTime();
- }
-
- /**
- * Get time between we receiving the operation and sending it to the Namenode.
- * @return Processing time in nanoseconds.
- */
- private long getProcessingTime() {
- if (START_TIME.get() != null && START_TIME.get() > 0 &&
- PROXY_TIME.get() != null && PROXY_TIME.get() > 0) {
- return PROXY_TIME.get() - START_TIME.get();
- }
- return -1;
- }
-
- /**
- * Get time between now and when the operation was forwarded to the Namenode.
- * @return Current proxy time in nanoseconds.
- */
- private long getProxyTime() {
- if (PROXY_TIME.get() != null && PROXY_TIME.get() > 0) {
- return getNow() - PROXY_TIME.get();
- }
- return -1;
- }
-
- @Override
- public FederationRPCMetrics getRPCMetrics() {
- return this.metrics;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
deleted file mode 100644
index c4e5b5b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
+++ /dev/null
@@ -1,634 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.metrics;
-
-import static org.apache.hadoop.util.Time.now;
-
-import java.io.IOException;
-import java.lang.management.ManagementFactory;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-
-import javax.management.NotCompliantMBeanException;
-import javax.management.ObjectName;
-import javax.management.StandardMBean;
-
-import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
-import org.apache.hadoop.hdfs.DFSUtilClient;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
-import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
-import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
-import org.apache.hadoop.hdfs.server.federation.router.Router;
-import org.apache.hadoop.hdfs.server.federation.router.RouterRpcServer;
-import org.apache.hadoop.hdfs.server.federation.store.MembershipStore;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoResponse;
-import org.apache.hadoop.hdfs.server.namenode.NameNodeMXBean;
-import org.apache.hadoop.hdfs.server.namenode.NameNodeStatusMXBean;
-import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean;
-import org.apache.hadoop.ipc.StandbyException;
-import org.apache.hadoop.metrics2.util.MBeans;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.util.VersionInfo;
-import org.eclipse.jetty.util.ajax.JSON;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Expose the Namenode metrics as the Router was one.
- */
-public class NamenodeBeanMetrics
- implements FSNamesystemMBean, NameNodeMXBean, NameNodeStatusMXBean {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(NamenodeBeanMetrics.class);
-
- private final Router router;
-
- /** FSNamesystem bean. */
- private ObjectName fsBeanName;
- /** FSNamesystemState bean. */
- private ObjectName fsStateBeanName;
- /** NameNodeInfo bean. */
- private ObjectName nnInfoBeanName;
- /** NameNodeStatus bean. */
- private ObjectName nnStatusBeanName;
-
-
- public NamenodeBeanMetrics(Router router) {
- this.router = router;
-
- try {
- // TODO this needs to be done with the Metrics from FSNamesystem
- StandardMBean bean = new StandardMBean(this, FSNamesystemMBean.class);
- this.fsBeanName = MBeans.register("NameNode", "FSNamesystem", bean);
- LOG.info("Registered FSNamesystem MBean: {}", this.fsBeanName);
- } catch (NotCompliantMBeanException e) {
- throw new RuntimeException("Bad FSNamesystem MBean setup", e);
- }
-
- try {
- StandardMBean bean = new StandardMBean(this, FSNamesystemMBean.class);
- this.fsStateBeanName =
- MBeans.register("NameNode", "FSNamesystemState", bean);
- LOG.info("Registered FSNamesystemState MBean: {}", this.fsStateBeanName);
- } catch (NotCompliantMBeanException e) {
- throw new RuntimeException("Bad FSNamesystemState MBean setup", e);
- }
-
- try {
- StandardMBean bean = new StandardMBean(this, NameNodeMXBean.class);
- this.nnInfoBeanName = MBeans.register("NameNode", "NameNodeInfo", bean);
- LOG.info("Registered NameNodeInfo MBean: {}", this.nnInfoBeanName);
- } catch (NotCompliantMBeanException e) {
- throw new RuntimeException("Bad NameNodeInfo MBean setup", e);
- }
-
- try {
- StandardMBean bean = new StandardMBean(this, NameNodeStatusMXBean.class);
- this.nnStatusBeanName =
- MBeans.register("NameNode", "NameNodeStatus", bean);
- LOG.info("Registered NameNodeStatus MBean: {}", this.nnStatusBeanName);
- } catch (NotCompliantMBeanException e) {
- throw new RuntimeException("Bad NameNodeStatus MBean setup", e);
- }
- }
-
- /**
- * De-register the JMX interfaces.
- */
- public void close() {
- if (fsStateBeanName != null) {
- MBeans.unregister(fsStateBeanName);
- fsStateBeanName = null;
- }
- if (nnInfoBeanName != null) {
- MBeans.unregister(nnInfoBeanName);
- nnInfoBeanName = null;
- }
- // Remove the NameNode status bean
- if (nnStatusBeanName != null) {
- MBeans.unregister(nnStatusBeanName);
- nnStatusBeanName = null;
- }
- }
-
- private FederationMetrics getFederationMetrics() {
- return this.router.getMetrics();
- }
-
- /////////////////////////////////////////////////////////
- // NameNodeMXBean
- /////////////////////////////////////////////////////////
-
- @Override
- public String getVersion() {
- return VersionInfo.getVersion() + ", r" + VersionInfo.getRevision();
- }
-
- @Override
- public String getSoftwareVersion() {
- return VersionInfo.getVersion();
- }
-
- @Override
- public long getUsed() {
- return getFederationMetrics().getUsedCapacity();
- }
-
- @Override
- public long getFree() {
- return getFederationMetrics().getRemainingCapacity();
- }
-
- @Override
- public long getTotal() {
- return getFederationMetrics().getTotalCapacity();
- }
-
- @Override
- public long getProvidedCapacity() {
- return getFederationMetrics().getProvidedSpace();
- }
-
- @Override
- public String getSafemode() {
- // We assume that the global federated view is never in safe mode
- return "";
- }
-
- @Override
- public boolean isUpgradeFinalized() {
- // We assume the upgrade is always finalized in a federated biew
- return true;
- }
-
- @Override
- public RollingUpgradeInfo.Bean getRollingUpgradeStatus() {
- return null;
- }
-
- @Override
- public long getNonDfsUsedSpace() {
- return 0;
- }
-
- @Override
- public float getPercentUsed() {
- return DFSUtilClient.getPercentUsed(getCapacityUsed(), getCapacityTotal());
- }
-
- @Override
- public float getPercentRemaining() {
- return DFSUtilClient.getPercentUsed(
- getCapacityRemaining(), getCapacityTotal());
- }
-
- @Override
- public long getCacheUsed() {
- return 0;
- }
-
- @Override
- public long getCacheCapacity() {
- return 0;
- }
-
- @Override
- public long getBlockPoolUsedSpace() {
- return 0;
- }
-
- @Override
- public float getPercentBlockPoolUsed() {
- return 0;
- }
-
- @Override
- public long getTotalBlocks() {
- return getFederationMetrics().getNumBlocks();
- }
-
- @Override
- public long getNumberOfMissingBlocks() {
- return getFederationMetrics().getNumOfMissingBlocks();
- }
-
- @Override
- @Deprecated
- public long getPendingReplicationBlocks() {
- return getFederationMetrics().getNumOfBlocksPendingReplication();
- }
-
- @Override
- public long getPendingReconstructionBlocks() {
- return getFederationMetrics().getNumOfBlocksPendingReplication();
- }
-
- @Override
- @Deprecated
- public long getUnderReplicatedBlocks() {
- return getFederationMetrics().getNumOfBlocksUnderReplicated();
- }
-
- @Override
- public long getLowRedundancyBlocks() {
- return getFederationMetrics().getNumOfBlocksUnderReplicated();
- }
-
- @Override
- public long getPendingDeletionBlocks() {
- return getFederationMetrics().getNumOfBlocksPendingDeletion();
- }
-
- @Override
- public long getScheduledReplicationBlocks() {
- return -1;
- }
-
- @Override
- public long getNumberOfMissingBlocksWithReplicationFactorOne() {
- return 0;
- }
-
- @Override
- public String getCorruptFiles() {
- return "N/A";
- }
-
- @Override
- public int getThreads() {
- return ManagementFactory.getThreadMXBean().getThreadCount();
- }
-
- @Override
- public String getLiveNodes() {
- return this.getNodes(DatanodeReportType.LIVE);
- }
-
- @Override
- public String getDeadNodes() {
- return this.getNodes(DatanodeReportType.DEAD);
- }
-
- @Override
- public String getDecomNodes() {
- return this.getNodes(DatanodeReportType.DECOMMISSIONING);
- }
-
- /**
- * Get all the nodes in the federation from a particular type.
- * TODO this is expensive, we may want to cache it.
- * @param type Type of the datanodes to check.
- * @return JSON with the nodes.
- */
- private String getNodes(DatanodeReportType type) {
- final Map<String, Map<String, Object>> info = new HashMap<>();
- try {
- RouterRpcServer rpcServer = this.router.getRpcServer();
- DatanodeInfo[] datanodes = rpcServer.getDatanodeReport(type);
- for (DatanodeInfo node : datanodes) {
- Map<String, Object> innerinfo = new HashMap<>();
- innerinfo.put("infoAddr", node.getInfoAddr());
- innerinfo.put("infoSecureAddr", node.getInfoSecureAddr());
- innerinfo.put("xferaddr", node.getXferAddr());
- innerinfo.put("location", node.getNetworkLocation());
- innerinfo.put("lastContact", getLastContact(node));
- innerinfo.put("usedSpace", node.getDfsUsed());
- innerinfo.put("adminState", node.getAdminState().toString());
- innerinfo.put("nonDfsUsedSpace", node.getNonDfsUsed());
- innerinfo.put("capacity", node.getCapacity());
- innerinfo.put("numBlocks", -1); // node.numBlocks()
- innerinfo.put("version", (node.getSoftwareVersion() == null ?
- "UNKNOWN" : node.getSoftwareVersion()));
- innerinfo.put("used", node.getDfsUsed());
- innerinfo.put("remaining", node.getRemaining());
- innerinfo.put("blockScheduled", -1); // node.getBlocksScheduled()
- innerinfo.put("blockPoolUsed", node.getBlockPoolUsed());
- innerinfo.put("blockPoolUsedPercent", node.getBlockPoolUsedPercent());
- innerinfo.put("volfails", -1); // node.getVolumeFailures()
- info.put(node.getHostName() + ":" + node.getXferPort(),
- Collections.unmodifiableMap(innerinfo));
- }
- } catch (StandbyException e) {
- LOG.error("Cannot get {} nodes, Router in safe mode", type);
- } catch (IOException e) {
- LOG.error("Cannot get " + type + " nodes", e);
- }
- return JSON.toString(info);
- }
-
- @Override
- public String getClusterId() {
- try {
- return getNamespaceInfo(FederationNamespaceInfo::getClusterId).toString();
- } catch (IOException e) {
- LOG.error("Cannot fetch cluster ID metrics {}", e.getMessage());
- return "";
- }
- }
-
- @Override
- public String getBlockPoolId() {
- try {
- return
- getNamespaceInfo(FederationNamespaceInfo::getBlockPoolId).toString();
- } catch (IOException e) {
- LOG.error("Cannot fetch block pool ID metrics {}", e.getMessage());
- return "";
- }
- }
-
- /**
- * Build a set of unique values found in all namespaces.
- *
- * @param f Method reference of the appropriate FederationNamespaceInfo
- * getter function
- * @return Set of unique string values found in all discovered namespaces.
- * @throws IOException if the query could not be executed.
- */
- private Collection<String> getNamespaceInfo(
- Function<FederationNamespaceInfo, String> f) throws IOException {
- StateStoreService stateStore = router.getStateStore();
- MembershipStore membershipStore =
- stateStore.getRegisteredRecordStore(MembershipStore.class);
-
- GetNamespaceInfoRequest request = GetNamespaceInfoRequest.newInstance();
- GetNamespaceInfoResponse response =
- membershipStore.getNamespaceInfo(request);
- return response.getNamespaceInfo().stream()
- .map(f)
- .collect(Collectors.toSet());
- }
-
- @Override
- public String getNameDirStatuses() {
- return "N/A";
- }
-
- @Override
- public String getNodeUsage() {
- return "N/A";
- }
-
- @Override
- public String getNameJournalStatus() {
- return "N/A";
- }
-
- @Override
- public String getJournalTransactionInfo() {
- return "N/A";
- }
-
- @Override
- public long getNNStartedTimeInMillis() {
- return this.router.getStartTime();
- }
-
- @Override
- public String getCompileInfo() {
- return VersionInfo.getDate() + " by " + VersionInfo.getUser() +
- " from " + VersionInfo.getBranch();
- }
-
- @Override
- public int getDistinctVersionCount() {
- return 0;
- }
-
- @Override
- public Map<String, Integer> getDistinctVersions() {
- return null;
- }
-
- /////////////////////////////////////////////////////////
- // FSNamesystemMBean
- /////////////////////////////////////////////////////////
-
- @Override
- public String getFSState() {
- // We assume is not in safe mode
- return "Operational";
- }
-
- @Override
- public long getBlocksTotal() {
- return this.getTotalBlocks();
- }
-
- @Override
- public long getCapacityTotal() {
- return this.getTotal();
- }
-
- @Override
- public long getCapacityRemaining() {
- return this.getFree();
- }
-
- @Override
- public long getCapacityUsed() {
- return this.getUsed();
- }
-
- @Override
- public long getProvidedCapacityTotal() {
- return getProvidedCapacity();
- }
-
- @Override
- public long getFilesTotal() {
- return getFederationMetrics().getNumFiles();
- }
-
- @Override
- public int getTotalLoad() {
- return -1;
- }
-
- @Override
- public int getNumLiveDataNodes() {
- return this.router.getMetrics().getNumLiveNodes();
- }
-
- @Override
- public int getNumDeadDataNodes() {
- return this.router.getMetrics().getNumDeadNodes();
- }
-
- @Override
- public int getNumStaleDataNodes() {
- return -1;
- }
-
- @Override
- public int getNumDecomLiveDataNodes() {
- return this.router.getMetrics().getNumDecomLiveNodes();
- }
-
- @Override
- public int getNumDecomDeadDataNodes() {
- return this.router.getMetrics().getNumDecomDeadNodes();
- }
-
- @Override
- public int getNumDecommissioningDataNodes() {
- return this.router.getMetrics().getNumDecommissioningNodes();
- }
-
- @Override
- public int getNumInMaintenanceLiveDataNodes() {
- return 0;
- }
-
- @Override
- public int getNumInMaintenanceDeadDataNodes() {
- return 0;
- }
-
- @Override
- public int getNumEnteringMaintenanceDataNodes() {
- return 0;
- }
-
- @Override
- public int getVolumeFailuresTotal() {
- return 0;
- }
-
- @Override
- public long getEstimatedCapacityLostTotal() {
- return 0;
- }
-
- @Override
- public String getSnapshotStats() {
- return null;
- }
-
- @Override
- public long getMaxObjects() {
- return 0;
- }
-
- @Override
- public long getBlockDeletionStartTime() {
- return -1;
- }
-
- @Override
- public int getNumStaleStorages() {
- return -1;
- }
-
- @Override
- public String getTopUserOpCounts() {
- return "N/A";
- }
-
- @Override
- public int getFsLockQueueLength() {
- return 0;
- }
-
- @Override
- public long getTotalSyncCount() {
- return 0;
- }
-
- @Override
- public String getTotalSyncTimes() {
- return "";
- }
-
- private long getLastContact(DatanodeInfo node) {
- return (now() - node.getLastUpdate()) / 1000;
- }
-
- /////////////////////////////////////////////////////////
- // NameNodeStatusMXBean
- /////////////////////////////////////////////////////////
-
- @Override
- public String getNNRole() {
- return NamenodeRole.NAMENODE.toString();
- }
-
- @Override
- public String getState() {
- return HAServiceState.ACTIVE.toString();
- }
-
- @Override
- public String getHostAndPort() {
- return NetUtils.getHostPortString(router.getRpcServerAddress());
- }
-
- @Override
- public boolean isSecurityEnabled() {
- return false;
- }
-
- @Override
- public long getLastHATransitionTime() {
- return 0;
- }
-
- @Override
- public long getBytesWithFutureGenerationStamps() {
- return 0;
- }
-
- @Override
- public String getSlowPeersReport() {
- return "N/A";
- }
-
- @Override
- public String getSlowDisksReport() {
- return "N/A";
- }
-
- @Override
- public long getNumberOfSnapshottableDirs() {
- return 0;
- }
-
- @Override
- public String getEnteringMaintenanceNodes() {
- return "N/A";
- }
-
- @Override
- public String getNameDirSize() {
- return "N/A";
- }
-
- @Override
- public int getNumEncryptionZones() {
- return 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMBean.java
deleted file mode 100644
index 5e4ccab..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMBean.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.metrics;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * JMX interface for the State Store metrics.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public interface StateStoreMBean {
-
- long getReadOps();
-
- double getReadAvg();
-
- long getWriteOps();
-
- double getWriteAvg();
-
- long getFailureOps();
-
- double getFailureAvg();
-
- long getRemoveOps();
-
- double getRemoveAvg();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
deleted file mode 100644
index 09253a2..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.metrics;
-
-import static org.apache.hadoop.metrics2.impl.MsInfo.ProcessName;
-import static org.apache.hadoop.metrics2.impl.MsInfo.SessionId;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metrics2.MetricsSystem;
-import org.apache.hadoop.metrics2.annotation.Metric;
-import org.apache.hadoop.metrics2.annotation.Metrics;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.lib.MetricsRegistry;
-import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
-import org.apache.hadoop.metrics2.lib.MutableRate;
-
-import com.google.common.annotations.VisibleForTesting;
-
-/**
- * Implementations of the JMX interface for the State Store metrics.
- */
-@Metrics(name = "StateStoreActivity", about = "Router metrics",
- context = "dfs")
-public final class StateStoreMetrics implements StateStoreMBean {
-
- private final MetricsRegistry registry = new MetricsRegistry("router");
-
- @Metric("GET transactions")
- private MutableRate reads;
- @Metric("PUT transactions")
- private MutableRate writes;
- @Metric("REMOVE transactions")
- private MutableRate removes;
- @Metric("Failed transactions")
- private MutableRate failures;
-
- private Map<String, MutableGaugeInt> cacheSizes;
-
- private StateStoreMetrics(Configuration conf) {
- registry.tag(SessionId, "RouterSession");
- registry.tag(ProcessName, "Router");
- cacheSizes = new HashMap<>();
- }
-
- public static StateStoreMetrics create(Configuration conf) {
- MetricsSystem ms = DefaultMetricsSystem.instance();
- return ms.register(new StateStoreMetrics(conf));
- }
-
- public void shutdown() {
- DefaultMetricsSystem.shutdown();
- reset();
- }
-
- public void addRead(long latency) {
- reads.add(latency);
- }
-
- public long getReadOps() {
- return reads.lastStat().numSamples();
- }
-
- public double getReadAvg() {
- return reads.lastStat().mean();
- }
-
- public void addWrite(long latency) {
- writes.add(latency);
- }
-
- public long getWriteOps() {
- return writes.lastStat().numSamples();
- }
-
- public double getWriteAvg() {
- return writes.lastStat().mean();
- }
-
- public void addFailure(long latency) {
- failures.add(latency);
- }
-
- public long getFailureOps() {
- return failures.lastStat().numSamples();
- }
-
- public double getFailureAvg() {
- return failures.lastStat().mean();
- }
-
- public void addRemove(long latency) {
- removes.add(latency);
- }
-
- public long getRemoveOps() {
- return removes.lastStat().numSamples();
- }
-
- public double getRemoveAvg() {
- return removes.lastStat().mean();
- }
-
- /**
- * Set the size of the cache for a State Store interface.
- *
- * @param name Name of the record to cache.
- * @param size Number of records.
- */
- public void setCacheSize(String name, int size) {
- String counterName = "Cache" + name + "Size";
- MutableGaugeInt counter = cacheSizes.get(counterName);
- if (counter == null) {
- counter = registry.newGauge(counterName, name, size);
- cacheSizes.put(counterName, counter);
- }
- counter.set(size);
- }
-
- @VisibleForTesting
- public void reset() {
- reads.resetMinMax();
- writes.resetMinMax();
- removes.resetMinMax();
- failures.resetMinMax();
-
- reads.lastStat().reset();
- writes.lastStat().reset();
- removes.lastStat().reset();
- failures.lastStat().reset();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/package-info.java
deleted file mode 100644
index c56c823..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/package-info.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Report metrics for Router-based Federation.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-package org.apache.hadoop.hdfs.server.federation.metrics;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java
deleted file mode 100644
index 1773b34..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.resolver;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * Locates the most active NN for a given nameservice ID or blockpool ID. This
- * interface is used by the {@link org.apache.hadoop.hdfs.server.federation.
- * router.RouterRpcServer RouterRpcServer} to:
- * <ul>
- * <li>Determine the target NN for a given subcluster.
- * <li>List of all namespaces discovered/active in the federation.
- * <li>Update the currently active NN empirically.
- * </ul>
- * The interface is also used by the {@link org.apache.hadoop.hdfs.server.
- * federation.router.NamenodeHeartbeatService NamenodeHeartbeatService} to
- * register a discovered NN.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public interface ActiveNamenodeResolver {
-
- /**
- * Report a successful, active NN address for a nameservice or blockPool.
- *
- * @param ns Nameservice identifier.
- * @param successfulAddress The address the successful responded to the
- * command.
- * @throws IOException If the state store cannot be accessed.
- */
- void updateActiveNamenode(
- String ns, InetSocketAddress successfulAddress) throws IOException;
-
- /**
- * Returns a prioritized list of the most recent cached registration entries
- * for a single nameservice ID.
- * Returns an empty list if none are found. Returns entries in preference of:
- * <ul>
- * <li>The most recent ACTIVE NN
- * <li>The most recent STANDBY NN
- * <li>The most recent UNAVAILABLE NN
- * </ul>
- *
- * @param nameserviceId Nameservice identifier.
- * @return Prioritized list of namenode contexts.
- * @throws IOException If the state store cannot be accessed.
- */
- List<? extends FederationNamenodeContext>
- getNamenodesForNameserviceId(String nameserviceId) throws IOException;
-
- /**
- * Returns a prioritized list of the most recent cached registration entries
- * for a single block pool ID.
- * Returns an empty list if none are found. Returns entries in preference of:
- * <ul>
- * <li>The most recent ACTIVE NN
- * <li>The most recent STANDBY NN
- * <li>The most recent UNAVAILABLE NN
- * </ul>
- *
- * @param blockPoolId Block pool identifier for the nameservice.
- * @return Prioritized list of namenode contexts.
- * @throws IOException If the state store cannot be accessed.
- */
- List<? extends FederationNamenodeContext>
- getNamenodesForBlockPoolId(String blockPoolId) throws IOException;
-
- /**
- * Register a namenode in the State Store.
- *
- * @param report Namenode status report.
- * @return True if the node was registered and successfully committed to the
- * data store.
- * @throws IOException Throws exception if the namenode could not be
- * registered.
- */
- boolean registerNamenode(NamenodeStatusReport report) throws IOException;
-
- /**
- * Get a list of all namespaces that are registered and active in the
- * federation.
- *
- * @return List of name spaces in the federation
- * @throws IOException Throws exception if the namespace list is not
- * available.
- */
- Set<FederationNamespaceInfo> getNamespaces() throws IOException;
-
- /**
- * Assign a unique identifier for the parent router service.
- * Required to report the status to the namenode resolver.
- *
- * @param routerId Unique string identifier for the router.
- */
- void setRouterId(String routerId);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeContext.java
deleted file mode 100644
index 68ef02a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeContext.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.resolver;
-
-/**
- * Interface for a discovered NN and its current server endpoints.
- */
-public interface FederationNamenodeContext {
-
- /**
- * Get the RPC server address of the namenode.
- *
- * @return RPC server address in the form of host:port.
- */
- String getRpcAddress();
-
- /**
- * Get the Service RPC server address of the namenode.
- *
- * @return Service RPC server address in the form of host:port.
- */
- String getServiceAddress();
-
- /**
- * Get the Lifeline RPC server address of the namenode.
- *
- * @return Lifeline RPC server address in the form of host:port.
- */
- String getLifelineAddress();
-
- /**
- * Get the HTTP server address of the namenode.
- *
- * @return HTTP address in the form of host:port.
- */
- String getWebAddress();
-
- /**
- * Get the unique key representing the namenode.
- *
- * @return Combination of the nameservice and the namenode IDs.
- */
- String getNamenodeKey();
-
- /**
- * Identifier for the nameservice/namespace.
- *
- * @return Namenode nameservice identifier.
- */
- String getNameserviceId();
-
- /**
- * Identifier for the namenode.
- *
- * @return String
- */
- String getNamenodeId();
-
- /**
- * The current state of the namenode (active, standby, etc).
- *
- * @return FederationNamenodeServiceState State of the namenode.
- */
- FederationNamenodeServiceState getState();
-
- /**
- * The update date.
- *
- * @return Long with the update date.
- */
- long getDateModified();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java
deleted file mode 100644
index c773f82..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.resolver;
-
-import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
-
-/**
- * Namenode state in the federation. The order of this enum is used to evaluate
- * NN priority for RPC calls.
- */
-public enum FederationNamenodeServiceState {
- ACTIVE, // HAServiceState.ACTIVE or operational.
- STANDBY, // HAServiceState.STANDBY.
- UNAVAILABLE, // When the namenode cannot be reached.
- EXPIRED; // When the last update is too old.
-
- public static FederationNamenodeServiceState getState(HAServiceState state) {
- switch(state) {
- case ACTIVE:
- return FederationNamenodeServiceState.ACTIVE;
- case STANDBY:
- return FederationNamenodeServiceState.STANDBY;
- case INITIALIZING:
- return FederationNamenodeServiceState.UNAVAILABLE;
- case STOPPING:
- return FederationNamenodeServiceState.UNAVAILABLE;
- default:
- return FederationNamenodeServiceState.UNAVAILABLE;
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java
deleted file mode 100644
index edcd308..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.resolver;
-
-import org.apache.hadoop.hdfs.server.federation.router.RemoteLocationContext;
-
-/**
- * Represents information about a single nameservice/namespace in a federated
- * HDFS cluster.
- */
-public class FederationNamespaceInfo extends RemoteLocationContext {
-
- /** Block pool identifier. */
- private final String blockPoolId;
- /** Cluster identifier. */
- private final String clusterId;
- /** Nameservice identifier. */
- private final String nameserviceId;
-
- public FederationNamespaceInfo(String bpId, String clId, String nsId) {
- this.blockPoolId = bpId;
- this.clusterId = clId;
- this.nameserviceId = nsId;
- }
-
- @Override
- public String getNameserviceId() {
- return this.nameserviceId;
- }
-
- @Override
- public String getDest() {
- return this.nameserviceId;
- }
-
- /**
- * The HDFS cluster id for this namespace.
- *
- * @return Cluster identifier.
- */
- public String getClusterId() {
- return this.clusterId;
- }
-
- /**
- * The HDFS block pool id for this namespace.
- *
- * @return Block pool identifier.
- */
- public String getBlockPoolId() {
- return this.blockPoolId;
- }
-
- @Override
- public String toString() {
- return this.nameserviceId + "->" + this.blockPoolId + ":" + this.clusterId;
- }
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntryResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntryResponse.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntryResponse.java
new file mode 100644
index 0000000..9bc7f92
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntryResponse.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API response for adding a mount table entry to the state store.
+ */
+public abstract class AddMountTableEntryResponse {
+
+ public static AddMountTableEntryResponse newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(AddMountTableEntryResponse.class);
+ }
+
+ @Public
+ @Unstable
+ public abstract boolean getStatus();
+
+ @Public
+ @Unstable
+ public abstract void setStatus(boolean result);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/EnterSafeModeRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/EnterSafeModeRequest.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/EnterSafeModeRequest.java
new file mode 100644
index 0000000..89df1bb
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/EnterSafeModeRequest.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API request for the Router entering safe mode state and updating
+ * its state in State Store.
+ */
+public abstract class EnterSafeModeRequest {
+ public static EnterSafeModeRequest newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(EnterSafeModeRequest.class);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/EnterSafeModeResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/EnterSafeModeResponse.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/EnterSafeModeResponse.java
new file mode 100644
index 0000000..4c994f6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/EnterSafeModeResponse.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API response for the Router entering safe mode state and updating
+ * its state in State Store.
+ */
+public abstract class EnterSafeModeResponse {
+
+ public static EnterSafeModeResponse newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(EnterSafeModeResponse.class);
+ }
+
+ public static EnterSafeModeResponse newInstance(boolean status)
+ throws IOException {
+ EnterSafeModeResponse response = newInstance();
+ response.setStatus(status);
+ return response;
+ }
+
+ @Public
+ @Unstable
+ public abstract boolean getStatus();
+
+ @Public
+ @Unstable
+ public abstract void setStatus(boolean result);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetMountTableEntriesRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetMountTableEntriesRequest.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetMountTableEntriesRequest.java
new file mode 100644
index 0000000..cd6c278
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetMountTableEntriesRequest.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API request for listing mount table entries present in the state store.
+ */
+public abstract class GetMountTableEntriesRequest {
+
+ public static GetMountTableEntriesRequest newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(GetMountTableEntriesRequest.class);
+ }
+
+ public static GetMountTableEntriesRequest newInstance(String srcPath)
+ throws IOException {
+ GetMountTableEntriesRequest request = newInstance();
+ request.setSrcPath(srcPath);
+ return request;
+ }
+
+ @Public
+ @Unstable
+ public abstract String getSrcPath();
+
+ @Public
+ @Unstable
+ public abstract void setSrcPath(String path);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetMountTableEntriesResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetMountTableEntriesResponse.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetMountTableEntriesResponse.java
new file mode 100644
index 0000000..cebc3f6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetMountTableEntriesResponse.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+
+/**
+ * API response for listing mount table entries present in the state store.
+ */
+public abstract class GetMountTableEntriesResponse {
+
+ public static GetMountTableEntriesResponse newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(GetMountTableEntriesResponse.class);
+ }
+
+ @Public
+ @Unstable
+ public abstract List<MountTable> getEntries() throws IOException;
+
+ @Public
+ @Unstable
+ public abstract void setEntries(List<MountTable> entries)
+ throws IOException;
+
+ @Public
+ @Unstable
+ public abstract long getTimestamp();
+
+ @Public
+ @Unstable
+ public abstract void setTimestamp(long time);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamenodeRegistrationsRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamenodeRegistrationsRequest.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamenodeRegistrationsRequest.java
new file mode 100644
index 0000000..568feaf
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamenodeRegistrationsRequest.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+
+/**
+ * API request for listing namenode registrations present in the state store.
+ */
+public abstract class GetNamenodeRegistrationsRequest {
+
+ public static GetNamenodeRegistrationsRequest newInstance()
+ throws IOException {
+ return StateStoreSerializer.newRecord(
+ GetNamenodeRegistrationsRequest.class);
+ }
+
+ public static GetNamenodeRegistrationsRequest newInstance(
+ MembershipState member) throws IOException {
+ GetNamenodeRegistrationsRequest request = newInstance();
+ request.setPartialMembership(member);
+ return request;
+ }
+
+ @Public
+ @Unstable
+ public abstract MembershipState getPartialMembership();
+
+ @Public
+ @Unstable
+ public abstract void setPartialMembership(MembershipState member);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamenodeRegistrationsResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamenodeRegistrationsResponse.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamenodeRegistrationsResponse.java
new file mode 100644
index 0000000..0d60c90
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamenodeRegistrationsResponse.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+
+/**
+ * API response for listing namenode registrations present in the state store.
+ */
+public abstract class GetNamenodeRegistrationsResponse {
+
+ public static GetNamenodeRegistrationsResponse newInstance()
+ throws IOException {
+ return StateStoreSerializer.newRecord(
+ GetNamenodeRegistrationsResponse.class);
+ }
+
+ public static GetNamenodeRegistrationsResponse newInstance(
+ List<MembershipState> records) throws IOException {
+ GetNamenodeRegistrationsResponse response = newInstance();
+ response.setNamenodeMemberships(records);
+ return response;
+ }
+
+ @Public
+ @Unstable
+ public abstract List<MembershipState> getNamenodeMemberships()
+ throws IOException;
+
+ @Public
+ @Unstable
+ public abstract void setNamenodeMemberships(
+ List<MembershipState> records) throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamespaceInfoRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamespaceInfoRequest.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamespaceInfoRequest.java
new file mode 100644
index 0000000..b5cc01b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamespaceInfoRequest.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API response for listing HDFS namespaces present in the state store.
+ */
+public abstract class GetNamespaceInfoRequest {
+
+ public static GetNamespaceInfoRequest newInstance() {
+ return StateStoreSerializer.newRecord(GetNamespaceInfoRequest.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamespaceInfoResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamespaceInfoResponse.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamespaceInfoResponse.java
new file mode 100644
index 0000000..f541453
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetNamespaceInfoResponse.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API response for listing HDFS namespaces present in the state store.
+ */
+public abstract class GetNamespaceInfoResponse {
+
+ public static GetNamespaceInfoResponse newInstance() {
+ return StateStoreSerializer.newRecord(GetNamespaceInfoResponse.class);
+ }
+
+ public static GetNamespaceInfoResponse newInstance(
+ Set<FederationNamespaceInfo> namespaces) throws IOException {
+ GetNamespaceInfoResponse response = newInstance();
+ response.setNamespaceInfo(namespaces);
+ return response;
+ }
+
+ @Public
+ @Unstable
+ public abstract Set<FederationNamespaceInfo> getNamespaceInfo();
+
+ @Public
+ @Unstable
+ public abstract void setNamespaceInfo(
+ Set<FederationNamespaceInfo> namespaceInfo);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationRequest.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationRequest.java
new file mode 100644
index 0000000..9ba5788
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationRequest.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API request for retrieving a single router registration present in the state
+ * store.
+ */
+public abstract class GetRouterRegistrationRequest {
+
+ public static GetRouterRegistrationRequest newInstance() {
+ return StateStoreSerializer.newRecord(GetRouterRegistrationRequest.class);
+ }
+
+ public static GetRouterRegistrationRequest newInstance(String routerId) {
+ GetRouterRegistrationRequest request = newInstance();
+ request.setRouterId(routerId);
+ return request;
+ }
+
+ @Public
+ @Unstable
+ public abstract String getRouterId();
+
+ @Public
+ @Unstable
+ public abstract void setRouterId(String routerId);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationResponse.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationResponse.java
new file mode 100644
index 0000000..b569376
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationResponse.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+
+/**
+ * API response for retrieving a single router registration present in the state
+ * store.
+ */
+public abstract class GetRouterRegistrationResponse {
+
+ public static GetRouterRegistrationResponse newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(GetRouterRegistrationResponse.class);
+ }
+
+ @Public
+ @Unstable
+ public abstract RouterState getRouter() throws IOException;
+
+ @Public
+ @Unstable
+ public abstract void setRouter(RouterState router) throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsRequest.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsRequest.java
new file mode 100644
index 0000000..b70cccf
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsRequest.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API request for retrieving a all non-expired router registrations present in
+ * the state store.
+ */
+public abstract class GetRouterRegistrationsRequest {
+
+ public static GetRouterRegistrationsRequest newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(GetRouterRegistrationsRequest.class);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsResponse.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsResponse.java
new file mode 100644
index 0000000..94e35c6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsResponse.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+
+/**
+ * API response for retrieving a all non-expired router registrations present in
+ * the state store.
+ */
+public abstract class GetRouterRegistrationsResponse {
+
+ public static GetRouterRegistrationsResponse newInstance()
+ throws IOException {
+ return StateStoreSerializer.newRecord(GetRouterRegistrationsResponse.class);
+ }
+
+ @Public
+ @Unstable
+ public abstract List<RouterState> getRouters() throws IOException;
+
+ @Public
+ @Unstable
+ public abstract void setRouters(List<RouterState> routers)
+ throws IOException;
+
+ @Public
+ @Unstable
+ public abstract long getTimestamp();
+
+ @Public
+ @Unstable
+ public abstract void setTimestamp(long time);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetSafeModeRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetSafeModeRequest.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetSafeModeRequest.java
new file mode 100644
index 0000000..00a8986
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetSafeModeRequest.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API request for verifying if current Router state is safe mode.
+ */
+public abstract class GetSafeModeRequest {
+ public static GetSafeModeRequest newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(GetSafeModeRequest.class);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetSafeModeResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetSafeModeResponse.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetSafeModeResponse.java
new file mode 100644
index 0000000..0d7e229
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetSafeModeResponse.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API response for verifying if current Router state is safe mode.
+ */
+public abstract class GetSafeModeResponse {
+
+ public static GetSafeModeResponse newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(GetSafeModeResponse.class);
+ }
+
+ public static GetSafeModeResponse newInstance(boolean isInSafeMode)
+ throws IOException {
+ GetSafeModeResponse response = newInstance();
+ response.setSafeMode(isInSafeMode);
+ return response;
+ }
+
+ @Public
+ @Unstable
+ public abstract boolean isInSafeMode();
+
+ @Public
+ @Unstable
+ public abstract void setSafeMode(boolean isInSafeMode);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/LeaveSafeModeRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/LeaveSafeModeRequest.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/LeaveSafeModeRequest.java
new file mode 100644
index 0000000..2d64a1e
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/LeaveSafeModeRequest.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API request for the Router leaving safe mode state and updating
+ * its state in State Store.
+ */
+public abstract class LeaveSafeModeRequest {
+ public static LeaveSafeModeRequest newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(LeaveSafeModeRequest.class);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/LeaveSafeModeResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/LeaveSafeModeResponse.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/LeaveSafeModeResponse.java
new file mode 100644
index 0000000..555f4fd
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/LeaveSafeModeResponse.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API response for the Router leaving safe mode state and updating
+ * its state in State Store.
+ */
+public abstract class LeaveSafeModeResponse {
+
+ public static LeaveSafeModeResponse newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(LeaveSafeModeResponse.class);
+ }
+
+ public static LeaveSafeModeResponse newInstance(boolean status)
+ throws IOException {
+ LeaveSafeModeResponse response = newInstance();
+ response.setStatus(status);
+ return response;
+ }
+
+ @Public
+ @Unstable
+ public abstract boolean getStatus();
+
+ @Public
+ @Unstable
+ public abstract void setStatus(boolean result);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/NamenodeHeartbeatRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/NamenodeHeartbeatRequest.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/NamenodeHeartbeatRequest.java
new file mode 100644
index 0000000..9506026
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/NamenodeHeartbeatRequest.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+
+/**
+ * API request for registering a namenode with the state store.
+ */
+public abstract class NamenodeHeartbeatRequest {
+
+ public static NamenodeHeartbeatRequest newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(NamenodeHeartbeatRequest.class);
+ }
+
+ public static NamenodeHeartbeatRequest newInstance(MembershipState namenode)
+ throws IOException {
+ NamenodeHeartbeatRequest request = newInstance();
+ request.setNamenodeMembership(namenode);
+ return request;
+ }
+
+ @Private
+ @Unstable
+ public abstract MembershipState getNamenodeMembership()
+ throws IOException;
+
+ @Private
+ @Unstable
+ public abstract void setNamenodeMembership(MembershipState report)
+ throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/NamenodeHeartbeatResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/NamenodeHeartbeatResponse.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/NamenodeHeartbeatResponse.java
new file mode 100644
index 0000000..acb7a6f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/NamenodeHeartbeatResponse.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API response for registering a namenode with the state store.
+ */
+public abstract class NamenodeHeartbeatResponse {
+
+ public static NamenodeHeartbeatResponse newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(NamenodeHeartbeatResponse.class);
+ }
+
+ public static NamenodeHeartbeatResponse newInstance(boolean status)
+ throws IOException {
+ NamenodeHeartbeatResponse response = newInstance();
+ response.setResult(status);
+ return response;
+ }
+
+ @Private
+ @Unstable
+ public abstract boolean getResult();
+
+ @Private
+ @Unstable
+ public abstract void setResult(boolean result);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntryRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntryRequest.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntryRequest.java
new file mode 100644
index 0000000..642ee0d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntryRequest.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API request for removing a mount table path present in the state store.
+ */
+public abstract class RemoveMountTableEntryRequest {
+
+ public static RemoveMountTableEntryRequest newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(RemoveMountTableEntryRequest.class);
+ }
+
+ public static RemoveMountTableEntryRequest newInstance(String path)
+ throws IOException {
+ RemoveMountTableEntryRequest request = newInstance();
+ request.setSrcPath(path);
+ return request;
+ }
+
+ @Public
+ @Unstable
+ public abstract String getSrcPath();
+
+ @Public
+ @Unstable
+ public abstract void setSrcPath(String path);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntryResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntryResponse.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntryResponse.java
new file mode 100644
index 0000000..70f117d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntryResponse.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API response for removing a mount table path present in the state store.
+ */
+public abstract class RemoveMountTableEntryResponse {
+
+ public static RemoveMountTableEntryResponse newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(RemoveMountTableEntryResponse.class);
+ }
+
+ @Public
+ @Unstable
+ public abstract boolean getStatus();
+
+ @Public
+ @Unstable
+ public abstract void setStatus(boolean result);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RouterHeartbeatRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RouterHeartbeatRequest.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RouterHeartbeatRequest.java
new file mode 100644
index 0000000..97d637b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RouterHeartbeatRequest.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+
+/**
+ * API request for registering a router with the state store.
+ */
+public abstract class RouterHeartbeatRequest {
+
+ public static RouterHeartbeatRequest newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(RouterHeartbeatRequest.class);
+ }
+
+ public static RouterHeartbeatRequest newInstance(RouterState router)
+ throws IOException {
+ RouterHeartbeatRequest request = newInstance();
+ request.setRouter(router);
+ return request;
+ }
+
+ @Public
+ @Unstable
+ public abstract RouterState getRouter() throws IOException;
+
+ @Public
+ @Unstable
+ public abstract void setRouter(RouterState routerState);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RouterHeartbeatResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RouterHeartbeatResponse.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RouterHeartbeatResponse.java
new file mode 100644
index 0000000..5912e9f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RouterHeartbeatResponse.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API response for registering a router with the state store.
+ */
+public abstract class RouterHeartbeatResponse {
+
+ public static RouterHeartbeatResponse newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(RouterHeartbeatResponse.class);
+ }
+
+ public static RouterHeartbeatResponse newInstance(boolean status)
+ throws IOException {
+ RouterHeartbeatResponse response = newInstance();
+ response.setStatus(status);
+ return response;
+ }
+
+ @Public
+ @Unstable
+ public abstract boolean getStatus();
+
+ @Public
+ @Unstable
+ public abstract void setStatus(boolean result);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateMountTableEntryRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateMountTableEntryRequest.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateMountTableEntryRequest.java
new file mode 100644
index 0000000..afd5128
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateMountTableEntryRequest.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+
+/**
+ * API request for updating the destination of an existing mount point in the
+ * state store.
+ */
+public abstract class UpdateMountTableEntryRequest {
+
+ public static UpdateMountTableEntryRequest newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(UpdateMountTableEntryRequest.class);
+ }
+
+ public static UpdateMountTableEntryRequest newInstance(MountTable entry)
+ throws IOException {
+ UpdateMountTableEntryRequest request = newInstance();
+ request.setEntry(entry);
+ return request;
+ }
+
+ @Public
+ @Unstable
+ public abstract MountTable getEntry() throws IOException;
+
+ @Public
+ @Unstable
+ public abstract void setEntry(MountTable mount) throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateMountTableEntryResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateMountTableEntryResponse.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateMountTableEntryResponse.java
new file mode 100644
index 0000000..7097e10
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateMountTableEntryResponse.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API response for updating the destination of an existing mount point in the
+ * state store.
+ */
+public abstract class UpdateMountTableEntryResponse {
+
+ public static UpdateMountTableEntryResponse newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(UpdateMountTableEntryResponse.class);
+ }
+
+ @Public
+ @Unstable
+ public abstract boolean getStatus();
+
+ @Public
+ @Unstable
+ public abstract void setStatus(boolean result);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateNamenodeRegistrationRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateNamenodeRegistrationRequest.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateNamenodeRegistrationRequest.java
new file mode 100644
index 0000000..4459e33
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateNamenodeRegistrationRequest.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API request for overriding an existing namenode registration in the state
+ * store.
+ */
+public abstract class UpdateNamenodeRegistrationRequest {
+
+ public static UpdateNamenodeRegistrationRequest newInstance()
+ throws IOException {
+ return StateStoreSerializer.newRecord(
+ UpdateNamenodeRegistrationRequest.class);
+ }
+
+ public static UpdateNamenodeRegistrationRequest newInstance(
+ String nameserviceId, String namenodeId,
+ FederationNamenodeServiceState state) throws IOException {
+ UpdateNamenodeRegistrationRequest request = newInstance();
+ request.setNameserviceId(nameserviceId);
+ request.setNamenodeId(namenodeId);
+ request.setState(state);
+ return request;
+ }
+
+ @Private
+ @Unstable
+ public abstract String getNameserviceId();
+
+ @Private
+ @Unstable
+ public abstract String getNamenodeId();
+
+ @Private
+ @Unstable
+ public abstract FederationNamenodeServiceState getState();
+
+ @Private
+ @Unstable
+ public abstract void setNameserviceId(String nsId);
+
+ @Private
+ @Unstable
+ public abstract void setNamenodeId(String nnId);
+
+ @Private
+ @Unstable
+ public abstract void setState(FederationNamenodeServiceState state);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateNamenodeRegistrationResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateNamenodeRegistrationResponse.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateNamenodeRegistrationResponse.java
new file mode 100644
index 0000000..1f0d556
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/UpdateNamenodeRegistrationResponse.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API response for overriding an existing namenode registration in the state
+ * store.
+ */
+public abstract class UpdateNamenodeRegistrationResponse {
+
+ public static UpdateNamenodeRegistrationResponse newInstance() {
+ return StateStoreSerializer.newRecord(
+ UpdateNamenodeRegistrationResponse.class);
+ }
+
+ public static UpdateNamenodeRegistrationResponse newInstance(boolean status)
+ throws IOException {
+ UpdateNamenodeRegistrationResponse response = newInstance();
+ response.setResult(status);
+ return response;
+ }
+
+ @Private
+ @Unstable
+ public abstract boolean getResult();
+
+ @Private
+ @Unstable
+ public abstract void setResult(boolean result);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntryRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntryRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntryRequestPBImpl.java
new file mode 100644
index 0000000..35455d2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntryRequestPBImpl.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryRequestProtoOrBuilder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProto;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.MountTablePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * AddMountTableEntryRequest.
+ */
+public class AddMountTableEntryRequestPBImpl
+ extends AddMountTableEntryRequest implements PBRecord {
+
+ private FederationProtocolPBTranslator<AddMountTableEntryRequestProto,
+ AddMountTableEntryRequestProto.Builder,
+ AddMountTableEntryRequestProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<AddMountTableEntryRequestProto,
+ AddMountTableEntryRequestProto.Builder,
+ AddMountTableEntryRequestProtoOrBuilder>(
+ AddMountTableEntryRequestProto.class);
+
+ public AddMountTableEntryRequestPBImpl() {
+ }
+
+ public AddMountTableEntryRequestPBImpl(AddMountTableEntryRequestProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public AddMountTableEntryRequestProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public MountTable getEntry() {
+ MountTableRecordProto entryProto =
+ this.translator.getProtoOrBuilder().getEntry();
+ return new MountTablePBImpl(entryProto);
+ }
+
+ @Override
+ public void setEntry(MountTable mount) {
+ if (mount instanceof MountTablePBImpl) {
+ MountTablePBImpl mountPB = (MountTablePBImpl)mount;
+ MountTableRecordProto mountProto = mountPB.getProto();
+ translator.getBuilder().setEntry(mountProto);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntryResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntryResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntryResponsePBImpl.java
new file mode 100644
index 0000000..c1d9a65
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntryResponsePBImpl.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryResponseProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * AddMountTableEntryResponse.
+ */
+public class AddMountTableEntryResponsePBImpl
+ extends AddMountTableEntryResponse implements PBRecord {
+
+ private FederationProtocolPBTranslator<AddMountTableEntryResponseProto,
+ AddMountTableEntryResponseProto.Builder,
+ AddMountTableEntryResponseProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<AddMountTableEntryResponseProto,
+ AddMountTableEntryResponseProto.Builder,
+ AddMountTableEntryResponseProtoOrBuilder>(
+ AddMountTableEntryResponseProto.class);
+
+ public AddMountTableEntryResponsePBImpl() {
+ }
+
+ public AddMountTableEntryResponsePBImpl(
+ AddMountTableEntryResponseProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public AddMountTableEntryResponseProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public boolean getStatus() {
+ return this.translator.getProtoOrBuilder().getStatus();
+ }
+
+ @Override
+ public void setStatus(boolean result) {
+ this.translator.getBuilder().setStatus(result);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/EnterSafeModeRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/EnterSafeModeRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/EnterSafeModeRequestPBImpl.java
new file mode 100644
index 0000000..50e14af
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/EnterSafeModeRequestPBImpl.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.EnterSafeModeRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.EnterSafeModeRequestProtoOrBuilder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationResponseProto.Builder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * EnterSafeModeRequest.
+ */
+public class EnterSafeModeRequestPBImpl extends EnterSafeModeRequest
+ implements PBRecord {
+
+ private FederationProtocolPBTranslator<EnterSafeModeRequestProto,
+ Builder, EnterSafeModeRequestProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<>(EnterSafeModeRequestProto.class);
+
+ public EnterSafeModeRequestPBImpl() {
+ }
+
+ public EnterSafeModeRequestPBImpl(EnterSafeModeRequestProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public EnterSafeModeRequestProto getProto() {
+ return translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/EnterSafeModeResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/EnterSafeModeResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/EnterSafeModeResponsePBImpl.java
new file mode 100644
index 0000000..9805821
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/EnterSafeModeResponsePBImpl.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.EnterSafeModeResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.EnterSafeModeResponseProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.EnterSafeModeResponseProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * EnterSafeModeResponse.
+ */
+public class EnterSafeModeResponsePBImpl extends EnterSafeModeResponse
+ implements PBRecord {
+
+ private FederationProtocolPBTranslator<EnterSafeModeResponseProto,
+ Builder, EnterSafeModeResponseProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<>(
+ EnterSafeModeResponseProto.class);
+
+ public EnterSafeModeResponsePBImpl() {
+ }
+
+ public EnterSafeModeResponsePBImpl(EnterSafeModeResponseProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public EnterSafeModeResponseProto getProto() {
+ return translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public boolean getStatus() {
+ return this.translator.getProtoOrBuilder().getStatus();
+ }
+
+ @Override
+ public void setStatus(boolean result) {
+ this.translator.getBuilder().setStatus(result);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/FederationProtocolPBTranslator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/FederationProtocolPBTranslator.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/FederationProtocolPBTranslator.java
new file mode 100644
index 0000000..baad113
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/FederationProtocolPBTranslator.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+
+import org.apache.commons.codec.binary.Base64;
+
+import com.google.protobuf.GeneratedMessage;
+import com.google.protobuf.Message;
+import com.google.protobuf.Message.Builder;
+import com.google.protobuf.MessageOrBuilder;
+
+/**
+ * Helper class for setting/getting data elements in an object backed by a
+ * protobuf implementation.
+ */
+public class FederationProtocolPBTranslator<P extends GeneratedMessage,
+ B extends Builder, T extends MessageOrBuilder> {
+
+ /** Optional proto byte stream used to create this object. */
+ private P proto;
+ /** The class of the proto handler for this translator. */
+ private Class<P> protoClass;
+ /** Internal builder, used to store data that has been set. */
+ private B builder;
+
+ public FederationProtocolPBTranslator(Class<P> protoType) {
+ this.protoClass = protoType;
+ }
+
+ /**
+ * Called if this translator is to be created from an existing protobuf byte
+ * stream.
+ *
+ * @param p The existing proto object to use to initialize the translator.
+ * @throws IllegalArgumentException
+ */
+ @SuppressWarnings("unchecked")
+ public void setProto(Message p) {
+ if (protoClass.isInstance(p)) {
+ if (this.builder != null) {
+ // Merge with builder
+ this.builder.mergeFrom((P) p);
+ } else {
+ // Store proto
+ this.proto = (P) p;
+ }
+ } else {
+ throw new IllegalArgumentException(
+ "Cannot decode proto type " + p.getClass().getName());
+ }
+ }
+
+ /**
+ * Create or return the cached protobuf builder for this translator.
+ *
+ * @return cached Builder instance
+ */
+ @SuppressWarnings("unchecked")
+ public B getBuilder() {
+ if (this.builder == null) {
+ try {
+ Method method = protoClass.getMethod("newBuilder");
+ this.builder = (B) method.invoke(null);
+ if (this.proto != null) {
+ // Merge in existing immutable proto
+ this.builder.mergeFrom(this.proto);
+ }
+ } catch (ReflectiveOperationException e) {
+ this.builder = null;
+ }
+ }
+ return this.builder;
+ }
+
+ /**
+ * Get the serialized proto object. If the translator was created from a byte
+ * stream, returns the intitial byte stream. Otherwise creates a new byte
+ * stream from the cached builder.
+ *
+ * @return Protobuf message object
+ */
+ @SuppressWarnings("unchecked")
+ public P build() {
+ if (this.builder != null) {
+ // serialize from builder (mutable) first
+ Message m = this.builder.build();
+ return (P) m;
+ } else if (this.proto != null) {
+ // Use immutable message source, message is unchanged
+ return this.proto;
+ }
+ return null;
+ }
+
+ /**
+ * Returns an interface to access data stored within this object. The object
+ * may have been initialized either via a builder or by an existing protobuf
+ * byte stream.
+ *
+ * @return MessageOrBuilder protobuf interface for the requested class.
+ */
+ @SuppressWarnings("unchecked")
+ public T getProtoOrBuilder() {
+ if (this.builder != null) {
+ // Use mutable builder if it exists
+ return (T) this.builder;
+ } else if (this.proto != null) {
+ // Use immutable message source
+ return (T) this.proto;
+ } else {
+ // Construct empty builder
+ return (T) this.getBuilder();
+ }
+ }
+
+ /**
+ * Read instance from base64 data.
+ * @param base64String
+ * @throws IOException
+ */
+ @SuppressWarnings("unchecked")
+ public void readInstance(String base64String) throws IOException {
+ byte[] bytes = Base64.decodeBase64(base64String);
+ Message msg = getBuilder().mergeFrom(bytes).build();
+ this.proto = (P) msg;
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMultipleDestinationResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMultipleDestinationResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMultipleDestinationResolver.java
new file mode 100644
index 0000000..3915c56
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMultipleDestinationResolver.java
@@ -0,0 +1,419 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import static org.apache.hadoop.hdfs.server.federation.resolver.order.HashResolver.extractTempFileName;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test the multiple destination resolver.
+ */
+public class TestMultipleDestinationResolver {
+
+ private MultipleDestinationMountTableResolver resolver;
+
+ @Before
+ public void setup() throws IOException {
+ Configuration conf = new Configuration();
+ resolver = new MultipleDestinationMountTableResolver(conf, null);
+
+ // We manually point /tmp to only subcluster0
+ Map<String, String> map1 = new HashMap<>();
+ map1.put("subcluster0", "/tmp");
+ resolver.addEntry(MountTable.newInstance("/tmp", map1));
+
+ // We manually point / to subcluster0,1,2 with default order (hash)
+ Map<String, String> mapDefault = new HashMap<>();
+ mapDefault.put("subcluster0", "/");
+ mapDefault.put("subcluster1", "/");
+ mapDefault.put("subcluster2", "/");
+ MountTable defaultEntry = MountTable.newInstance("/", mapDefault);
+ resolver.addEntry(defaultEntry);
+
+ // We manually point /hash to subcluster0,1,2 with hashing
+ Map<String, String> mapHash = new HashMap<>();
+ mapHash.put("subcluster0", "/hash");
+ mapHash.put("subcluster1", "/hash");
+ mapHash.put("subcluster2", "/hash");
+ MountTable hashEntry = MountTable.newInstance("/hash", mapHash);
+ hashEntry.setDestOrder(DestinationOrder.HASH);
+ resolver.addEntry(hashEntry);
+
+ // We manually point /hashall to subcluster0,1,2 with hashing (full tree)
+ Map<String, String> mapHashAll = new HashMap<>();
+ mapHashAll.put("subcluster0", "/hashall");
+ mapHashAll.put("subcluster1", "/hashall");
+ mapHashAll.put("subcluster2", "/hashall");
+ MountTable hashEntryAll = MountTable.newInstance("/hashall", mapHashAll);
+ hashEntryAll.setDestOrder(DestinationOrder.HASH_ALL);
+ resolver.addEntry(hashEntryAll);
+
+ // We point /local to subclusters 0, 1, 2 with the local order
+ Map<String, String> mapLocal = new HashMap<>();
+ mapLocal.put("subcluster0", "/local");
+ mapLocal.put("subcluster1", "/local");
+ mapLocal.put("subcluster2", "/local");
+ MountTable localEntry = MountTable.newInstance("/local", mapLocal);
+ localEntry.setDestOrder(DestinationOrder.LOCAL);
+ resolver.addEntry(localEntry);
+
+ // We point /random to subclusters 0, 1, 2 with the random order
+ Map<String, String> mapRandom = new HashMap<>();
+ mapRandom.put("subcluster0", "/random");
+ mapRandom.put("subcluster1", "/random");
+ mapRandom.put("subcluster2", "/random");
+ MountTable randomEntry = MountTable.newInstance("/random", mapRandom);
+ randomEntry.setDestOrder(DestinationOrder.RANDOM);
+ resolver.addEntry(randomEntry);
+
+ // Read only mount point
+ Map<String, String> mapReadOnly = new HashMap<>();
+ mapReadOnly.put("subcluster0", "/readonly");
+ mapReadOnly.put("subcluster1", "/readonly");
+ mapReadOnly.put("subcluster2", "/readonly");
+ MountTable readOnlyEntry = MountTable.newInstance("/readonly", mapReadOnly);
+ readOnlyEntry.setReadOnly(true);
+ resolver.addEntry(readOnlyEntry);
+ }
+
+ @Test
+ public void testHashEqualDistribution() throws IOException {
+ // First level
+ testEvenDistribution("/hash");
+ testEvenDistribution("/hash/folder0", false);
+
+ // All levels
+ testEvenDistribution("/hashall");
+ testEvenDistribution("/hashall/folder0");
+ }
+
+ @Test
+ public void testHashAll() throws IOException {
+ // Files should be spread across subclusters
+ PathLocation dest0 = resolver.getDestinationForPath("/hashall/file0.txt");
+ assertDest("subcluster0", dest0);
+ PathLocation dest1 = resolver.getDestinationForPath("/hashall/file1.txt");
+ assertDest("subcluster1", dest1);
+
+ // Files within folder should be spread across subclusters
+ PathLocation dest2 = resolver.getDestinationForPath("/hashall/folder0");
+ assertDest("subcluster2", dest2);
+ PathLocation dest3 = resolver.getDestinationForPath(
+ "/hashall/folder0/file0.txt");
+ assertDest("subcluster1", dest3);
+ PathLocation dest4 = resolver.getDestinationForPath(
+ "/hashall/folder0/file1.txt");
+ assertDest("subcluster0", dest4);
+
+ PathLocation dest5 = resolver.getDestinationForPath(
+ "/hashall/folder0/folder0/file0.txt");
+ assertDest("subcluster1", dest5);
+ PathLocation dest6 = resolver.getDestinationForPath(
+ "/hashall/folder0/folder0/file1.txt");
+ assertDest("subcluster1", dest6);
+ PathLocation dest7 = resolver.getDestinationForPath(
+ "/hashall/folder0/folder0/file2.txt");
+ assertDest("subcluster0", dest7);
+
+ PathLocation dest8 = resolver.getDestinationForPath("/hashall/folder1");
+ assertDest("subcluster1", dest8);
+ PathLocation dest9 = resolver.getDestinationForPath(
+ "/hashall/folder1/file0.txt");
+ assertDest("subcluster0", dest9);
+ PathLocation dest10 = resolver.getDestinationForPath(
+ "/hashall/folder1/file1.txt");
+ assertDest("subcluster1", dest10);
+
+ PathLocation dest11 = resolver.getDestinationForPath("/hashall/folder2");
+ assertDest("subcluster2", dest11);
+ PathLocation dest12 = resolver.getDestinationForPath(
+ "/hashall/folder2/file0.txt");
+ assertDest("subcluster0", dest12);
+ PathLocation dest13 = resolver.getDestinationForPath(
+ "/hashall/folder2/file1.txt");
+ assertDest("subcluster0", dest13);
+ PathLocation dest14 = resolver.getDestinationForPath(
+ "/hashall/folder2/file2.txt");
+ assertDest("subcluster1", dest14);
+ }
+
+ @Test
+ public void testHashFirst() throws IOException {
+ PathLocation dest0 = resolver.getDestinationForPath("/hashall/file0.txt");
+ assertDest("subcluster0", dest0);
+ PathLocation dest1 = resolver.getDestinationForPath("/hashall/file1.txt");
+ assertDest("subcluster1", dest1);
+
+ // All these must be in the same location: subcluster0
+ PathLocation dest2 = resolver.getDestinationForPath("/hash/folder0");
+ assertDest("subcluster0", dest2);
+ PathLocation dest3 = resolver.getDestinationForPath(
+ "/hash/folder0/file0.txt");
+ assertDest("subcluster0", dest3);
+ PathLocation dest4 = resolver.getDestinationForPath(
+ "/hash/folder0/file1.txt");
+ assertDest("subcluster0", dest4);
+
+ PathLocation dest5 = resolver.getDestinationForPath(
+ "/hash/folder0/folder0/file0.txt");
+ assertDest("subcluster0", dest5);
+ PathLocation dest6 = resolver.getDestinationForPath(
+ "/hash/folder0/folder0/file1.txt");
+ assertDest("subcluster0", dest6);
+
+ // All these must be in the same location: subcluster2
+ PathLocation dest7 = resolver.getDestinationForPath("/hash/folder1");
+ assertDest("subcluster2", dest7);
+ PathLocation dest8 = resolver.getDestinationForPath(
+ "/hash/folder1/file0.txt");
+ assertDest("subcluster2", dest8);
+ PathLocation dest9 = resolver.getDestinationForPath(
+ "/hash/folder1/file1.txt");
+ assertDest("subcluster2", dest9);
+
+ // All these must be in the same location: subcluster2
+ PathLocation dest10 = resolver.getDestinationForPath("/hash/folder2");
+ assertDest("subcluster2", dest10);
+ PathLocation dest11 = resolver.getDestinationForPath(
+ "/hash/folder2/file0.txt");
+ assertDest("subcluster2", dest11);
+ PathLocation dest12 = resolver.getDestinationForPath(
+ "/hash/folder2/file1.txt");
+ assertDest("subcluster2", dest12);
+ }
+
+ @Test
+ public void testRandomEqualDistribution() throws IOException {
+ testEvenDistribution("/random");
+ }
+
+ @Test
+ public void testSingleDestination() throws IOException {
+ // All the files in /tmp should be in subcluster0
+ for (int f = 0; f < 100; f++) {
+ String filename = "/tmp/b/c/file" + f + ".txt";
+ PathLocation destination = resolver.getDestinationForPath(filename);
+ RemoteLocation loc = destination.getDefaultLocation();
+ assertEquals("subcluster0", loc.getNameserviceId());
+ assertEquals(filename, loc.getDest());
+ }
+ }
+
+ @Test
+ public void testResolveSubdirectories() throws Exception {
+ // Simulate a testdir under a multi-destination mount.
+ Random r = new Random();
+ String testDir = "/sort/testdir" + r.nextInt();
+ String file1 = testDir + "/file1" + r.nextInt();
+ String file2 = testDir + "/file2" + r.nextInt();
+
+ // Verify both files resolve to the same namespace as the parent dir.
+ PathLocation testDirLocation = resolver.getDestinationForPath(testDir);
+ RemoteLocation defaultLoc = testDirLocation.getDefaultLocation();
+ String testDirNamespace = defaultLoc.getNameserviceId();
+
+ PathLocation file1Location = resolver.getDestinationForPath(file1);
+ RemoteLocation defaultLoc1 = file1Location.getDefaultLocation();
+ assertEquals(testDirNamespace, defaultLoc1.getNameserviceId());
+
+ PathLocation file2Location = resolver.getDestinationForPath(file2);
+ RemoteLocation defaultLoc2 = file2Location.getDefaultLocation();
+ assertEquals(testDirNamespace, defaultLoc2.getNameserviceId());
+ }
+
+ @Test
+ public void testExtractTempFileName() {
+ for (String teststring : new String[] {
+ "testfile1.txt.COPYING",
+ "testfile1.txt._COPYING_",
+ "testfile1.txt._COPYING_.attempt_1486662804109_0055_m_000042_0",
+ "testfile1.txt.tmp",
+ "_temp/testfile1.txt",
+ "_temporary/testfile1.txt.af77e2ab-4bc5-4959-ae08-299c880ee6b8",
+ "_temporary/0/_temporary/attempt_201706281636_0007_m_000003_46/" +
+ "testfile1.txt" }) {
+ String finalName = extractTempFileName(teststring);
+ assertEquals("testfile1.txt", finalName);
+ }
+
+ // False cases
+ assertEquals(
+ "file1.txt.COPYING1", extractTempFileName("file1.txt.COPYING1"));
+ assertEquals("file1.txt.tmp2", extractTempFileName("file1.txt.tmp2"));
+
+ // Speculation patterns
+ String finalName = extractTempFileName(
+ "_temporary/part-00007.af77e2ab-4bc5-4959-ae08-299c880ee6b8");
+ assertEquals("part-00007", finalName);
+ finalName = extractTempFileName(
+ "_temporary/0/_temporary/attempt_201706281636_0007_m_000003_46/" +
+ "part-00003");
+ assertEquals("part-00003", finalName);
+
+ // Subfolders
+ finalName = extractTempFileName("folder0/testfile1.txt._COPYING_");
+ assertEquals("folder0/testfile1.txt", finalName);
+ finalName = extractTempFileName(
+ "folder0/folder1/testfile1.txt._COPYING_");
+ assertEquals("folder0/folder1/testfile1.txt", finalName);
+ finalName = extractTempFileName(
+ "processedHrsData.txt/_temporary/0/_temporary/" +
+ "attempt_201706281636_0007_m_000003_46/part-00003");
+ assertEquals("processedHrsData.txt/part-00003", finalName);
+ }
+
+ @Test
+ public void testReadOnly() throws IOException {
+ MountTable mount = resolver.getMountPoint("/readonly");
+ assertTrue(mount.isReadOnly());
+
+ PathLocation dest0 = resolver.getDestinationForPath("/readonly/file0.txt");
+ assertDest("subcluster1", dest0);
+ PathLocation dest1 = resolver.getDestinationForPath("/readonly/file1.txt");
+ assertDest("subcluster2", dest1);
+
+ // All these must be in the same location: subcluster0
+ PathLocation dest2 = resolver.getDestinationForPath("/readonly/folder0");
+ assertDest("subcluster1", dest2);
+ PathLocation dest3 = resolver.getDestinationForPath(
+ "/readonly/folder0/file0.txt");
+ assertDest("subcluster1", dest3);
+ PathLocation dest4 = resolver.getDestinationForPath(
+ "/readonly/folder0/file1.txt");
+ assertDest("subcluster1", dest4);
+
+ PathLocation dest5 = resolver.getDestinationForPath(
+ "/readonly/folder0/folder0/file0.txt");
+ assertDest("subcluster1", dest5);
+ PathLocation dest6 = resolver.getDestinationForPath(
+ "/readonly/folder0/folder0/file1.txt");
+ assertDest("subcluster1", dest6);
+
+ // All these must be in the same location: subcluster2
+ PathLocation dest7 = resolver.getDestinationForPath("/readonly/folder1");
+ assertDest("subcluster2", dest7);
+ PathLocation dest8 = resolver.getDestinationForPath(
+ "/readonly/folder1/file0.txt");
+ assertDest("subcluster2", dest8);
+ PathLocation dest9 = resolver.getDestinationForPath(
+ "/readonly/folder1/file1.txt");
+ assertDest("subcluster2", dest9);
+
+ // All these must be in the same location: subcluster2
+ PathLocation dest10 = resolver.getDestinationForPath("/readonly/folder2");
+ assertDest("subcluster1", dest10);
+ PathLocation dest11 = resolver.getDestinationForPath(
+ "/readonly/folder2/file0.txt");
+ assertDest("subcluster1", dest11);
+ PathLocation dest12 = resolver.getDestinationForPath(
+ "/readonly/folder2/file1.txt");
+ assertDest("subcluster1", dest12);
+ }
+
+ @Test
+ public void testLocalResolver() throws IOException {
+ PathLocation dest0 =
+ resolver.getDestinationForPath("/local/folder0/file0.txt");
+ assertDest("subcluster0", dest0);
+ }
+
+ @Test
+ public void testRandomResolver() throws IOException {
+ Set<String> destinations = new HashSet<>();
+ for (int i = 0; i < 30; i++) {
+ PathLocation dest =
+ resolver.getDestinationForPath("/random/folder0/file0.txt");
+ RemoteLocation firstDest = dest.getDestinations().get(0);
+ String nsId = firstDest.getNameserviceId();
+ destinations.add(nsId);
+ }
+ assertEquals(3, destinations.size());
+ }
+
+ /**
+ * Test that a path has files distributed across destinations evenly.
+ * @param path Path to check.
+ * @throws IOException
+ */
+ private void testEvenDistribution(final String path) throws IOException {
+ testEvenDistribution(path, true);
+ }
+
+ /**
+ * Test that a path has files distributed across destinations evenly or not.
+ * @param path Path to check.
+ * @param even If the distribution should be even or not.
+ * @throws IOException If it cannot check it.
+ */
+ private void testEvenDistribution(final String path, final boolean even)
+ throws IOException {
+
+ // Subcluster -> Files
+ Map<String, Set<String>> results = new HashMap<>();
+ for (int f = 0; f < 10000; f++) {
+ String filename = path + "/file" + f + ".txt";
+ PathLocation destination = resolver.getDestinationForPath(filename);
+ RemoteLocation loc = destination.getDefaultLocation();
+ assertEquals(filename, loc.getDest());
+
+ String nsId = loc.getNameserviceId();
+ if (!results.containsKey(nsId)) {
+ results.put(nsId, new TreeSet<>());
+ }
+ results.get(nsId).add(filename);
+ }
+
+ if (!even) {
+ // All files should be in one subcluster
+ assertEquals(1, results.size());
+ } else {
+ // Files should be distributed somewhat evenly
+ assertEquals(3, results.size());
+ int count = 0;
+ for (Set<String> files : results.values()) {
+ count = count + files.size();
+ }
+ int avg = count / results.keySet().size();
+ for (Set<String> files : results.values()) {
+ int filesCount = files.size();
+ // Check that the count in each namespace is within 20% of avg
+ assertTrue(filesCount > 0);
+ assertTrue(Math.abs(filesCount - avg) < (avg / 5));
+ }
+ }
+ }
+
+ private static void assertDest(String expectedDest, PathLocation loc) {
+ assertEquals(expectedDest, loc.getDestinations().get(0).getNameserviceId());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestNamenodeResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestNamenodeResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestNamenodeResolver.java
new file mode 100644
index 0000000..00c2c13
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestNamenodeResolver.java
@@ -0,0 +1,284 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMENODES;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMESERVICES;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.ROUTERS;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createNamenodeReport;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyException;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.clearRecords;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.getStateStoreConfiguration;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.newStateStore;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.waitStateStore;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreUnavailableException;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test the basic {@link ActiveNamenodeResolver} functionality.
+ */
+public class TestNamenodeResolver {
+
+ private static StateStoreService stateStore;
+ private static ActiveNamenodeResolver namenodeResolver;
+
+ @BeforeClass
+ public static void create() throws Exception {
+
+ Configuration conf = getStateStoreConfiguration();
+
+ // Reduce expirations to 5 seconds
+ conf.setLong(
+ RBFConfigKeys.FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS,
+ TimeUnit.SECONDS.toMillis(5));
+
+ stateStore = newStateStore(conf);
+ assertNotNull(stateStore);
+
+ namenodeResolver = new MembershipNamenodeResolver(conf, stateStore);
+ namenodeResolver.setRouterId(ROUTERS[0]);
+ }
+
+ @AfterClass
+ public static void destroy() throws Exception {
+ stateStore.stop();
+ stateStore.close();
+ }
+
+ @Before
+ public void setup() throws IOException, InterruptedException {
+ // Wait for state store to connect
+ stateStore.loadDriver();
+ waitStateStore(stateStore, 10000);
+
+ // Clear NN registrations
+ boolean cleared = clearRecords(stateStore, MembershipState.class);
+ assertTrue(cleared);
+ }
+
+ @Test
+ public void testStateStoreDisconnected() throws Exception {
+
+ // Add an entry to the store
+ NamenodeStatusReport report = createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[0], HAServiceState.ACTIVE);
+ assertTrue(namenodeResolver.registerNamenode(report));
+
+ // Close the data store driver
+ stateStore.closeDriver();
+ assertFalse(stateStore.isDriverReady());
+
+ // Flush the caches
+ stateStore.refreshCaches(true);
+
+ // Verify commands fail due to no cached data and no state store
+ // connectivity.
+ List<? extends FederationNamenodeContext> nns =
+ namenodeResolver.getNamenodesForBlockPoolId(NAMESERVICES[0]);
+ assertNull(nns);
+
+ verifyException(namenodeResolver, "registerNamenode",
+ StateStoreUnavailableException.class,
+ new Class[] {NamenodeStatusReport.class}, new Object[] {report});
+ }
+
+ /**
+ * Verify the first registration on the resolver.
+ *
+ * @param nsId Nameservice identifier.
+ * @param nnId Namenode identifier within the nemeservice.
+ * @param resultsCount Number of results expected.
+ * @param state Expected state for the first one.
+ * @throws IOException If we cannot get the namenodes.
+ */
+ private void verifyFirstRegistration(String nsId, String nnId,
+ int resultsCount, FederationNamenodeServiceState state)
+ throws IOException {
+ List<? extends FederationNamenodeContext> namenodes =
+ namenodeResolver.getNamenodesForNameserviceId(nsId);
+ if (resultsCount == 0) {
+ assertNull(namenodes);
+ } else {
+ assertEquals(resultsCount, namenodes.size());
+ if (namenodes.size() > 0) {
+ FederationNamenodeContext namenode = namenodes.get(0);
+ assertEquals(state, namenode.getState());
+ assertEquals(nnId, namenode.getNamenodeId());
+ }
+ }
+ }
+
+ @Test
+ public void testRegistrationExpired()
+ throws InterruptedException, IOException {
+
+ // Populate the state store with a single NN element
+ // 1) ns0:nn0 - Active
+ // Wait for the entry to expire without heartbeating
+ // Verify the NN entry is not accessible once expired.
+ NamenodeStatusReport report = createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[0], HAServiceState.ACTIVE);
+ assertTrue(namenodeResolver.registerNamenode(report));
+
+ // Load cache
+ stateStore.refreshCaches(true);
+
+ // Verify
+ verifyFirstRegistration(
+ NAMESERVICES[0], NAMENODES[0], 1,
+ FederationNamenodeServiceState.ACTIVE);
+
+ // Wait past expiration (set in conf to 5 seconds)
+ Thread.sleep(6000);
+ // Reload cache
+ stateStore.refreshCaches(true);
+
+ // Verify entry is now expired and is no longer in the cache
+ verifyFirstRegistration(
+ NAMESERVICES[0], NAMENODES[0], 0,
+ FederationNamenodeServiceState.ACTIVE);
+
+ // Heartbeat again, updates dateModified
+ assertTrue(namenodeResolver.registerNamenode(report));
+ // Reload cache
+ stateStore.refreshCaches(true);
+
+ // Verify updated entry is marked active again and accessible to RPC server
+ verifyFirstRegistration(
+ NAMESERVICES[0], NAMENODES[0], 1,
+ FederationNamenodeServiceState.ACTIVE);
+ }
+
+ @Test
+ public void testRegistrationNamenodeSelection()
+ throws InterruptedException, IOException {
+
+ // 1) ns0:nn0 - Active
+ // 2) ns0:nn1 - Standby (newest)
+ // Verify the selected entry is the active entry
+ assertTrue(namenodeResolver.registerNamenode(
+ createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[0], HAServiceState.ACTIVE)));
+ Thread.sleep(100);
+ assertTrue(namenodeResolver.registerNamenode(
+ createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[1], HAServiceState.STANDBY)));
+
+ stateStore.refreshCaches(true);
+
+ verifyFirstRegistration(
+ NAMESERVICES[0], NAMENODES[0], 2,
+ FederationNamenodeServiceState.ACTIVE);
+
+ // 1) ns0:nn0 - Expired (stale)
+ // 2) ns0:nn1 - Standby (newest)
+ // Verify the selected entry is the standby entry as the active entry is
+ // stale
+ assertTrue(namenodeResolver.registerNamenode(
+ createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[0], HAServiceState.ACTIVE)));
+
+ // Expire active registration
+ Thread.sleep(6000);
+
+ // Refresh standby registration
+ assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[1], HAServiceState.STANDBY)));
+
+ // Verify that standby is selected (active is now expired)
+ stateStore.refreshCaches(true);
+ verifyFirstRegistration(NAMESERVICES[0], NAMENODES[1], 1,
+ FederationNamenodeServiceState.STANDBY);
+
+ // 1) ns0:nn0 - Active
+ // 2) ns0:nn1 - Unavailable (newest)
+ // Verify the selected entry is the active entry
+ assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[0], HAServiceState.ACTIVE)));
+ Thread.sleep(100);
+ assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[1], null)));
+ stateStore.refreshCaches(true);
+ verifyFirstRegistration(NAMESERVICES[0], NAMENODES[0], 2,
+ FederationNamenodeServiceState.ACTIVE);
+
+ // 1) ns0:nn0 - Unavailable (newest)
+ // 2) ns0:nn1 - Standby
+ // Verify the selected entry is the standby entry
+ assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[1], HAServiceState.STANDBY)));
+ Thread.sleep(1000);
+ assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[0], null)));
+
+ stateStore.refreshCaches(true);
+ verifyFirstRegistration(NAMESERVICES[0], NAMENODES[1], 2,
+ FederationNamenodeServiceState.STANDBY);
+
+ // 1) ns0:nn0 - Active (oldest)
+ // 2) ns0:nn1 - Standby
+ // 3) ns0:nn2 - Active (newest)
+ // Verify the selected entry is the newest active entry
+ assertTrue(namenodeResolver.registerNamenode(
+ createNamenodeReport(NAMESERVICES[0], NAMENODES[0], null)));
+ Thread.sleep(100);
+ assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[1], HAServiceState.STANDBY)));
+ Thread.sleep(100);
+ assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[2], HAServiceState.ACTIVE)));
+
+ stateStore.refreshCaches(true);
+ verifyFirstRegistration(NAMESERVICES[0], NAMENODES[2], 3,
+ FederationNamenodeServiceState.ACTIVE);
+
+ // 1) ns0:nn0 - Standby (oldest)
+ // 2) ns0:nn1 - Standby (newest)
+ // 3) ns0:nn2 - Standby
+ // Verify the selected entry is the newest standby entry
+ assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[0], HAServiceState.STANDBY)));
+ assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[2], HAServiceState.STANDBY)));
+ Thread.sleep(1500);
+ assertTrue(namenodeResolver.registerNamenode(createNamenodeReport(
+ NAMESERVICES[0], NAMENODES[1], HAServiceState.STANDBY)));
+
+ stateStore.refreshCaches(true);
+ verifyFirstRegistration(NAMESERVICES[0], NAMENODES[1], 3,
+ FederationNamenodeServiceState.STANDBY);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/order/TestLocalResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/order/TestLocalResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/order/TestLocalResolver.java
new file mode 100644
index 0000000..42ede62
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/order/TestLocalResolver.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver.order;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.resolver.MultipleDestinationMountTableResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.federation.router.Router;
+import org.apache.hadoop.hdfs.server.federation.store.MembershipStore;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+/**
+ * Test the {@link LocalResolver}.
+ */
+public class TestLocalResolver {
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testLocalResolver() throws IOException {
+
+ // Mock the subcluster mapping
+ Configuration conf = new Configuration();
+ Router router = mock(Router.class);
+ StateStoreService stateStore = mock(StateStoreService.class);
+ MembershipStore membership = mock(MembershipStore.class);
+ when(router.getStateStore()).thenReturn(stateStore);
+ when(stateStore.getRegisteredRecordStore(any(Class.class)))
+ .thenReturn(membership);
+ GetNamenodeRegistrationsResponse response =
+ GetNamenodeRegistrationsResponse.newInstance();
+ // Set the mapping for each client
+ List<MembershipState> records = new LinkedList<>();
+ records.add(newMembershipState("client0", "subcluster0"));
+ records.add(newMembershipState("client1", "subcluster1"));
+ records.add(newMembershipState("client2", "subcluster2"));
+ response.setNamenodeMemberships(records);
+ when(membership.getNamenodeRegistrations(
+ any(GetNamenodeRegistrationsRequest.class))).thenReturn(response);
+
+ // Mock the client resolution: it will be anything in sb
+ StringBuilder sb = new StringBuilder("clientX");
+ LocalResolver localResolver = new LocalResolver(conf, router);
+ LocalResolver spyLocalResolver = spy(localResolver);
+ doAnswer(new Answer<String>() {
+ @Override
+ public String answer(InvocationOnMock invocation) throws Throwable {
+ return sb.toString();
+ }
+ }).when(spyLocalResolver).getClientAddr();
+
+ // Add the mocks to the resolver
+ MultipleDestinationMountTableResolver resolver =
+ new MultipleDestinationMountTableResolver(conf, router);
+ resolver.addResolver(DestinationOrder.LOCAL, spyLocalResolver);
+
+
+ // We point /local to subclusters 0, 1, 2 with the local order
+ Map<String, String> mapLocal = new HashMap<>();
+ mapLocal.put("subcluster0", "/local");
+ mapLocal.put("subcluster1", "/local");
+ mapLocal.put("subcluster2", "/local");
+ MountTable localEntry = MountTable.newInstance("/local", mapLocal);
+ localEntry.setDestOrder(DestinationOrder.LOCAL);
+ resolver.addEntry(localEntry);
+
+ // Test first with the default destination
+ PathLocation dest = resolver.getDestinationForPath("/local/file0.txt");
+ assertDestination("subcluster0", dest);
+
+ // We change the client location and verify
+ setClient(sb, "client2");
+ dest = resolver.getDestinationForPath("/local/file0.txt");
+ assertDestination("subcluster2", dest);
+
+ setClient(sb, "client1");
+ dest = resolver.getDestinationForPath("/local/file0.txt");
+ assertDestination("subcluster1", dest);
+
+ setClient(sb, "client0");
+ dest = resolver.getDestinationForPath("/local/file0.txt");
+ assertDestination("subcluster0", dest);
+ }
+
+ private void assertDestination(String expectedNsId, PathLocation loc) {
+ List<RemoteLocation> dests = loc.getDestinations();
+ RemoteLocation dest = dests.get(0);
+ assertEquals(expectedNsId, dest.getNameserviceId());
+ }
+
+ private MembershipState newMembershipState(String addr, String nsId) {
+ return MembershipState.newInstance(
+ "routerId", nsId, "nn0", "cluster0", "blockPool0",
+ addr + ":8001", addr + ":8002", addr + ":8003", addr + ":8004",
+ FederationNamenodeServiceState.ACTIVE, false);
+ }
+
+ /**
+ * Set the address of the client issuing the request. We use a StringBuilder
+ * to modify the value in place for the mock.
+ * @param sb StringBuilder to set the client string.
+ * @param client Address of the client.
+ */
+ private static void setClient(StringBuilder sb, String client) {
+ sb.replace(0, sb.length(), client);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java
new file mode 100644
index 0000000..741d1f6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java
@@ -0,0 +1,157 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test functionalities of {@link ConnectionManager}, which manages a pool
+ * of connections to NameNodes.
+ */
+public class TestConnectionManager {
+ private Configuration conf;
+ private ConnectionManager connManager;
+ private static final String[] TEST_GROUP = new String[]{"TEST_GROUP"};
+ private static final UserGroupInformation TEST_USER1 =
+ UserGroupInformation.createUserForTesting("user1", TEST_GROUP);
+ private static final UserGroupInformation TEST_USER2 =
+ UserGroupInformation.createUserForTesting("user2", TEST_GROUP);
+ private static final UserGroupInformation TEST_USER3 =
+ UserGroupInformation.createUserForTesting("user3", TEST_GROUP);
+ private static final String TEST_NN_ADDRESS = "nn1:8080";
+
+ @Before
+ public void setup() throws Exception {
+ conf = new Configuration();
+ connManager = new ConnectionManager(conf);
+ NetUtils.addStaticResolution("nn1", "localhost");
+ NetUtils.createSocketAddrForHost("nn1", 8080);
+ connManager.start();
+ }
+
+ @After
+ public void shutdown() {
+ if (connManager != null) {
+ connManager.close();
+ }
+ }
+
+ @Test
+ public void testCleanup() throws Exception {
+ Map<ConnectionPoolId, ConnectionPool> poolMap = connManager.getPools();
+
+ ConnectionPool pool1 = new ConnectionPool(
+ conf, TEST_NN_ADDRESS, TEST_USER1, 0, 10);
+ addConnectionsToPool(pool1, 9, 4);
+ poolMap.put(new ConnectionPoolId(TEST_USER1, TEST_NN_ADDRESS), pool1);
+
+ ConnectionPool pool2 = new ConnectionPool(
+ conf, TEST_NN_ADDRESS, TEST_USER2, 0, 10);
+ addConnectionsToPool(pool2, 10, 10);
+ poolMap.put(new ConnectionPoolId(TEST_USER2, TEST_NN_ADDRESS), pool2);
+
+ checkPoolConnections(TEST_USER1, 9, 4);
+ checkPoolConnections(TEST_USER2, 10, 10);
+
+ // Clean up first pool, one connection should be removed, and second pool
+ // should remain the same.
+ connManager.cleanup(pool1);
+ checkPoolConnections(TEST_USER1, 8, 4);
+ checkPoolConnections(TEST_USER2, 10, 10);
+
+ // Clean up the first pool again, it should have no effect since it reached
+ // the MIN_ACTIVE_RATIO.
+ connManager.cleanup(pool1);
+ checkPoolConnections(TEST_USER1, 8, 4);
+ checkPoolConnections(TEST_USER2, 10, 10);
+
+ // Make sure the number of connections doesn't go below minSize
+ ConnectionPool pool3 = new ConnectionPool(
+ conf, TEST_NN_ADDRESS, TEST_USER3, 2, 10);
+ addConnectionsToPool(pool3, 10, 0);
+ poolMap.put(new ConnectionPoolId(TEST_USER2, TEST_NN_ADDRESS), pool3);
+ connManager.cleanup(pool3);
+ checkPoolConnections(TEST_USER3, 2, 0);
+ // With active connections added to pool, make sure it honors the
+ // MIN_ACTIVE_RATIO again
+ addConnectionsToPool(pool3, 10, 2);
+ connManager.cleanup(pool3);
+ checkPoolConnections(TEST_USER3, 4, 2);
+ }
+
+ @Test
+ public void testGetConnection() throws Exception {
+ Map<ConnectionPoolId, ConnectionPool> poolMap = connManager.getPools();
+ final int totalConns = 10;
+ int activeConns = 5;
+
+ ConnectionPool pool = new ConnectionPool(
+ conf, TEST_NN_ADDRESS, TEST_USER1, 0, 10);
+ addConnectionsToPool(pool, totalConns, activeConns);
+ poolMap.put(new ConnectionPoolId(TEST_USER1, TEST_NN_ADDRESS), pool);
+
+ // All remaining connections should be usable
+ final int remainingSlots = totalConns - activeConns;
+ for (int i = 0; i < remainingSlots; i++) {
+ ConnectionContext cc = pool.getConnection();
+ assertTrue(cc.isUsable());
+ cc.getClient();
+ activeConns++;
+ }
+
+ checkPoolConnections(TEST_USER1, totalConns, activeConns);
+
+ // Ask for more and this returns an active connection
+ ConnectionContext cc = pool.getConnection();
+ assertTrue(cc.isActive());
+ }
+
+ private void addConnectionsToPool(ConnectionPool pool, int numTotalConn,
+ int numActiveConn) throws IOException {
+ for (int i = 0; i < numTotalConn; i++) {
+ ConnectionContext cc = pool.newConnection();
+ pool.addConnection(cc);
+ if (i < numActiveConn) {
+ cc.getClient();
+ }
+ }
+ }
+
+ private void checkPoolConnections(UserGroupInformation ugi,
+ int numOfConns, int numOfActiveConns) {
+ for (Map.Entry<ConnectionPoolId, ConnectionPool> e :
+ connManager.getPools().entrySet()) {
+ if (e.getKey().getUgi() == ugi) {
+ assertEquals(numOfConns, e.getValue().getNumConnections());
+ assertEquals(numOfActiveConns, e.getValue().getNumActiveConnections());
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestNamenodeHeartbeat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestNamenodeHeartbeat.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestNamenodeHeartbeat.java
new file mode 100644
index 0000000..877fb02
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestNamenodeHeartbeat.java
@@ -0,0 +1,168 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMENODES;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMESERVICES;
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.MockResolver;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
+import org.apache.hadoop.service.Service.STATE;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+
+/**
+ * Test the service that heartbeats the state of the namenodes to the State
+ * Store.
+ */
+public class TestNamenodeHeartbeat {
+
+ private static RouterDFSCluster cluster;
+ private static ActiveNamenodeResolver namenodeResolver;
+ private static List<NamenodeHeartbeatService> services;
+
+ @Rule
+ public TestName name = new TestName();
+
+ @BeforeClass
+ public static void globalSetUp() throws Exception {
+
+ cluster = new RouterDFSCluster(true, 2);
+
+ // Start NNs and DNs and wait until ready
+ cluster.startCluster();
+
+ // Mock locator that records the heartbeats
+ List<String> nss = cluster.getNameservices();
+ String ns = nss.get(0);
+ Configuration conf = cluster.generateNamenodeConfiguration(ns);
+ namenodeResolver = new MockResolver(conf);
+ namenodeResolver.setRouterId("testrouter");
+
+ // Create one heartbeat service per NN
+ services = new ArrayList<>();
+ for (NamenodeContext nn : cluster.getNamenodes()) {
+ String nsId = nn.getNameserviceId();
+ String nnId = nn.getNamenodeId();
+ NamenodeHeartbeatService service = new NamenodeHeartbeatService(
+ namenodeResolver, nsId, nnId);
+ service.init(conf);
+ service.start();
+ services.add(service);
+ }
+ }
+
+ @AfterClass
+ public static void tearDown() throws IOException {
+ cluster.shutdown();
+ for (NamenodeHeartbeatService service: services) {
+ service.stop();
+ service.close();
+ }
+ }
+
+ @Test
+ public void testNamenodeHeartbeatService() throws IOException {
+
+ RouterDFSCluster testCluster = new RouterDFSCluster(true, 1);
+ Configuration heartbeatConfig = testCluster.generateNamenodeConfiguration(
+ NAMESERVICES[0]);
+ NamenodeHeartbeatService server = new NamenodeHeartbeatService(
+ namenodeResolver, NAMESERVICES[0], NAMENODES[0]);
+ server.init(heartbeatConfig);
+ assertEquals(STATE.INITED, server.getServiceState());
+ server.start();
+ assertEquals(STATE.STARTED, server.getServiceState());
+ server.stop();
+ assertEquals(STATE.STOPPED, server.getServiceState());
+ server.close();
+ }
+
+ @Test
+ public void testHearbeat() throws InterruptedException, IOException {
+
+ // Set NAMENODE1 to active for all nameservices
+ if (cluster.isHighAvailability()) {
+ for (String ns : cluster.getNameservices()) {
+ cluster.switchToActive(ns, NAMENODES[0]);
+ cluster.switchToStandby(ns, NAMENODES[1]);
+ }
+ }
+
+ // Wait for heartbeats to record
+ Thread.sleep(5000);
+
+ // Verify the locator has matching NN entries for each NS
+ for (String ns : cluster.getNameservices()) {
+ List<? extends FederationNamenodeContext> nns =
+ namenodeResolver.getNamenodesForNameserviceId(ns);
+
+ // Active
+ FederationNamenodeContext active = nns.get(0);
+ assertEquals(NAMENODES[0], active.getNamenodeId());
+
+ // Standby
+ FederationNamenodeContext standby = nns.get(1);
+ assertEquals(NAMENODES[1], standby.getNamenodeId());
+ }
+
+ // Switch active NNs in 1/2 nameservices
+ List<String> nss = cluster.getNameservices();
+ String failoverNS = nss.get(0);
+ String normalNs = nss.get(1);
+
+ cluster.switchToStandby(failoverNS, NAMENODES[0]);
+ cluster.switchToActive(failoverNS, NAMENODES[1]);
+
+ // Wait for heartbeats to record
+ Thread.sleep(5000);
+
+ // Verify the locator has recorded the failover for the failover NS
+ List<? extends FederationNamenodeContext> failoverNSs =
+ namenodeResolver.getNamenodesForNameserviceId(failoverNS);
+ // Active
+ FederationNamenodeContext active = failoverNSs.get(0);
+ assertEquals(NAMENODES[1], active.getNamenodeId());
+
+ // Standby
+ FederationNamenodeContext standby = failoverNSs.get(1);
+ assertEquals(NAMENODES[0], standby.getNamenodeId());
+
+ // Verify the locator has the same records for the other ns
+ List<? extends FederationNamenodeContext> normalNss =
+ namenodeResolver.getNamenodesForNameserviceId(normalNs);
+ // Active
+ active = normalNss.get(0);
+ assertEquals(NAMENODES[0], active.getNamenodeId());
+ // Standby
+ standby = normalNss.get(1);
+ assertEquals(NAMENODES[1], standby.getNamenodeId());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRBFConfigFields.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRBFConfigFields.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRBFConfigFields.java
new file mode 100644
index 0000000..e130b7b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRBFConfigFields.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.hadoop.conf.TestConfigurationFieldsBase;
+
+import java.util.HashSet;
+
+/**
+ * Unit test class to compare the following RBF configuration class:
+ * <p></p>
+ * {@link RBFConfigKeys}
+ * <p></p>
+ * against hdfs-rbf-default.xml for missing properties.
+ * <p></p>
+ * Refer to {@link org.apache.hadoop.conf.TestConfigurationFieldsBase}
+ * for how this class works.
+ */
+public class TestRBFConfigFields extends TestConfigurationFieldsBase {
+ @Override
+ public void initializeMemberVariables() {
+ xmlFilename = "hdfs-rbf-default.xml";
+ configurationClasses = new Class[] {RBFConfigKeys.class};
+
+ // Set error modes
+ errorIfMissingConfigProps = true;
+ errorIfMissingXmlProps = true;
+
+ // Initialize used variables
+ configurationPropsToSkipCompare = new HashSet<String>();
+
+ // Allocate
+ xmlPropsToSkipCompare = new HashSet<String>();
+ xmlPrefixToSkipCompare = new HashSet<String>();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
new file mode 100644
index 0000000..39398f7
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.MockResolver;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.service.Service.STATE;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * The the safe mode for the {@link Router} controlled by
+ * {@link SafeModeTimer}.
+ */
+public class TestRouter {
+
+ private static Configuration conf;
+
+ @BeforeClass
+ public static void create() throws IOException {
+ // Basic configuration without the state store
+ conf = new Configuration();
+ // 1 sec cache refresh
+ conf.setInt(RBFConfigKeys.DFS_ROUTER_CACHE_TIME_TO_LIVE_MS, 1);
+ // Mock resolver classes
+ conf.setClass(RBFConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS,
+ MockResolver.class, ActiveNamenodeResolver.class);
+ conf.setClass(RBFConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
+ MockResolver.class, FileSubclusterResolver.class);
+
+ // Bind to any available port
+ conf.set(RBFConfigKeys.DFS_ROUTER_RPC_BIND_HOST_KEY, "0.0.0.0");
+ conf.set(RBFConfigKeys.DFS_ROUTER_RPC_ADDRESS_KEY, "127.0.0.1:0");
+ conf.set(RBFConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_KEY, "127.0.0.1:0");
+ conf.set(RBFConfigKeys.DFS_ROUTER_ADMIN_BIND_HOST_KEY, "0.0.0.0");
+ conf.set(RBFConfigKeys.DFS_ROUTER_HTTP_ADDRESS_KEY, "127.0.0.1:0");
+ conf.set(RBFConfigKeys.DFS_ROUTER_HTTPS_ADDRESS_KEY, "127.0.0.1:0");
+ conf.set(RBFConfigKeys.DFS_ROUTER_HTTP_BIND_HOST_KEY, "0.0.0.0");
+
+ // Simulate a co-located NN
+ conf.set(DFSConfigKeys.DFS_NAMESERVICES, "ns0");
+ conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://" + "ns0");
+ conf.set(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + "ns0",
+ "127.0.0.1:0" + 0);
+ conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY + "." + "ns0",
+ "127.0.0.1:" + 0);
+ conf.set(DFSConfigKeys.DFS_NAMENODE_RPC_BIND_HOST_KEY + "." + "ns0",
+ "0.0.0.0");
+ }
+
+ @AfterClass
+ public static void destroy() {
+ }
+
+ @Before
+ public void setup() throws IOException, URISyntaxException {
+ }
+
+ @After
+ public void cleanup() {
+ }
+
+ private static void testRouterStartup(Configuration routerConfig)
+ throws InterruptedException, IOException {
+ Router router = new Router();
+ assertEquals(STATE.NOTINITED, router.getServiceState());
+ router.init(routerConfig);
+ assertEquals(STATE.INITED, router.getServiceState());
+ router.start();
+ assertEquals(STATE.STARTED, router.getServiceState());
+ router.stop();
+ assertEquals(STATE.STOPPED, router.getServiceState());
+ router.close();
+ }
+
+ @Test
+ public void testRouterService() throws InterruptedException, IOException {
+
+ // Admin only
+ testRouterStartup(new RouterConfigBuilder(conf).admin().build());
+
+ // Http only
+ testRouterStartup(new RouterConfigBuilder(conf).http().build());
+
+ // Rpc only
+ testRouterStartup(new RouterConfigBuilder(conf).rpc().build());
+
+ // Metrics only
+ testRouterStartup(new RouterConfigBuilder(conf).metrics().build());
+
+ // Statestore only
+ testRouterStartup(new RouterConfigBuilder(conf).stateStore().build());
+
+ // Heartbeat only
+ testRouterStartup(new RouterConfigBuilder(conf).heartbeat().build());
+
+ // Run with all services
+ testRouterStartup(new RouterConfigBuilder(conf).all().build());
+ }
+
+ @Test
+ public void testRouterRestartRpcService() throws IOException {
+
+ // Start
+ Router router = new Router();
+ router.init(new RouterConfigBuilder(conf).rpc().build());
+ router.start();
+
+ // Verify RPC server is running
+ assertNotNull(router.getRpcServerAddress());
+ RouterRpcServer rpcServer = router.getRpcServer();
+ assertNotNull(rpcServer);
+ assertEquals(STATE.STARTED, rpcServer.getServiceState());
+
+ // Stop router and RPC server
+ router.stop();
+ assertEquals(STATE.STOPPED, rpcServer.getServiceState());
+
+ router.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
new file mode 100644
index 0000000..a8ffded
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
@@ -0,0 +1,340 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.synchronizeRecords;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
+import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.impl.MountTableStoreImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.util.Time;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * The administrator interface of the {@link Router} implemented by
+ * {@link RouterAdminServer}.
+ */
+public class TestRouterAdmin {
+
+ private static StateStoreDFSCluster cluster;
+ private static RouterContext routerContext;
+ public static final String RPC_BEAN =
+ "Hadoop:service=Router,name=FederationRPC";
+ private static List<MountTable> mockMountTable;
+ private static StateStoreService stateStore;
+
+ @BeforeClass
+ public static void globalSetUp() throws Exception {
+ cluster = new StateStoreDFSCluster(false, 1);
+ // Build and start a router with State Store + admin + RPC
+ Configuration conf = new RouterConfigBuilder()
+ .stateStore()
+ .admin()
+ .rpc()
+ .build();
+ cluster.addRouterOverrides(conf);
+ cluster.startRouters();
+ routerContext = cluster.getRandomRouter();
+ mockMountTable = cluster.generateMockMountTable();
+ Router router = routerContext.getRouter();
+ stateStore = router.getStateStore();
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ cluster.stopRouter(routerContext);
+ }
+
+ @Before
+ public void testSetup() throws Exception {
+ assertTrue(
+ synchronizeRecords(stateStore, mockMountTable, MountTable.class));
+ }
+
+ @Test
+ public void testAddMountTable() throws IOException {
+ MountTable newEntry = MountTable.newInstance(
+ "/testpath", Collections.singletonMap("ns0", "/testdir"),
+ Time.now(), Time.now());
+
+ RouterClient client = routerContext.getAdminClient();
+ MountTableManager mountTable = client.getMountTableManager();
+
+ // Existing mount table size
+ List<MountTable> records = getMountTableEntries(mountTable);
+ assertEquals(records.size(), mockMountTable.size());
+
+ // Add
+ AddMountTableEntryRequest addRequest =
+ AddMountTableEntryRequest.newInstance(newEntry);
+ AddMountTableEntryResponse addResponse =
+ mountTable.addMountTableEntry(addRequest);
+ assertTrue(addResponse.getStatus());
+
+ // New mount table size
+ List<MountTable> records2 = getMountTableEntries(mountTable);
+ assertEquals(records2.size(), mockMountTable.size() + 1);
+ }
+
+ @Test
+ public void testAddDuplicateMountTable() throws IOException {
+ MountTable newEntry = MountTable.newInstance("/testpath",
+ Collections.singletonMap("ns0", "/testdir"), Time.now(), Time.now());
+
+ RouterClient client = routerContext.getAdminClient();
+ MountTableManager mountTable = client.getMountTableManager();
+
+ // Existing mount table size
+ List<MountTable> entries1 = getMountTableEntries(mountTable);
+ assertEquals(entries1.size(), mockMountTable.size());
+
+ // Add
+ AddMountTableEntryRequest addRequest =
+ AddMountTableEntryRequest.newInstance(newEntry);
+ AddMountTableEntryResponse addResponse =
+ mountTable.addMountTableEntry(addRequest);
+ assertTrue(addResponse.getStatus());
+
+ // New mount table size
+ List<MountTable> entries2 = getMountTableEntries(mountTable);
+ assertEquals(entries2.size(), mockMountTable.size() + 1);
+
+ // Add again, should fail
+ AddMountTableEntryResponse addResponse2 =
+ mountTable.addMountTableEntry(addRequest);
+ assertFalse(addResponse2.getStatus());
+ }
+
+ @Test
+ public void testAddReadOnlyMountTable() throws IOException {
+ MountTable newEntry = MountTable.newInstance(
+ "/readonly", Collections.singletonMap("ns0", "/testdir"),
+ Time.now(), Time.now());
+ newEntry.setReadOnly(true);
+
+ RouterClient client = routerContext.getAdminClient();
+ MountTableManager mountTable = client.getMountTableManager();
+
+ // Existing mount table size
+ List<MountTable> records = getMountTableEntries(mountTable);
+ assertEquals(records.size(), mockMountTable.size());
+
+ // Add
+ AddMountTableEntryRequest addRequest =
+ AddMountTableEntryRequest.newInstance(newEntry);
+ AddMountTableEntryResponse addResponse =
+ mountTable.addMountTableEntry(addRequest);
+ assertTrue(addResponse.getStatus());
+
+ // New mount table size
+ List<MountTable> records2 = getMountTableEntries(mountTable);
+ assertEquals(records2.size(), mockMountTable.size() + 1);
+
+ // Check that we have the read only entry
+ MountTable record = getMountTableEntry("/readonly");
+ assertEquals("/readonly", record.getSourcePath());
+ assertTrue(record.isReadOnly());
+
+ // Removing the new entry
+ RemoveMountTableEntryRequest removeRequest =
+ RemoveMountTableEntryRequest.newInstance("/readonly");
+ RemoveMountTableEntryResponse removeResponse =
+ mountTable.removeMountTableEntry(removeRequest);
+ assertTrue(removeResponse.getStatus());
+ }
+
+ @Test
+ public void testAddOrderMountTable() throws IOException {
+ testAddOrderMountTable(DestinationOrder.HASH);
+ testAddOrderMountTable(DestinationOrder.LOCAL);
+ testAddOrderMountTable(DestinationOrder.RANDOM);
+ testAddOrderMountTable(DestinationOrder.HASH_ALL);
+ }
+
+ private void testAddOrderMountTable(final DestinationOrder order)
+ throws IOException {
+ final String mnt = "/" + order;
+ MountTable newEntry = MountTable.newInstance(
+ mnt, Collections.singletonMap("ns0", "/testdir"),
+ Time.now(), Time.now());
+ newEntry.setDestOrder(order);
+
+ RouterClient client = routerContext.getAdminClient();
+ MountTableManager mountTable = client.getMountTableManager();
+
+ // Add
+ AddMountTableEntryRequest addRequest;
+ AddMountTableEntryResponse addResponse;
+ addRequest = AddMountTableEntryRequest.newInstance(newEntry);
+ addResponse = mountTable.addMountTableEntry(addRequest);
+ assertTrue(addResponse.getStatus());
+
+ // Check that we have the read only entry
+ MountTable record = getMountTableEntry(mnt);
+ assertEquals(mnt, record.getSourcePath());
+ assertEquals(order, record.getDestOrder());
+
+ // Removing the new entry
+ RemoveMountTableEntryRequest removeRequest =
+ RemoveMountTableEntryRequest.newInstance(mnt);
+ RemoveMountTableEntryResponse removeResponse =
+ mountTable.removeMountTableEntry(removeRequest);
+ assertTrue(removeResponse.getStatus());
+ }
+
+ @Test
+ public void testRemoveMountTable() throws IOException {
+
+ RouterClient client = routerContext.getAdminClient();
+ MountTableManager mountTable = client.getMountTableManager();
+
+ // Existing mount table size
+ List<MountTable> entries1 = getMountTableEntries(mountTable);
+ assertEquals(entries1.size(), mockMountTable.size());
+
+ // Remove an entry
+ RemoveMountTableEntryRequest removeRequest =
+ RemoveMountTableEntryRequest.newInstance("/");
+ mountTable.removeMountTableEntry(removeRequest);
+
+ // New mount table size
+ List<MountTable> entries2 = getMountTableEntries(mountTable);
+ assertEquals(entries2.size(), mockMountTable.size() - 1);
+ }
+
+ @Test
+ public void testEditMountTable() throws IOException {
+
+ RouterClient client = routerContext.getAdminClient();
+ MountTableManager mountTable = client.getMountTableManager();
+
+ // Verify starting condition
+ MountTable entry = getMountTableEntry("/");
+ assertEquals(
+ Collections.singletonList(new RemoteLocation("ns0", "/")),
+ entry.getDestinations());
+
+ // Edit the entry for /
+ MountTable updatedEntry = MountTable.newInstance(
+ "/", Collections.singletonMap("ns1", "/"), Time.now(), Time.now());
+ UpdateMountTableEntryRequest updateRequest =
+ UpdateMountTableEntryRequest.newInstance(updatedEntry);
+ mountTable.updateMountTableEntry(updateRequest);
+
+ // Verify edited condition
+ entry = getMountTableEntry("/");
+ assertEquals(
+ Collections.singletonList(new RemoteLocation("ns1", "/")),
+ entry.getDestinations());
+ }
+
+ @Test
+ public void testGetMountTable() throws IOException {
+
+ RouterClient client = routerContext.getAdminClient();
+ MountTableManager mountTable = client.getMountTableManager();
+
+ // Verify size of table
+ List<MountTable> entries = getMountTableEntries(mountTable);
+ assertEquals(mockMountTable.size(), entries.size());
+
+ // Verify all entries are present
+ int matches = 0;
+ for (MountTable e : entries) {
+ for (MountTable entry : mockMountTable) {
+ assertEquals(e.getDestinations().size(), 1);
+ assertNotNull(e.getDateCreated());
+ assertNotNull(e.getDateModified());
+ if (entry.getSourcePath().equals(e.getSourcePath())) {
+ matches++;
+ }
+ }
+ }
+ assertEquals(matches, mockMountTable.size());
+ }
+
+ @Test
+ public void testGetSingleMountTableEntry() throws IOException {
+ MountTable entry = getMountTableEntry("/ns0");
+ assertNotNull(entry);
+ assertEquals(entry.getSourcePath(), "/ns0");
+ }
+
+ /**
+ * Gets an existing mount table record in the state store.
+ *
+ * @param mount The mount point of the record to remove.
+ * @return The matching record if found, null if it is not found.
+ * @throws IOException If the state store could not be accessed.
+ */
+ private MountTable getMountTableEntry(final String mount) throws IOException {
+ // Refresh the cache
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+
+ GetMountTableEntriesRequest request =
+ GetMountTableEntriesRequest.newInstance(mount);
+ RouterClient client = routerContext.getAdminClient();
+ MountTableManager mountTable = client.getMountTableManager();
+ List<MountTable> results = getMountTableEntries(mountTable, request);
+ if (results.size() > 0) {
+ // First result is sorted to have the shortest mount string length
+ return results.get(0);
+ }
+ return null;
+ }
+
+ private List<MountTable> getMountTableEntries(MountTableManager mountTable)
+ throws IOException {
+ GetMountTableEntriesRequest request =
+ GetMountTableEntriesRequest.newInstance("/");
+ return getMountTableEntries(mountTable, request);
+ }
+
+ private List<MountTable> getMountTableEntries(MountTableManager mountTable,
+ GetMountTableEntriesRequest request) throws IOException {
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+ GetMountTableEntriesResponse response =
+ mountTable.getMountTableEntries(request);
+ return response.getEntries();
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateNamenodeRegistrationRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateNamenodeRegistrationRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateNamenodeRegistrationRequestPBImpl.java
new file mode 100644
index 0000000..5091360
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateNamenodeRegistrationRequestPBImpl.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateNamenodeRegistrationRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateNamenodeRegistrationRequestProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateNamenodeRegistrationRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * OverrideNamenodeRegistrationRequest.
+ */
+public class UpdateNamenodeRegistrationRequestPBImpl
+ extends UpdateNamenodeRegistrationRequest implements PBRecord {
+
+ private FederationProtocolPBTranslator<
+ UpdateNamenodeRegistrationRequestProto,
+ UpdateNamenodeRegistrationRequestProto.Builder,
+ UpdateNamenodeRegistrationRequestProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<
+ UpdateNamenodeRegistrationRequestProto,
+ UpdateNamenodeRegistrationRequestProto.Builder,
+ UpdateNamenodeRegistrationRequestProtoOrBuilder>(
+ UpdateNamenodeRegistrationRequestProto.class);
+
+ public UpdateNamenodeRegistrationRequestPBImpl() {
+ }
+
+ @Override
+ public UpdateNamenodeRegistrationRequestProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message protocol) {
+ this.translator.setProto(protocol);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public String getNameserviceId() {
+ return this.translator.getProtoOrBuilder().getNameserviceId();
+ }
+
+ @Override
+ public String getNamenodeId() {
+ return this.translator.getProtoOrBuilder().getNamenodeId();
+ }
+
+ @Override
+ public FederationNamenodeServiceState getState() {
+ return FederationNamenodeServiceState
+ .valueOf(this.translator.getProtoOrBuilder().getState());
+ }
+
+ @Override
+ public void setNameserviceId(String nsId) {
+ this.translator.getBuilder().setNameserviceId(nsId);
+ }
+
+ @Override
+ public void setNamenodeId(String nnId) {
+ this.translator.getBuilder().setNamenodeId(nnId);
+ }
+
+ @Override
+ public void setState(FederationNamenodeServiceState state) {
+ this.translator.getBuilder().setState(state.toString());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateNamenodeRegistrationResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateNamenodeRegistrationResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateNamenodeRegistrationResponsePBImpl.java
new file mode 100644
index 0000000..4558f06
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateNamenodeRegistrationResponsePBImpl.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateNamenodeRegistrationResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateNamenodeRegistrationResponseProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateNamenodeRegistrationResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * OverrideNamenodeRegistrationResponse.
+ */
+public class UpdateNamenodeRegistrationResponsePBImpl
+ extends UpdateNamenodeRegistrationResponse implements PBRecord {
+
+ private FederationProtocolPBTranslator<
+ UpdateNamenodeRegistrationResponseProto,
+ UpdateNamenodeRegistrationResponseProto.Builder,
+ UpdateNamenodeRegistrationResponseProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<
+ UpdateNamenodeRegistrationResponseProto,
+ UpdateNamenodeRegistrationResponseProto.Builder,
+ UpdateNamenodeRegistrationResponseProtoOrBuilder>(
+ UpdateNamenodeRegistrationResponseProto.class);
+
+ public UpdateNamenodeRegistrationResponsePBImpl() {
+ }
+
+ @Override
+ public UpdateNamenodeRegistrationResponseProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public boolean getResult() {
+ return this.translator.getProtoOrBuilder().getStatus();
+ }
+
+ @Override
+ public void setResult(boolean result) {
+ this.translator.getBuilder().setStatus(result);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/package-info.java
new file mode 100644
index 0000000..43c94be
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/package-info.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Protobuf implementations of FederationProtocolBase request/response objects
+ * used by state store APIs. Each state store API is defined in the
+ * org.apache.hadoop.hdfs.server.federation.store.protocol package.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/package-info.java
new file mode 100644
index 0000000..0249d2c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/package-info.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Contains the abstract definitions of the API request and response objects for
+ * the various state store APIs. The state store supports multiple interface
+ * APIs and multiple data records. Each protocol object requires a serialization
+ * implementation, the default is protobuf.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java
new file mode 100644
index 0000000..d5e60ce
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java
@@ -0,0 +1,215 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+import java.util.Map;
+
+import org.apache.hadoop.util.Time;
+
+/**
+ * Abstract base of a data record in the StateStore. All StateStore records are
+ * derived from this class. Data records are persisted in the data store and
+ * are identified by their primary key. Each data record contains:
+ * <ul>
+ * <li>A primary key consisting of a combination of record data fields.
+ * <li>A modification date.
+ * <li>A creation date.
+ * </ul>
+ */
+public abstract class BaseRecord implements Comparable<BaseRecord> {
+ public static final String ERROR_MSG_CREATION_TIME_NEGATIVE =
+ "The creation time for the record cannot be negative.";
+ public static final String ERROR_MSG_MODIFICATION_TIME_NEGATIVE =
+ "The modification time for the record cannot be negative.";
+
+ /**
+ * Set the modification time for the record.
+ *
+ * @param time Modification time of the record.
+ */
+ public abstract void setDateModified(long time);
+
+ /**
+ * Get the modification time for the record.
+ *
+ * @return Modification time of the record.
+ */
+ public abstract long getDateModified();
+
+ /**
+ * Set the creation time for the record.
+ *
+ * @param time Creation time of the record.
+ */
+ public abstract void setDateCreated(long time);
+
+ /**
+ * Get the creation time for the record.
+ *
+ * @return Creation time of the record
+ */
+ public abstract long getDateCreated();
+
+ /**
+ * Get the expiration time for the record.
+ *
+ * @return Expiration time for the record.
+ */
+ public abstract long getExpirationMs();
+
+ /**
+ * Map of primary key names->values for the record. The primary key can be a
+ * combination of 1-n different State Store serialized values.
+ *
+ * @return Map of key/value pairs that constitute this object's primary key.
+ */
+ public abstract Map<String, String> getPrimaryKeys();
+
+ /**
+ * Initialize the object.
+ */
+ public void init() {
+ // Call this after the object has been constructed
+ initDefaultTimes();
+ }
+
+ /**
+ * Initialize default times. The driver may update these timestamps on insert
+ * and/or update. This should only be called when initializing an object that
+ * is not backed by a data store.
+ */
+ private void initDefaultTimes() {
+ long now = Time.now();
+ this.setDateCreated(now);
+ this.setDateModified(now);
+ }
+
+ /**
+ * Join the primary keys into one single primary key.
+ *
+ * @return A string that is guaranteed to be unique amongst all records of
+ * this type.
+ */
+ public String getPrimaryKey() {
+ return generateMashupKey(getPrimaryKeys());
+ }
+
+ /**
+ * Generates a cache key from a map of values.
+ *
+ * @param keys Map of values.
+ * @return String mashup of key values.
+ */
+ protected static String generateMashupKey(final Map<String, String> keys) {
+ StringBuilder builder = new StringBuilder();
+ for (Object value : keys.values()) {
+ if (builder.length() > 0) {
+ builder.append("-");
+ }
+ builder.append(value);
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Check if this record matches a partial record.
+ *
+ * @param other Partial record.
+ * @return If this record matches.
+ */
+ public boolean like(BaseRecord other) {
+ if (other == null) {
+ return false;
+ }
+ Map<String, String> thisKeys = this.getPrimaryKeys();
+ Map<String, String> otherKeys = other.getPrimaryKeys();
+ if (thisKeys == null) {
+ return otherKeys == null;
+ }
+ return thisKeys.equals(otherKeys);
+ }
+
+ /**
+ * Override equals check to use primary key(s) for comparison.
+ */
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof BaseRecord)) {
+ return false;
+ }
+
+ BaseRecord baseObject = (BaseRecord) obj;
+ Map<String, String> keyset1 = this.getPrimaryKeys();
+ Map<String, String> keyset2 = baseObject.getPrimaryKeys();
+ return keyset1.equals(keyset2);
+ }
+
+ /**
+ * Override hash code to use primary key(s) for comparison.
+ */
+ @Override
+ public int hashCode() {
+ Map<String, String> keyset = this.getPrimaryKeys();
+ return keyset.hashCode();
+ }
+
+ @Override
+ public int compareTo(BaseRecord record) {
+ if (record == null) {
+ return -1;
+ }
+ // Descending date order
+ return (int) (record.getDateModified() - this.getDateModified());
+ }
+
+ /**
+ * Called when the modification time and current time is available, checks for
+ * expirations.
+ *
+ * @param currentTime The current timestamp in ms from the data store, to be
+ * compared against the modification and creation dates of the
+ * object.
+ * @return boolean True if the record has been updated and should be
+ * committed to the data store. Override for customized behavior.
+ */
+ public boolean checkExpired(long currentTime) {
+ long expiration = getExpirationMs();
+ if (getDateModified() > 0 && expiration > 0) {
+ return (getDateModified() + expiration) < currentTime;
+ }
+ return false;
+ }
+
+ /**
+ * Validates the record. Called when the record is created, populated from the
+ * state store, and before committing to the state store. If validate failed,
+ * there throws an exception.
+ */
+ public void validate() {
+ if (getDateCreated() <= 0) {
+ throw new IllegalArgumentException(ERROR_MSG_CREATION_TIME_NEGATIVE);
+ } else if (getDateModified() <= 0) {
+ throw new IllegalArgumentException(ERROR_MSG_MODIFICATION_TIME_NEGATIVE);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return getPrimaryKey();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipState.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipState.java
new file mode 100644
index 0000000..e33dedf
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipState.java
@@ -0,0 +1,336 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.ACTIVE;
+import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.EXPIRED;
+import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.UNAVAILABLE;
+
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * Data schema for storing NN registration information in the
+ * {@link org.apache.hadoop.hdfs.server.federation.store.StateStoreService
+ * FederationStateStoreService}.
+ */
+public abstract class MembershipState extends BaseRecord
+ implements FederationNamenodeContext {
+ public static final String ERROR_MSG_NO_NS_SPECIFIED =
+ "Invalid registration, no nameservice specified ";
+ public static final String ERROR_MSG_NO_WEB_ADDR_SPECIFIED =
+ "Invalid registration, no web address specified ";
+ public static final String ERROR_MSG_NO_RPC_ADDR_SPECIFIED =
+ "Invalid registration, no rpc address specified ";
+ public static final String ERROR_MSG_NO_BP_SPECIFIED =
+ "Invalid registration, no block pool specified ";
+
+ /** Expiration time in ms for this entry. */
+ private static long expirationMs;
+
+
+ /** Comparator based on the name.*/
+ public static final Comparator<MembershipState> NAME_COMPARATOR =
+ new Comparator<MembershipState>() {
+ public int compare(MembershipState m1, MembershipState m2) {
+ return m1.compareNameTo(m2);
+ }
+ };
+
+
+ /**
+ * Constructors.
+ */
+ public MembershipState() {
+ super();
+ }
+
+ /**
+ * Create a new membership instance.
+ * @return Membership instance.
+ * @throws IOException
+ */
+ public static MembershipState newInstance() {
+ MembershipState record =
+ StateStoreSerializer.newRecord(MembershipState.class);
+ record.init();
+ return record;
+ }
+
+ /**
+ * Create a new membership instance.
+ *
+ * @param router Identifier of the router.
+ * @param nameservice Identifier of the nameservice.
+ * @param namenode Identifier of the namenode.
+ * @param clusterId Identifier of the cluster.
+ * @param blockPoolId Identifier of the blockpool.
+ * @param rpcAddress RPC address.
+ * @param serviceAddress Service RPC address.
+ * @param lifelineAddress Lifeline RPC address.
+ * @param webAddress HTTP address.
+ * @param state State of the federation.
+ * @param safemode If the safe mode is enabled.
+ * @return Membership instance.
+ * @throws IOException If we cannot create the instance.
+ */
+ public static MembershipState newInstance(String router, String nameservice,
+ String namenode, String clusterId, String blockPoolId, String rpcAddress,
+ String serviceAddress, String lifelineAddress, String webAddress,
+ FederationNamenodeServiceState state, boolean safemode) {
+
+ MembershipState record = MembershipState.newInstance();
+ record.setRouterId(router);
+ record.setNameserviceId(nameservice);
+ record.setNamenodeId(namenode);
+ record.setRpcAddress(rpcAddress);
+ record.setServiceAddress(serviceAddress);
+ record.setLifelineAddress(lifelineAddress);
+ record.setWebAddress(webAddress);
+ record.setIsSafeMode(safemode);
+ record.setState(state);
+ record.setClusterId(clusterId);
+ record.setBlockPoolId(blockPoolId);
+ record.validate();
+ return record;
+ }
+
+ public abstract void setRouterId(String routerId);
+
+ public abstract String getRouterId();
+
+ public abstract void setNameserviceId(String nameserviceId);
+
+ public abstract void setNamenodeId(String namenodeId);
+
+ public abstract void setWebAddress(String webAddress);
+
+ public abstract void setRpcAddress(String rpcAddress);
+
+ public abstract void setServiceAddress(String serviceAddress);
+
+ public abstract void setLifelineAddress(String lifelineAddress);
+
+ public abstract void setIsSafeMode(boolean isSafeMode);
+
+ public abstract void setClusterId(String clusterId);
+
+ public abstract void setBlockPoolId(String blockPoolId);
+
+ public abstract void setState(FederationNamenodeServiceState state);
+
+ public abstract String getNameserviceId();
+
+ public abstract String getNamenodeId();
+
+ public abstract String getClusterId();
+
+ public abstract String getBlockPoolId();
+
+ public abstract String getRpcAddress();
+
+ public abstract String getServiceAddress();
+
+ public abstract String getLifelineAddress();
+
+ public abstract String getWebAddress();
+
+ public abstract boolean getIsSafeMode();
+
+ public abstract FederationNamenodeServiceState getState();
+
+ public abstract void setStats(MembershipStats stats);
+
+ public abstract MembershipStats getStats();
+
+ public abstract void setLastContact(long contact);
+
+ public abstract long getLastContact();
+
+ @Override
+ public boolean like(BaseRecord o) {
+ if (o instanceof MembershipState) {
+ MembershipState other = (MembershipState)o;
+ if (getRouterId() != null &&
+ !getRouterId().equals(other.getRouterId())) {
+ return false;
+ }
+ if (getNameserviceId() != null &&
+ !getNameserviceId().equals(other.getNameserviceId())) {
+ return false;
+ }
+ if (getNamenodeId() != null &&
+ !getNamenodeId().equals(other.getNamenodeId())) {
+ return false;
+ }
+ if (getRpcAddress() != null &&
+ !getRpcAddress().equals(other.getRpcAddress())) {
+ return false;
+ }
+ if (getClusterId() != null &&
+ !getClusterId().equals(other.getClusterId())) {
+ return false;
+ }
+ if (getBlockPoolId() != null &&
+ !getBlockPoolId().equals(other.getBlockPoolId())) {
+ return false;
+ }
+ if (getState() != null &&
+ !getState().equals(other.getState())) {
+ return false;
+ }
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return getRouterId() + "->" + getNameserviceId() + ":" + getNamenodeId()
+ + ":" + getRpcAddress() + "-" + getState();
+ }
+
+ @Override
+ public SortedMap<String, String> getPrimaryKeys() {
+ SortedMap<String, String> map = new TreeMap<String, String>();
+ map.put("routerId", getRouterId());
+ map.put("nameserviceId", getNameserviceId());
+ map.put("namenodeId", getNamenodeId());
+ return map;
+ }
+
+ /**
+ * Check if the namenode is available.
+ *
+ * @return If the namenode is available.
+ */
+ public boolean isAvailable() {
+ return getState() == ACTIVE;
+ }
+
+ /**
+ * Validates the entry. Throws an IllegalArgementException if the data record
+ * is missing required information.
+ */
+ @Override
+ public void validate() {
+ super.validate();
+ if (getNameserviceId() == null || getNameserviceId().length() == 0) {
+ throw new IllegalArgumentException(
+ ERROR_MSG_NO_NS_SPECIFIED + this);
+ }
+ if (getWebAddress() == null || getWebAddress().length() == 0) {
+ throw new IllegalArgumentException(
+ ERROR_MSG_NO_WEB_ADDR_SPECIFIED + this);
+ }
+ if (getRpcAddress() == null || getRpcAddress().length() == 0) {
+ throw new IllegalArgumentException(
+ ERROR_MSG_NO_RPC_ADDR_SPECIFIED + this);
+ }
+ if (!isBadState() &&
+ (getBlockPoolId().isEmpty() || getBlockPoolId().length() == 0)) {
+ throw new IllegalArgumentException(
+ ERROR_MSG_NO_BP_SPECIFIED + this);
+ }
+ }
+
+
+ /**
+ * Overrides the cached getBlockPoolId() with an update. The state will be
+ * reset when the cache is flushed
+ *
+ * @param newState Service state of the namenode.
+ */
+ public void overrideState(FederationNamenodeServiceState newState) {
+ this.setState(newState);
+ }
+
+ /**
+ * Sort by nameservice, namenode, and router.
+ *
+ * @param other Another membership to compare to.
+ * @return If this object goes before the parameter.
+ */
+ public int compareNameTo(MembershipState other) {
+ int ret = this.getNameserviceId().compareTo(other.getNameserviceId());
+ if (ret == 0) {
+ ret = this.getNamenodeId().compareTo(other.getNamenodeId());
+ }
+ if (ret == 0) {
+ ret = this.getRouterId().compareTo(other.getRouterId());
+ }
+ return ret;
+ }
+
+ /**
+ * Get the identifier of this namenode registration.
+ * @return Identifier of the namenode.
+ */
+ public String getNamenodeKey() {
+ return getNamenodeKey(this.getNameserviceId(), this.getNamenodeId());
+ }
+
+ /**
+ * Generate the identifier for a Namenode in the HDFS federation.
+ *
+ * @param nsId Nameservice of the Namenode.
+ * @param nnId Namenode within the Nameservice (HA).
+ * @return Namenode identifier within the federation.
+ */
+ public static String getNamenodeKey(String nsId, String nnId) {
+ return nsId + "-" + nnId;
+ }
+
+ /**
+ * Check if the membership is in a bad state (expired or unavailable).
+ * @return If the membership is in a bad state (expired or unavailable).
+ */
+ private boolean isBadState() {
+ return this.getState() == EXPIRED || this.getState() == UNAVAILABLE;
+ }
+
+ @Override
+ public boolean checkExpired(long currentTime) {
+ if (super.checkExpired(currentTime)) {
+ this.setState(EXPIRED);
+ // Commit it
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public long getExpirationMs() {
+ return MembershipState.expirationMs;
+ }
+
+ /**
+ * Set the expiration time for this class.
+ *
+ * @param time Expiration time in milliseconds.
+ */
+ public static void setExpirationMs(long time) {
+ MembershipState.expirationMs = time;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipStats.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipStats.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipStats.java
new file mode 100644
index 0000000..654140c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipStats.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+import java.io.IOException;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * Data schema for storing NN stats in the
+ * {@link org.apache.hadoop.hdfs.server.federation.store.StateStoreService
+ * StateStoreService}.
+ */
+public abstract class MembershipStats extends BaseRecord {
+
+ public static MembershipStats newInstance() throws IOException {
+ MembershipStats record =
+ StateStoreSerializer.newRecord(MembershipStats.class);
+ record.init();
+ return record;
+ }
+
+ public abstract void setTotalSpace(long space);
+
+ public abstract long getTotalSpace();
+
+ public abstract void setAvailableSpace(long space);
+
+ public abstract long getAvailableSpace();
+
+ public abstract void setProvidedSpace(long capacity);
+
+ public abstract long getProvidedSpace();
+
+ public abstract void setNumOfFiles(long files);
+
+ public abstract long getNumOfFiles();
+
+ public abstract void setNumOfBlocks(long blocks);
+
+ public abstract long getNumOfBlocks();
+
+ public abstract void setNumOfBlocksMissing(long blocks);
+
+ public abstract long getNumOfBlocksMissing();
+
+ public abstract void setNumOfBlocksPendingReplication(long blocks);
+
+ public abstract long getNumOfBlocksPendingReplication();
+
+ public abstract void setNumOfBlocksUnderReplicated(long blocks);
+
+ public abstract long getNumOfBlocksUnderReplicated();
+
+ public abstract void setNumOfBlocksPendingDeletion(long blocks);
+
+ public abstract long getNumOfBlocksPendingDeletion();
+
+ public abstract void setNumOfActiveDatanodes(int nodes);
+
+ public abstract int getNumOfActiveDatanodes();
+
+ public abstract void setNumOfDeadDatanodes(int nodes);
+
+ public abstract int getNumOfDeadDatanodes();
+
+ public abstract void setNumOfDecommissioningDatanodes(int nodes);
+
+ public abstract int getNumOfDecommissioningDatanodes();
+
+ public abstract void setNumOfDecomActiveDatanodes(int nodes);
+
+ public abstract int getNumOfDecomActiveDatanodes();
+
+ public abstract void setNumOfDecomDeadDatanodes(int nodes);
+
+ public abstract int getNumOfDecomDeadDatanodes();
+
+ @Override
+ public SortedMap<String, String> getPrimaryKeys() {
+ // This record is not stored directly, no key needed
+ SortedMap<String, String> map = new TreeMap<String, String>();
+ return map;
+ }
+
+ @Override
+ public long getExpirationMs() {
+ // This record is not stored directly, no expiration needed
+ return -1;
+ }
+
+ @Override
+ public void setDateModified(long time) {
+ // We don't store this record directly
+ }
+
+ @Override
+ public long getDateModified() {
+ // We don't store this record directly
+ return 0;
+ }
+
+ @Override
+ public void setDateCreated(long time) {
+ // We don't store this record directly
+ }
+
+ @Override
+ public long getDateCreated() {
+ // We don't store this record directly
+ return 0;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
new file mode 100644
index 0000000..0eab043
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
@@ -0,0 +1,430 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
+import org.apache.hadoop.hdfs.server.federation.router.RouterPermissionChecker;
+import org.apache.hadoop.hdfs.server.federation.router.RouterQuotaUsage;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Data schema for
+ * {@link org.apache.hadoop.hdfs.server.federation.store.
+ * MountTableStore FederationMountTableStore} data stored in the
+ * {@link org.apache.hadoop.hdfs.server.federation.store.
+ * StateStoreService FederationStateStoreService}. Supports string
+ * serialization.
+ */
+public abstract class MountTable extends BaseRecord {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MountTable.class);
+ public static final String ERROR_MSG_NO_SOURCE_PATH =
+ "Invalid entry, no source path specified ";
+ public static final String ERROR_MSG_MUST_START_WITH_BACK_SLASH =
+ "Invalid entry, all mount points must start with / ";
+ public static final String ERROR_MSG_NO_DEST_PATH_SPECIFIED =
+ "Invalid entry, no destination paths specified ";
+ public static final String ERROR_MSG_INVAILD_DEST_NS =
+ "Invalid entry, invalid destination nameservice ";
+ public static final String ERROR_MSG_INVAILD_DEST_PATH =
+ "Invalid entry, invalid destination path ";
+ public static final String ERROR_MSG_ALL_DEST_MUST_START_WITH_BACK_SLASH =
+ "Invalid entry, all destination must start with / ";
+
+ /** Comparator for paths which considers the /. */
+ public static final Comparator<String> PATH_COMPARATOR =
+ new Comparator<String>() {
+ @Override
+ public int compare(String o1, String o2) {
+ String s1 = o1.replace('/', ' ');
+ String s2 = o2.replace('/', ' ');
+ return s1.compareTo(s2);
+ }
+ };
+
+ /** Comparator based on the mount table source. */
+ public static final Comparator<MountTable> SOURCE_COMPARATOR =
+ new Comparator<MountTable>() {
+ public int compare(MountTable m1, MountTable m2) {
+ String src1 = m1.getSourcePath();
+ String src2 = m2.getSourcePath();
+ return PATH_COMPARATOR.compare(src1, src2);
+ }
+ };
+
+
+ /**
+ * Default constructor for a mount table entry.
+ */
+ public MountTable() {
+ super();
+ }
+
+ public static MountTable newInstance() {
+ MountTable record = StateStoreSerializer.newRecord(MountTable.class);
+ record.init();
+ return record;
+ }
+
+ /**
+ * Constructor for a mount table entry with a single destinations.
+ *
+ * @param src Source path in the mount entry.
+ * @param destinations Nameservice destination of the mount point.
+ * @param dateCreated Created date.
+ * @param dateModified Modified date.
+ * @throws IOException
+ */
+ public static MountTable newInstance(final String src,
+ final Map<String, String> destinations,
+ long dateCreated, long dateModified) throws IOException {
+
+ MountTable record = newInstance(src, destinations);
+ record.setDateCreated(dateCreated);
+ record.setDateModified(dateModified);
+ return record;
+ }
+
+ /**
+ * Constructor for a mount table entry with multiple destinations.
+ *
+ * @param src Source path in the mount entry.
+ * @param destinations Nameservice destinations of the mount point.
+ * @throws IOException
+ */
+ public static MountTable newInstance(final String src,
+ final Map<String, String> destinations) throws IOException {
+ MountTable record = newInstance();
+
+ // Normalize the mount path
+ record.setSourcePath(normalizeFileSystemPath(src));
+
+ // Build a list of remote locations
+ final List<RemoteLocation> locations = new LinkedList<>();
+ for (Entry<String, String> entry : destinations.entrySet()) {
+ String nsId = entry.getKey();
+ String path = normalizeFileSystemPath(entry.getValue());
+ RemoteLocation location = new RemoteLocation(nsId, path);
+ locations.add(location);
+ }
+
+ // Set the serialized dest string
+ record.setDestinations(locations);
+
+ // Set permission fields
+ UserGroupInformation ugi = NameNode.getRemoteUser();
+ record.setOwnerName(ugi.getShortUserName());
+ String group = ugi.getGroups().isEmpty() ? ugi.getShortUserName()
+ : ugi.getPrimaryGroupName();
+ record.setGroupName(group);
+ record.setMode(new FsPermission(
+ RouterPermissionChecker.MOUNT_TABLE_PERMISSION_DEFAULT));
+
+ // Set quota for mount table
+ RouterQuotaUsage quota = new RouterQuotaUsage.Builder()
+ .fileAndDirectoryCount(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT)
+ .quota(HdfsConstants.QUOTA_DONT_SET)
+ .spaceConsumed(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT)
+ .spaceQuota(HdfsConstants.QUOTA_DONT_SET).build();
+ record.setQuota(quota);
+
+ // Validate
+ record.validate();
+ return record;
+ }
+
+ /**
+ * Get source path in the federated namespace.
+ *
+ * @return Source path in the federated namespace.
+ */
+ public abstract String getSourcePath();
+
+ /**
+ * Set source path in the federated namespace.
+ *
+ * @param path Source path in the federated namespace.
+ */
+ public abstract void setSourcePath(String path);
+
+ /**
+ * Get a list of destinations (namespace + path) present for this entry.
+ *
+ * @return List of RemoteLocation destinations. Null if no destinations.
+ */
+ public abstract List<RemoteLocation> getDestinations();
+
+ /**
+ * Set the destination paths.
+ *
+ * @param paths Destination paths.
+ */
+ public abstract void setDestinations(List<RemoteLocation> dests);
+
+ /**
+ * Add a new destination to this mount table entry.
+ */
+ public abstract boolean addDestination(String nsId, String path);
+
+ /**
+ * Check if the entry is read only.
+ *
+ * @return If the entry is read only.
+ */
+ public abstract boolean isReadOnly();
+
+ /**
+ * Set an entry to be read only.
+ *
+ * @param ro If the entry is read only.
+ */
+ public abstract void setReadOnly(boolean ro);
+
+ /**
+ * Get the order of the destinations for this mount table entry.
+ *
+ * @return Order of the destinations.
+ */
+ public abstract DestinationOrder getDestOrder();
+
+ /**
+ * Set the order of the destinations for this mount table entry.
+ *
+ * @param order Order of the destinations.
+ */
+ public abstract void setDestOrder(DestinationOrder order);
+
+ /**
+ * Get owner name of this mount table entry.
+ *
+ * @return Owner name
+ */
+ public abstract String getOwnerName();
+
+ /**
+ * Set owner name of this mount table entry.
+ *
+ * @param owner Owner name for mount table entry
+ */
+ public abstract void setOwnerName(String owner);
+
+ /**
+ * Get group name of this mount table entry.
+ *
+ * @return Group name
+ */
+ public abstract String getGroupName();
+
+ /**
+ * Set group name of this mount table entry.
+ *
+ * @param group Group name for mount table entry
+ */
+ public abstract void setGroupName(String group);
+
+ /**
+ * Get permission of this mount table entry.
+ *
+ * @return FsPermission permission mode
+ */
+ public abstract FsPermission getMode();
+
+ /**
+ * Set permission for this mount table entry.
+ *
+ * @param mode Permission for mount table entry
+ */
+ public abstract void setMode(FsPermission mode);
+
+ /**
+ * Get quota of this mount table entry.
+ *
+ * @return RouterQuotaUsage quota usage
+ */
+ public abstract RouterQuotaUsage getQuota();
+
+ /**
+ * Set quota for this mount table entry.
+ *
+ * @param quota QuotaUsage for mount table entry
+ */
+ public abstract void setQuota(RouterQuotaUsage quota);
+
+ /**
+ * Get the default location.
+ * @return The default location.
+ */
+ public RemoteLocation getDefaultLocation() {
+ List<RemoteLocation> dests = this.getDestinations();
+ if (dests == null || dests.isEmpty()) {
+ return null;
+ }
+ return dests.get(0);
+ }
+
+ @Override
+ public boolean like(final BaseRecord o) {
+ if (o instanceof MountTable) {
+ MountTable other = (MountTable)o;
+ if (getSourcePath() != null &&
+ !getSourcePath().equals(other.getSourcePath())) {
+ return false;
+ }
+ if (getDestinations() != null &&
+ !getDestinations().equals(other.getDestinations())) {
+ return false;
+ }
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(this.getSourcePath());
+ sb.append("->");
+ List<RemoteLocation> destinations = this.getDestinations();
+ sb.append(destinations);
+ if (destinations != null && destinations.size() > 1) {
+ sb.append("[" + this.getDestOrder() + "]");
+ }
+ if (this.isReadOnly()) {
+ sb.append("[RO]");
+ }
+
+ if (this.getOwnerName() != null) {
+ sb.append("[owner:").append(this.getOwnerName()).append("]");
+ }
+
+ if (this.getGroupName() != null) {
+ sb.append("[group:").append(this.getGroupName()).append("]");
+ }
+
+ if (this.getMode() != null) {
+ sb.append("[mode:").append(this.getMode()).append("]");
+ }
+
+ if (this.getQuota() != null) {
+ sb.append("[quota:").append(this.getQuota()).append("]");
+ }
+
+ return sb.toString();
+ }
+
+ @Override
+ public SortedMap<String, String> getPrimaryKeys() {
+ SortedMap<String, String> map = new TreeMap<>();
+ map.put("sourcePath", this.getSourcePath());
+ return map;
+ }
+
+ @Override
+ public void validate() {
+ super.validate();
+ if (this.getSourcePath() == null || this.getSourcePath().length() == 0) {
+ throw new IllegalArgumentException(
+ ERROR_MSG_NO_SOURCE_PATH + this);
+ }
+ if (!this.getSourcePath().startsWith("/")) {
+ throw new IllegalArgumentException(
+ ERROR_MSG_MUST_START_WITH_BACK_SLASH + this);
+ }
+ if (this.getDestinations() == null || this.getDestinations().size() == 0) {
+ throw new IllegalArgumentException(
+ ERROR_MSG_NO_DEST_PATH_SPECIFIED + this);
+ }
+ for (RemoteLocation loc : getDestinations()) {
+ String nsId = loc.getNameserviceId();
+ if (nsId == null || nsId.length() == 0) {
+ throw new IllegalArgumentException(
+ ERROR_MSG_INVAILD_DEST_NS + this);
+ }
+ if (loc.getDest() == null || loc.getDest().length() == 0) {
+ throw new IllegalArgumentException(
+ ERROR_MSG_INVAILD_DEST_PATH + this);
+ }
+ if (!loc.getDest().startsWith("/")) {
+ throw new IllegalArgumentException(
+ ERROR_MSG_ALL_DEST_MUST_START_WITH_BACK_SLASH + this);
+ }
+ }
+ }
+
+ @Override
+ public long getExpirationMs() {
+ return 0;
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder(17, 31)
+ .append(this.getSourcePath())
+ .append(this.getDestinations())
+ .append(this.isReadOnly())
+ .append(this.getDestOrder())
+ .toHashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof MountTable) {
+ MountTable other = (MountTable)obj;
+ if (!this.getSourcePath().equals(other.getSourcePath())) {
+ return false;
+ } else if (!this.getDestinations().equals(other.getDestinations())) {
+ return false;
+ } else if (this.isReadOnly() != other.isReadOnly()) {
+ return false;
+ } else if (!this.getDestOrder().equals(other.getDestOrder())) {
+ return false;
+ }
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Normalize a path for that filesystem.
+ *
+ * @param path Path to normalize.
+ * @return Normalized path.
+ */
+ private static String normalizeFileSystemPath(final String path) {
+ Path normalizedPath = new Path(path);
+ return normalizedPath.toString();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/Query.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/Query.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/Query.java
new file mode 100644
index 0000000..3c59abf
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/Query.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+/**
+ * Check if a record matches a query. The query is usually a partial record.
+ *
+ * @param <T> Type of the record to query.
+ */
+public class Query<T extends BaseRecord> {
+
+ /** Partial object to compare against. */
+ private final T partial;
+
+
+ /**
+ * Create a query to search for a partial record.
+ *
+ * @param partial It defines the attributes to search.
+ */
+ public Query(final T part) {
+ this.partial = part;
+ }
+
+ /**
+ * Get the partial record used to query.
+ *
+ * @return The partial record used for the query.
+ */
+ public T getPartial() {
+ return this.partial;
+ }
+
+ /**
+ * Check if a record matches the primary keys or the partial record.
+ *
+ * @param other Record to check.
+ * @return If the record matches. Don't match if there is no partial.
+ */
+ public boolean matches(T other) {
+ if (this.partial == null) {
+ return false;
+ }
+ return this.partial.like(other);
+ }
+
+ @Override
+ public String toString() {
+ return "Checking: " + this.partial;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/QueryResult.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/QueryResult.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/QueryResult.java
new file mode 100644
index 0000000..64c2c71
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/QueryResult.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Encapsulates a state store query result that includes a set of records and a
+ * time stamp for the result.
+ */
+public class QueryResult<T extends BaseRecord> {
+
+ /** Data result. */
+ private final List<T> records;
+ /** Time stamp of the data results. */
+ private final long timestamp;
+
+ public QueryResult(final List<T> recs, final long time) {
+ this.records = recs;
+ this.timestamp = time;
+ }
+
+ /**
+ * Get the result of the query.
+ *
+ * @return List of records.
+ */
+ public List<T> getRecords() {
+ return Collections.unmodifiableList(this.records);
+ }
+
+ /**
+ * The timetamp in driver time of this query.
+ *
+ * @return Timestamp in driver time.
+ */
+ public long getTimestamp() {
+ return this.timestamp;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/RouterState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/RouterState.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/RouterState.java
new file mode 100644
index 0000000..c90abcc
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/RouterState.java
@@ -0,0 +1,168 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+import java.io.IOException;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.hadoop.hdfs.server.federation.router.FederationUtil;
+import org.apache.hadoop.hdfs.server.federation.router.RouterServiceState;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Entry to log the state of a
+ * {@link org.apache.hadoop.hdfs.server.federation.router.Router Router} in the
+ * {@link org.apache.hadoop.hdfs.server.federation.store.StateStoreService
+ * FederationStateStoreService}.
+ */
+public abstract class RouterState extends BaseRecord {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RouterState.class);
+
+ /** Expiration time in ms for this entry. */
+ private static long expirationMs;
+
+ /**
+ * Constructors.
+ */
+ public RouterState() {
+ super();
+ }
+
+ public static RouterState newInstance() {
+ RouterState record = StateStoreSerializer.newRecord(RouterState.class);
+ record.init();
+ return record;
+ }
+
+ public static RouterState newInstance(String addr, long startTime,
+ RouterServiceState status) {
+ RouterState record = newInstance();
+ record.setDateStarted(startTime);
+ record.setAddress(addr);
+ record.setStatus(status);
+ record.setCompileInfo(FederationUtil.getCompileInfo());
+ record.setVersion(FederationUtil.getVersion());
+ return record;
+ }
+
+ public abstract void setAddress(String address);
+
+ public abstract void setDateStarted(long dateStarted);
+
+ public abstract String getAddress();
+
+ public abstract StateStoreVersion getStateStoreVersion() throws IOException;
+
+ public abstract void setStateStoreVersion(StateStoreVersion version);
+
+ public abstract RouterServiceState getStatus();
+
+ public abstract void setStatus(RouterServiceState newStatus);
+
+ public abstract String getVersion();
+
+ public abstract void setVersion(String version);
+
+ public abstract String getCompileInfo();
+
+ public abstract void setCompileInfo(String info);
+
+ public abstract long getDateStarted();
+
+ /**
+ * Get the identifier for the Router. It uses the address.
+ *
+ * @return Identifier for the Router.
+ */
+ public String getRouterId() {
+ return getAddress();
+ }
+
+ @Override
+ public boolean like(BaseRecord o) {
+ if (o instanceof RouterState) {
+ RouterState other = (RouterState)o;
+ if (getAddress() != null &&
+ !getAddress().equals(other.getAddress())) {
+ return false;
+ }
+ if (getStatus() != null &&
+ !getStatus().equals(other.getStatus())) {
+ return false;
+ }
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return getAddress() + " -> " + getStatus() + "," + getVersion();
+ }
+
+ @Override
+ public SortedMap<String, String> getPrimaryKeys() {
+ SortedMap<String, String> map = new TreeMap<>();
+ map.put("address", getAddress());
+ return map;
+ }
+
+ @Override
+ public void validate() {
+ super.validate();
+ if ((getAddress() == null || getAddress().length() == 0) &&
+ getStatus() != RouterServiceState.INITIALIZING) {
+ throw new IllegalArgumentException(
+ "Invalid router entry, no address specified " + this);
+ }
+ }
+
+ @Override
+ public int compareTo(BaseRecord other) {
+ if (other == null) {
+ return -1;
+ } else if (other instanceof RouterState) {
+ RouterState router = (RouterState) other;
+ return this.getAddress().compareTo(router.getAddress());
+ } else {
+ return super.compareTo(other);
+ }
+ }
+
+ @Override
+ public boolean checkExpired(long currentTime) {
+ if (super.checkExpired(currentTime)) {
+ setStatus(RouterServiceState.EXPIRED);
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public long getExpirationMs() {
+ return RouterState.expirationMs;
+ }
+
+ public static void setExpirationMs(long time) {
+ RouterState.expirationMs = time;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/StateStoreVersion.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/StateStoreVersion.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/StateStoreVersion.java
new file mode 100644
index 0000000..ce86eb0
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/StateStoreVersion.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * Entry to track the version of the State Store data stored in the State Store
+ * by a Router.
+ */
+public abstract class StateStoreVersion extends BaseRecord {
+
+ public static StateStoreVersion newInstance() {
+ return StateStoreSerializer.newRecord(StateStoreVersion.class);
+ }
+
+ public static StateStoreVersion newInstance(long membershipVersion,
+ long mountTableVersion) {
+ StateStoreVersion record = newInstance();
+ record.setMembershipVersion(membershipVersion);
+ record.setMountTableVersion(mountTableVersion);
+ return record;
+ }
+
+ public abstract long getMembershipVersion();
+
+ public abstract void setMembershipVersion(long version);
+
+ public abstract long getMountTableVersion();
+
+ public abstract void setMountTableVersion(long version);
+
+ @Override
+ public SortedMap<String, String> getPrimaryKeys() {
+ // This record is not stored directly, no key needed
+ SortedMap<String, String> map = new TreeMap<String, String>();
+ return map;
+ }
+
+ @Override
+ public long getExpirationMs() {
+ // This record is not stored directly, no expiration needed
+ return -1;
+ }
+
+ @Override
+ public void setDateModified(long time) {
+ // We don't store this record directly
+ }
+
+ @Override
+ public long getDateModified() {
+ // We don't store this record directly
+ return 0;
+ }
+
+ @Override
+ public void setDateCreated(long time) {
+ // We don't store this record directly
+ }
+
+ @Override
+ public long getDateCreated() {
+ // We don't store this record directly
+ return 0;
+ }
+
+ @Override
+ public String toString() {
+ return "Membership: " + getMembershipVersion() +
+ " Mount Table: " + getMountTableVersion();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatePBImpl.java
new file mode 100644
index 0000000..614957b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatePBImpl.java
@@ -0,0 +1,335 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipRecordProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipRecordProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipRecordProtoOrBuilder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipStatsRecordProto;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.FederationProtocolPBTranslator;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipStats;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the MembershipState record.
+ */
+public class MembershipStatePBImpl extends MembershipState implements PBRecord {
+
+ private FederationProtocolPBTranslator<NamenodeMembershipRecordProto, Builder,
+ NamenodeMembershipRecordProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<NamenodeMembershipRecordProto,
+ Builder, NamenodeMembershipRecordProtoOrBuilder>(
+ NamenodeMembershipRecordProto.class);
+
+ public MembershipStatePBImpl() {
+ }
+
+ public MembershipStatePBImpl(NamenodeMembershipRecordProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public NamenodeMembershipRecordProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public void setRouterId(String routerId) {
+ Builder builder = this.translator.getBuilder();
+ if (routerId == null) {
+ builder.clearRouterId();
+ } else {
+ builder.setRouterId(routerId);
+ }
+ }
+
+ @Override
+ public void setNameserviceId(String nameserviceId) {
+ Builder builder = this.translator.getBuilder();
+ if (nameserviceId == null) {
+ builder.clearNameserviceId();
+ } else {
+ builder.setNameserviceId(nameserviceId);
+ }
+ }
+
+ @Override
+ public void setNamenodeId(String namenodeId) {
+ Builder builder = this.translator.getBuilder();
+ if (namenodeId == null) {
+ builder.clearNamenodeId();
+ } else {
+ builder.setNamenodeId(namenodeId);
+ }
+ }
+
+ @Override
+ public void setWebAddress(String webAddress) {
+ Builder builder = this.translator.getBuilder();
+ if (webAddress == null) {
+ builder.clearWebAddress();
+ } else {
+ builder.setWebAddress(webAddress);
+ }
+ }
+
+ @Override
+ public void setRpcAddress(String rpcAddress) {
+ Builder builder = this.translator.getBuilder();
+ if (rpcAddress == null) {
+ builder.clearRpcAddress();
+ } else {
+ builder.setRpcAddress(rpcAddress);
+ }
+ }
+
+ @Override
+ public void setServiceAddress(String serviceAddress) {
+ this.translator.getBuilder().setServiceAddress(serviceAddress);
+ }
+
+ @Override
+ public void setLifelineAddress(String lifelineAddress) {
+ Builder builder = this.translator.getBuilder();
+ if (lifelineAddress == null) {
+ builder.clearLifelineAddress();
+ } else {
+ builder.setLifelineAddress(lifelineAddress);
+ }
+ }
+
+ @Override
+ public void setIsSafeMode(boolean isSafeMode) {
+ Builder builder = this.translator.getBuilder();
+ builder.setIsSafeMode(isSafeMode);
+ }
+
+ @Override
+ public void setClusterId(String clusterId) {
+ Builder builder = this.translator.getBuilder();
+ if (clusterId == null) {
+ builder.clearClusterId();
+ } else {
+ builder.setClusterId(clusterId);
+ }
+ }
+
+ @Override
+ public void setBlockPoolId(String blockPoolId) {
+ Builder builder = this.translator.getBuilder();
+ if (blockPoolId == null) {
+ builder.clearBlockPoolId();
+ } else {
+ builder.setBlockPoolId(blockPoolId);
+ }
+ }
+
+ @Override
+ public void setState(FederationNamenodeServiceState state) {
+ Builder builder = this.translator.getBuilder();
+ if (state == null) {
+ builder.clearState();
+ } else {
+ builder.setState(state.toString());
+ }
+ }
+
+ @Override
+ public String getRouterId() {
+ NamenodeMembershipRecordProtoOrBuilder proto =
+ this.translator.getProtoOrBuilder();
+ if (!proto.hasRouterId()) {
+ return null;
+ }
+ return proto.getRouterId();
+ }
+
+ @Override
+ public String getNameserviceId() {
+ NamenodeMembershipRecordProtoOrBuilder proto =
+ this.translator.getProtoOrBuilder();
+ if (!proto.hasNameserviceId()) {
+ return null;
+ }
+ return this.translator.getProtoOrBuilder().getNameserviceId();
+ }
+
+ @Override
+ public String getNamenodeId() {
+ NamenodeMembershipRecordProtoOrBuilder proto =
+ this.translator.getProtoOrBuilder();
+ if (!proto.hasNamenodeId()) {
+ return null;
+ }
+ return this.translator.getProtoOrBuilder().getNamenodeId();
+ }
+
+ @Override
+ public String getClusterId() {
+ NamenodeMembershipRecordProtoOrBuilder proto =
+ this.translator.getProtoOrBuilder();
+ if (!proto.hasClusterId()) {
+ return null;
+ }
+ return this.translator.getProtoOrBuilder().getClusterId();
+ }
+
+ @Override
+ public String getBlockPoolId() {
+ NamenodeMembershipRecordProtoOrBuilder proto =
+ this.translator.getProtoOrBuilder();
+ if (!proto.hasBlockPoolId()) {
+ return null;
+ }
+ return this.translator.getProtoOrBuilder().getBlockPoolId();
+ }
+
+ @Override
+ public String getRpcAddress() {
+ NamenodeMembershipRecordProtoOrBuilder proto =
+ this.translator.getProtoOrBuilder();
+ if (!proto.hasRpcAddress()) {
+ return null;
+ }
+ return this.translator.getProtoOrBuilder().getRpcAddress();
+ }
+
+ @Override
+ public String getServiceAddress() {
+ NamenodeMembershipRecordProtoOrBuilder proto =
+ this.translator.getProtoOrBuilder();
+ if (!proto.hasServiceAddress()) {
+ return null;
+ }
+ return this.translator.getProtoOrBuilder().getServiceAddress();
+ }
+
+ @Override
+ public String getWebAddress() {
+ NamenodeMembershipRecordProtoOrBuilder proto =
+ this.translator.getProtoOrBuilder();
+ if (!proto.hasWebAddress()) {
+ return null;
+ }
+ return this.translator.getProtoOrBuilder().getWebAddress();
+ }
+
+ @Override
+ public String getLifelineAddress() {
+ NamenodeMembershipRecordProtoOrBuilder proto =
+ this.translator.getProtoOrBuilder();
+ if (!proto.hasLifelineAddress()) {
+ return null;
+ }
+ return this.translator.getProtoOrBuilder().getLifelineAddress();
+ }
+
+ @Override
+ public boolean getIsSafeMode() {
+ return this.translator.getProtoOrBuilder().getIsSafeMode();
+ }
+
+ @Override
+ public FederationNamenodeServiceState getState() {
+ FederationNamenodeServiceState ret =
+ FederationNamenodeServiceState.UNAVAILABLE;
+ NamenodeMembershipRecordProtoOrBuilder proto =
+ this.translator.getProtoOrBuilder();
+ if (!proto.hasState()) {
+ return null;
+ }
+ try {
+ ret = FederationNamenodeServiceState.valueOf(proto.getState());
+ } catch (IllegalArgumentException e) {
+ // Ignore this error
+ }
+ return ret;
+ }
+
+ @Override
+ public void setStats(MembershipStats stats) {
+ if (stats instanceof MembershipStatsPBImpl) {
+ MembershipStatsPBImpl statsPB = (MembershipStatsPBImpl)stats;
+ NamenodeMembershipStatsRecordProto statsProto =
+ (NamenodeMembershipStatsRecordProto)statsPB.getProto();
+ this.translator.getBuilder().setStats(statsProto);
+ }
+ }
+
+ @Override
+ public MembershipStats getStats() {
+ NamenodeMembershipStatsRecordProto statsProto =
+ this.translator.getProtoOrBuilder().getStats();
+ MembershipStats stats =
+ StateStoreSerializer.newRecord(MembershipStats.class);
+ if (stats instanceof MembershipStatsPBImpl) {
+ MembershipStatsPBImpl statsPB = (MembershipStatsPBImpl)stats;
+ statsPB.setProto(statsProto);
+ return statsPB;
+ } else {
+ throw new IllegalArgumentException(
+ "Cannot get stats for the membership");
+ }
+ }
+
+ @Override
+ public void setLastContact(long contact) {
+ this.translator.getBuilder().setLastContact(contact);
+ }
+
+ @Override
+ public long getLastContact() {
+ return this.translator.getProtoOrBuilder().getLastContact();
+ }
+
+ @Override
+ public void setDateModified(long time) {
+ this.translator.getBuilder().setDateModified(time);
+ }
+
+ @Override
+ public long getDateModified() {
+ return this.translator.getProtoOrBuilder().getDateModified();
+ }
+
+ @Override
+ public void setDateCreated(long time) {
+ this.translator.getBuilder().setDateCreated(time);
+ }
+
+ @Override
+ public long getDateCreated() {
+ return this.translator.getProtoOrBuilder().getDateCreated();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatsPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatsPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatsPBImpl.java
new file mode 100644
index 0000000..3347bc6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatsPBImpl.java
@@ -0,0 +1,201 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipStatsRecordProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipStatsRecordProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipStatsRecordProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.FederationProtocolPBTranslator;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipStats;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the MembershipStats record.
+ */
+public class MembershipStatsPBImpl extends MembershipStats
+ implements PBRecord {
+
+ private FederationProtocolPBTranslator<NamenodeMembershipStatsRecordProto,
+ Builder, NamenodeMembershipStatsRecordProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<NamenodeMembershipStatsRecordProto,
+ Builder, NamenodeMembershipStatsRecordProtoOrBuilder>(
+ NamenodeMembershipStatsRecordProto.class);
+
+ public MembershipStatsPBImpl() {
+ }
+
+ @Override
+ public NamenodeMembershipStatsRecordProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public void setTotalSpace(long space) {
+ this.translator.getBuilder().setTotalSpace(space);
+ }
+
+ @Override
+ public long getTotalSpace() {
+ return this.translator.getProtoOrBuilder().getTotalSpace();
+ }
+
+ @Override
+ public void setAvailableSpace(long space) {
+ this.translator.getBuilder().setAvailableSpace(space);
+ }
+
+ @Override
+ public long getAvailableSpace() {
+ return this.translator.getProtoOrBuilder().getAvailableSpace();
+ }
+
+ @Override
+ public void setProvidedSpace(long capacity) {
+ this.translator.getBuilder().setProvidedSpace(capacity);
+ }
+
+ @Override
+ public long getProvidedSpace() {
+ return this.translator.getProtoOrBuilder().getProvidedSpace();
+ }
+
+ @Override
+ public void setNumOfFiles(long files) {
+ this.translator.getBuilder().setNumOfFiles(files);
+ }
+
+ @Override
+ public long getNumOfFiles() {
+ return this.translator.getProtoOrBuilder().getNumOfFiles();
+ }
+
+ @Override
+ public void setNumOfBlocks(long blocks) {
+ this.translator.getBuilder().setNumOfBlocks(blocks);
+ }
+
+ @Override
+ public long getNumOfBlocks() {
+ return this.translator.getProtoOrBuilder().getNumOfBlocks();
+ }
+
+ @Override
+ public void setNumOfBlocksMissing(long blocks) {
+ this.translator.getBuilder().setNumOfBlocksMissing(blocks);
+ }
+
+ @Override
+ public long getNumOfBlocksMissing() {
+ return this.translator.getProtoOrBuilder().getNumOfBlocksMissing();
+ }
+
+ @Override
+ public void setNumOfBlocksPendingReplication(long blocks) {
+ this.translator.getBuilder().setNumOfBlocksPendingReplication(blocks);
+ }
+
+ @Override
+ public long getNumOfBlocksPendingReplication() {
+ return this.translator.getProtoOrBuilder()
+ .getNumOfBlocksPendingReplication();
+ }
+
+ @Override
+ public void setNumOfBlocksUnderReplicated(long blocks) {
+ this.translator.getBuilder().setNumOfBlocksUnderReplicated(blocks);
+ }
+
+ @Override
+ public long getNumOfBlocksUnderReplicated() {
+ return this.translator.getProtoOrBuilder().getNumOfBlocksUnderReplicated();
+ }
+
+ @Override
+ public void setNumOfBlocksPendingDeletion(long blocks) {
+ this.translator.getBuilder().setNumOfBlocksPendingDeletion(blocks);
+ }
+
+ @Override
+ public long getNumOfBlocksPendingDeletion() {
+ return this.translator.getProtoOrBuilder().getNumOfBlocksPendingDeletion();
+ }
+
+ @Override
+ public void setNumOfActiveDatanodes(int nodes) {
+ this.translator.getBuilder().setNumOfActiveDatanodes(nodes);
+ }
+
+ @Override
+ public int getNumOfActiveDatanodes() {
+ return this.translator.getProtoOrBuilder().getNumOfActiveDatanodes();
+ }
+
+ @Override
+ public void setNumOfDeadDatanodes(int nodes) {
+ this.translator.getBuilder().setNumOfDeadDatanodes(nodes);
+ }
+
+ @Override
+ public int getNumOfDeadDatanodes() {
+ return this.translator.getProtoOrBuilder().getNumOfDeadDatanodes();
+ }
+
+ @Override
+ public void setNumOfDecommissioningDatanodes(int nodes) {
+ this.translator.getBuilder().setNumOfDecommissioningDatanodes(nodes);
+ }
+
+ @Override
+ public int getNumOfDecommissioningDatanodes() {
+ return this.translator.getProtoOrBuilder()
+ .getNumOfDecommissioningDatanodes();
+ }
+
+ @Override
+ public void setNumOfDecomActiveDatanodes(int nodes) {
+ this.translator.getBuilder().setNumOfDecomActiveDatanodes(nodes);
+ }
+
+ @Override
+ public int getNumOfDecomActiveDatanodes() {
+ return this.translator.getProtoOrBuilder().getNumOfDecomActiveDatanodes();
+ }
+
+ @Override
+ public void setNumOfDecomDeadDatanodes(int nodes) {
+ this.translator.getBuilder().setNumOfDecomDeadDatanodes(nodes);
+ }
+
+ @Override
+ public int getNumOfDecomDeadDatanodes() {
+ return this.translator.getProtoOrBuilder().getNumOfDecomDeadDatanodes();
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/LocalResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/LocalResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/LocalResolver.java
new file mode 100644
index 0000000..4d76c89
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/LocalResolver.java
@@ -0,0 +1,297 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver.order;
+
+import static org.apache.hadoop.util.Time.monotonicNow;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.net.HostAndPort;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.security.PrivilegedAction;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
+import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.router.Router;
+import org.apache.hadoop.hdfs.server.federation.router.RouterRpcServer;
+import org.apache.hadoop.hdfs.server.federation.store.MembershipStore;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
+import org.apache.hadoop.ipc.RPC.Server;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * The local subcluster (where the writer is) should be tried first. The writer
+ * is defined from the RPC query received in the RPC server.
+ */
+public class LocalResolver implements OrderedResolver {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(LocalResolver.class);
+
+ /** Configuration key to set the minimum time to update the local cache.*/
+ public static final String MIN_UPDATE_PERIOD_KEY =
+ RBFConfigKeys.FEDERATION_ROUTER_PREFIX + "local-resolver.update-period";
+ /** 10 seconds by default. */
+ private static final long MIN_UPDATE_PERIOD_DEFAULT =
+ TimeUnit.SECONDS.toMillis(10);
+
+
+ /** Router service. */
+ private final Router router;
+ /** Minimum update time. */
+ private final long minUpdateTime;
+
+ /** Node IP -> Subcluster. */
+ private Map<String, String> nodeSubcluster = null;
+ /** Last time the subcluster map was updated. */
+ private long lastUpdated;
+
+
+ public LocalResolver(final Configuration conf, final Router routerService) {
+ this.minUpdateTime = conf.getTimeDuration(
+ MIN_UPDATE_PERIOD_KEY, MIN_UPDATE_PERIOD_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ this.router = routerService;
+ }
+
+ /**
+ * Get the local name space. This relies on the RPC Server to get the address
+ * from the client.
+ *
+ * TODO we only support DN and NN locations, we need to add others like
+ * Resource Managers.
+ *
+ * @param path Path ignored by this policy.
+ * @param loc Federated location with multiple destinations.
+ * @return Local name space. Null if we don't know about this machine.
+ */
+ @Override
+ public String getFirstNamespace(final String path, final PathLocation loc) {
+ String localSubcluster = null;
+ String clientAddr = getClientAddr();
+ Map<String, String> nodeToSubcluster = getSubclusterMappings();
+ if (nodeToSubcluster != null) {
+ localSubcluster = nodeToSubcluster.get(clientAddr);
+ if (localSubcluster != null) {
+ LOG.debug("Local namespace for {} is {}", clientAddr, localSubcluster);
+ } else {
+ LOG.error("Cannot get local namespace for {}", clientAddr);
+ }
+ } else {
+ LOG.error("Cannot get node mapping when resolving {} at {} from {}",
+ path, loc, clientAddr);
+ }
+ return localSubcluster;
+ }
+
+ @VisibleForTesting
+ String getClientAddr() {
+ return Server.getRemoteAddress();
+ }
+
+ /**
+ * Get the mapping from nodes to subcluster. It gets this mapping from the
+ * subclusters through expensive calls (e.g., RPC) and uses caching to avoid
+ * too many calls. The cache might be updated asynchronously to reduce
+ * latency.
+ *
+ * @return Node IP -> Subcluster.
+ */
+ @VisibleForTesting
+ synchronized Map<String, String> getSubclusterMappings() {
+ if (nodeSubcluster == null ||
+ (monotonicNow() - lastUpdated) > minUpdateTime) {
+ // Fetch the mapping asynchronously
+ Thread updater = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ Map<String, String> mapping = new HashMap<>();
+
+ Map<String, String> dnSubcluster = getDatanodesSubcluster();
+ if (dnSubcluster != null) {
+ mapping.putAll(dnSubcluster);
+ }
+
+ Map<String, String> nnSubcluster = getNamenodesSubcluster();
+ if (nnSubcluster != null) {
+ mapping.putAll(nnSubcluster);
+ }
+ nodeSubcluster = mapping;
+ lastUpdated = monotonicNow();
+ }
+ });
+ updater.start();
+
+ // Wait until initialized
+ if (nodeSubcluster == null) {
+ try {
+ LOG.debug("Wait to get the mapping for the first time");
+ updater.join();
+ } catch (InterruptedException e) {
+ LOG.error("Cannot wait for the updater to finish");
+ }
+ }
+ }
+ return nodeSubcluster;
+ }
+
+ /**
+ * Get the Datanode mapping from the subclusters from the Namenodes. This
+ * needs to be done as a privileged action to use the user for the Router and
+ * not the one from the client in the RPC call.
+ *
+ * @return DN IP -> Subcluster.
+ */
+ private Map<String, String> getDatanodesSubcluster() {
+
+ final RouterRpcServer rpcServer = getRpcServer();
+ if (rpcServer == null) {
+ LOG.error("Cannot access the Router RPC server");
+ return null;
+ }
+
+ Map<String, String> ret = new HashMap<>();
+ try {
+ // We need to get the DNs as a privileged user
+ UserGroupInformation loginUser = UserGroupInformation.getLoginUser();
+ Map<String, DatanodeStorageReport[]> dnMap = loginUser.doAs(
+ new PrivilegedAction<Map<String, DatanodeStorageReport[]>>() {
+ @Override
+ public Map<String, DatanodeStorageReport[]> run() {
+ try {
+ return rpcServer.getDatanodeStorageReportMap(
+ DatanodeReportType.ALL);
+ } catch (IOException e) {
+ LOG.error("Cannot get the datanodes from the RPC server", e);
+ return null;
+ }
+ }
+ });
+ for (Entry<String, DatanodeStorageReport[]> entry : dnMap.entrySet()) {
+ String nsId = entry.getKey();
+ DatanodeStorageReport[] dns = entry.getValue();
+ for (DatanodeStorageReport dn : dns) {
+ DatanodeInfo dnInfo = dn.getDatanodeInfo();
+ String ipAddr = dnInfo.getIpAddr();
+ ret.put(ipAddr, nsId);
+ }
+ }
+ } catch (IOException e) {
+ LOG.error("Cannot get Datanodes from the Namenodes: {}", e.getMessage());
+ }
+ return ret;
+ }
+
+ /**
+ * Get the Namenode mapping from the subclusters from the Membership store. As
+ * the Routers are usually co-located with Namenodes, we also check for the
+ * local address for this Router here.
+ *
+ * @return NN IP -> Subcluster.
+ */
+ private Map<String, String> getNamenodesSubcluster() {
+
+ final MembershipStore membershipStore = getMembershipStore();
+ if (membershipStore == null) {
+ LOG.error("Cannot access the Membership store");
+ return null;
+ }
+
+ // Manage requests from this hostname (127.0.0.1)
+ String localIp = "127.0.0.1";
+ String localHostname = localIp;
+ try {
+ localHostname = InetAddress.getLocalHost().getHostName();
+ } catch (UnknownHostException e) {
+ LOG.error("Cannot get local host name");
+ }
+
+ Map<String, String> ret = new HashMap<>();
+ try {
+ // Get the values from the store
+ GetNamenodeRegistrationsRequest request =
+ GetNamenodeRegistrationsRequest.newInstance();
+ GetNamenodeRegistrationsResponse response =
+ membershipStore.getNamenodeRegistrations(request);
+ final List<MembershipState> nns = response.getNamenodeMemberships();
+ for (MembershipState nn : nns) {
+ try {
+ String nsId = nn.getNameserviceId();
+ String rpcAddress = nn.getRpcAddress();
+ String hostname = HostAndPort.fromString(rpcAddress).getHostText();
+ ret.put(hostname, nsId);
+ if (hostname.equals(localHostname)) {
+ ret.put(localIp, nsId);
+ }
+
+ InetAddress addr = InetAddress.getByName(hostname);
+ String ipAddr = addr.getHostAddress();
+ ret.put(ipAddr, nsId);
+ } catch (Exception e) {
+ LOG.error("Cannot get address for {}: {}", nn, e.getMessage());
+ }
+ }
+ } catch (IOException ioe) {
+ LOG.error("Cannot get Namenodes from the State Store: {}",
+ ioe.getMessage());
+ }
+ return ret;
+ }
+
+ /**
+ * Get the Router RPC server.
+ *
+ * @return Router RPC server. Null if not possible.
+ */
+ private RouterRpcServer getRpcServer() {
+ if (this.router == null) {
+ return null;
+ }
+ return router.getRpcServer();
+ }
+
+ /**
+ * Get the Membership store.
+ *
+ * @return Membership store.
+ */
+ private MembershipStore getMembershipStore() {
+ StateStoreService stateStore = router.getStateStore();
+ if (stateStore == null) {
+ return null;
+ }
+ return stateStore.getRegisteredRecordStore(MembershipStore.class);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/OrderedResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/OrderedResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/OrderedResolver.java
new file mode 100644
index 0000000..3a3ccf7
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/OrderedResolver.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver.order;
+
+import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
+
+
+/**
+ * Policy that decides which should be the first location accessed given
+ * multiple destinations.
+ */
+public interface OrderedResolver {
+
+ /**
+ * Get the first namespace based on this resolver approach.
+ *
+ * @param path Path to check.
+ * @param loc Federated location with multiple destinations.
+ * @return First namespace out of the locations.
+ */
+ String getFirstNamespace(String path, PathLocation loc);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/RandomResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/RandomResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/RandomResolver.java
new file mode 100644
index 0000000..022aa48
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/RandomResolver.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver.order;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Order the destinations randomly.
+ */
+public class RandomResolver implements OrderedResolver {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RandomResolver.class);
+
+
+ /** Random number generator. */
+ private static final Random RANDOM = new Random();
+
+ /**
+ * Get a random name space from the path.
+ *
+ * @param path Path ignored by this policy.
+ * @param loc Federated location with multiple destinations.
+ * @return Random name space.
+ */
+ public String getFirstNamespace(final String path, final PathLocation loc) {
+ if (loc == null) {
+ return null;
+ }
+ Set<String> namespaces = loc.getNamespaces();
+ if (namespaces == null || namespaces.isEmpty()) {
+ LOG.error("Cannot get namespaces for {}", loc);
+ return null;
+ }
+ List<String> nssList = new ArrayList<>(namespaces);
+ int index = RANDOM.nextInt(nssList.size());
+ return nssList.get(index);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/package-info.java
new file mode 100644
index 0000000..f90152f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/package-info.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A federated location can be resolved to multiple subclusters. This package
+ * takes care of the order in which this multiple destinations should be used.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+
+package org.apache.hadoop.hdfs.server.federation.resolver.order;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/package-info.java
new file mode 100644
index 0000000..d8be9e3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/package-info.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The resolver package contains indepedent data resolvers used in HDFS
+ * federation. The data resolvers collect data from the cluster, including from
+ * the state store. The resolvers expose APIs used by HDFS federation to collect
+ * aggregated, cached data for use in Real-time request processing. The
+ * resolvers are perf-sensitive and are used in the flow of the
+ * {@link RouterRpcServer} request path.
+ * <p>
+ * The principal resolvers are:
+ * <ul>
+ * <li>{@link ActiveNamenodeResolver} Real-time interface for locating the most
+ * recently active NN for a nameservice.
+ * <li>{@link FileSubclusterResolver} Real-time interface for determining the NN
+ * and local file path for a given file/folder based on the global namespace
+ * path.
+ * </ul>
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java
new file mode 100644
index 0000000..1d27b51
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.hadoop.hdfs.NameNodeProxiesClient.ProxyAndInfo;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.ipc.RPC;
+
+/**
+ * Context to track a connection in a {@link ConnectionPool}. When a client uses
+ * a connection, it increments a counter to mark it as active. Once the client
+ * is done with the connection, it decreases the counter. It also takes care of
+ * closing the connection once is not active.
+ */
+public class ConnectionContext {
+
+ /** Client for the connection. */
+ private final ProxyAndInfo<ClientProtocol> client;
+ /** How many threads are using this connection. */
+ private int numThreads = 0;
+ /** If the connection is closed. */
+ private boolean closed = false;
+
+
+ public ConnectionContext(ProxyAndInfo<ClientProtocol> connection) {
+ this.client = connection;
+ }
+
+ /**
+ * Check if the connection is active.
+ *
+ * @return True if the connection is active.
+ */
+ public synchronized boolean isActive() {
+ return this.numThreads > 0;
+ }
+
+ /**
+ * Check if the connection is closed.
+ *
+ * @return If the connection is closed.
+ */
+ public synchronized boolean isClosed() {
+ return this.closed;
+ }
+
+ /**
+ * Check if the connection can be used. It checks if the connection is used by
+ * another thread or already closed.
+ *
+ * @return True if the connection can be used.
+ */
+ public synchronized boolean isUsable() {
+ return !isActive() && !isClosed();
+ }
+
+ /**
+ * Get the connection client.
+ *
+ * @return Connection client.
+ */
+ public synchronized ProxyAndInfo<ClientProtocol> getClient() {
+ this.numThreads++;
+ return this.client;
+ }
+
+ /**
+ * Release this connection. If the connection was closed, close the proxy.
+ * Otherwise, mark the connection as not used by us anymore.
+ */
+ public synchronized void release() {
+ if (--this.numThreads == 0 && this.closed) {
+ close();
+ }
+ }
+
+ /**
+ * We will not use this connection anymore. If it's not being used, we close
+ * it. Otherwise, we let release() do it once we are done with it.
+ */
+ public synchronized void close() {
+ this.closed = true;
+ if (this.numThreads == 0) {
+ ClientProtocol proxy = this.client.getProxy();
+ // Nobody should be using this anymore so it should close right away
+ RPC.stopProxy(proxy);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java
new file mode 100644
index 0000000..97c6403
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java
@@ -0,0 +1,435 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Time;
+import org.eclipse.jetty.util.ajax.JSON;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Implements a pool of connections for the {@link Router} to be able to open
+ * many connections to many Namenodes.
+ */
+public class ConnectionManager {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ConnectionManager.class);
+
+ /** Number of parallel new connections to create. */
+ protected static final int MAX_NEW_CONNECTIONS = 100;
+
+ /** Minimum amount of active connections: 50%. */
+ protected static final float MIN_ACTIVE_RATIO = 0.5f;
+
+
+ /** Configuration for the connection manager, pool and sockets. */
+ private final Configuration conf;
+
+ /** Min number of connections per user + nn. */
+ private final int minSize = 1;
+ /** Max number of connections per user + nn. */
+ private final int maxSize;
+
+ /** How often we close a pool for a particular user + nn. */
+ private final long poolCleanupPeriodMs;
+ /** How often we close a connection in a pool. */
+ private final long connectionCleanupPeriodMs;
+
+ /** Map of connection pools, one pool per user + NN. */
+ private final Map<ConnectionPoolId, ConnectionPool> pools;
+ /** Lock for accessing pools. */
+ private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ private final Lock readLock = readWriteLock.readLock();
+ private final Lock writeLock = readWriteLock.writeLock();
+
+ /** Queue for creating new connections. */
+ private final BlockingQueue<ConnectionPool> creatorQueue =
+ new ArrayBlockingQueue<>(MAX_NEW_CONNECTIONS);
+ /** Create new connections asynchronously. */
+ private final ConnectionCreator creator;
+ /** Periodic executor to remove stale connection pools. */
+ private final ScheduledThreadPoolExecutor cleaner =
+ new ScheduledThreadPoolExecutor(1);
+
+ /** If the connection manager is running. */
+ private boolean running = false;
+
+
+ /**
+ * Creates a proxy client connection pool manager.
+ *
+ * @param config Configuration for the connections.
+ */
+ public ConnectionManager(Configuration config) {
+ this.conf = config;
+
+ // Configure minimum and maximum connection pools
+ this.maxSize = this.conf.getInt(
+ RBFConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_POOL_SIZE,
+ RBFConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_POOL_SIZE_DEFAULT);
+
+ // Map with the connections indexed by UGI and Namenode
+ this.pools = new HashMap<>();
+
+ // Create connections in a thread asynchronously
+ this.creator = new ConnectionCreator(creatorQueue);
+ this.creator.setDaemon(true);
+
+ // Cleanup periods
+ this.poolCleanupPeriodMs = this.conf.getLong(
+ RBFConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_POOL_CLEAN,
+ RBFConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_POOL_CLEAN_DEFAULT);
+ LOG.info("Cleaning connection pools every {} seconds",
+ TimeUnit.MILLISECONDS.toSeconds(this.poolCleanupPeriodMs));
+ this.connectionCleanupPeriodMs = this.conf.getLong(
+ RBFConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_CLEAN_MS,
+ RBFConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_CLEAN_MS_DEFAULT);
+ LOG.info("Cleaning connections every {} seconds",
+ TimeUnit.MILLISECONDS.toSeconds(this.connectionCleanupPeriodMs));
+ }
+
+ /**
+ * Start the connection manager.
+ */
+ public void start() {
+ // Start the thread that creates connections asynchronously
+ this.creator.start();
+
+ // Schedule a task to remove stale connection pools and sockets
+ long recyleTimeMs = Math.min(
+ poolCleanupPeriodMs, connectionCleanupPeriodMs);
+ LOG.info("Cleaning every {} seconds",
+ TimeUnit.MILLISECONDS.toSeconds(recyleTimeMs));
+ this.cleaner.scheduleAtFixedRate(
+ new CleanupTask(), 0, recyleTimeMs, TimeUnit.MILLISECONDS);
+
+ // Mark the manager as running
+ this.running = true;
+ }
+
+ /**
+ * Stop the connection manager by closing all the pools.
+ */
+ public void close() {
+ this.creator.shutdown();
+ this.cleaner.shutdown();
+ this.running = false;
+
+ writeLock.lock();
+ try {
+ for (ConnectionPool pool : this.pools.values()) {
+ pool.close();
+ }
+ this.pools.clear();
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /**
+ * Fetches the next available proxy client in the pool. Each client connection
+ * is reserved for a single user and cannot be reused until free.
+ *
+ * @param ugi User group information.
+ * @param nnAddress Namenode address for the connection.
+ * @return Proxy client to connect to nnId as UGI.
+ * @throws IOException If the connection cannot be obtained.
+ */
+ public ConnectionContext getConnection(
+ UserGroupInformation ugi, String nnAddress) throws IOException {
+
+ // Check if the manager is shutdown
+ if (!this.running) {
+ LOG.error(
+ "Cannot get a connection to {} because the manager isn't running",
+ nnAddress);
+ return null;
+ }
+
+ // Try to get the pool if created
+ ConnectionPoolId connectionId = new ConnectionPoolId(ugi, nnAddress);
+ ConnectionPool pool = null;
+ readLock.lock();
+ try {
+ pool = this.pools.get(connectionId);
+ } finally {
+ readLock.unlock();
+ }
+
+ // Create the pool if not created before
+ if (pool == null) {
+ writeLock.lock();
+ try {
+ pool = this.pools.get(connectionId);
+ if (pool == null) {
+ pool = new ConnectionPool(
+ this.conf, nnAddress, ugi, this.minSize, this.maxSize);
+ this.pools.put(connectionId, pool);
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ ConnectionContext conn = pool.getConnection();
+
+ // Add a new connection to the pool if it wasn't usable
+ if (conn == null || !conn.isUsable()) {
+ if (!this.creatorQueue.offer(pool)) {
+ LOG.error("Cannot add more than {} connections at the same time",
+ MAX_NEW_CONNECTIONS);
+ }
+ }
+
+ if (conn != null && conn.isClosed()) {
+ LOG.error("We got a closed connection from {}", pool);
+ conn = null;
+ }
+
+ return conn;
+ }
+
+ /**
+ * Get the number of connection pools.
+ *
+ * @return Number of connection pools.
+ */
+ public int getNumConnectionPools() {
+ readLock.lock();
+ try {
+ return pools.size();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ /**
+ * Get number of open connections.
+ *
+ * @return Number of open connections.
+ */
+ public int getNumConnections() {
+ int total = 0;
+ readLock.lock();
+ try {
+ for (ConnectionPool pool : this.pools.values()) {
+ total += pool.getNumConnections();
+ }
+ } finally {
+ readLock.unlock();
+ }
+ return total;
+ }
+
+ /**
+ * Get number of active connections.
+ *
+ * @return Number of active connections.
+ */
+ public int getNumActiveConnections() {
+ int total = 0;
+ readLock.lock();
+ try {
+ for (ConnectionPool pool : this.pools.values()) {
+ total += pool.getNumActiveConnections();
+ }
+ } finally {
+ readLock.unlock();
+ }
+ return total;
+ }
+
+ /**
+ * Get the number of connections to be created.
+ *
+ * @return Number of connections to be created.
+ */
+ public int getNumCreatingConnections() {
+ return this.creatorQueue.size();
+ }
+
+ /**
+ * Get a JSON representation of the connection pool.
+ *
+ * @return JSON representation of all the connection pools.
+ */
+ public String getJSON() {
+ final Map<String, String> info = new TreeMap<>();
+ readLock.lock();
+ try {
+ for (Entry<ConnectionPoolId, ConnectionPool> entry :
+ this.pools.entrySet()) {
+ ConnectionPoolId connectionPoolId = entry.getKey();
+ ConnectionPool pool = entry.getValue();
+ info.put(connectionPoolId.toString(), pool.getJSON());
+ }
+ } finally {
+ readLock.unlock();
+ }
+ return JSON.toString(info);
+ }
+
+ @VisibleForTesting
+ Map<ConnectionPoolId, ConnectionPool> getPools() {
+ return this.pools;
+ }
+
+ /**
+ * Clean the unused connections for this pool.
+ *
+ * @param pool Connection pool to cleanup.
+ */
+ @VisibleForTesting
+ void cleanup(ConnectionPool pool) {
+ if (pool.getNumConnections() > pool.getMinSize()) {
+ // Check if the pool hasn't been active in a while or not 50% are used
+ long timeSinceLastActive = Time.now() - pool.getLastActiveTime();
+ int total = pool.getNumConnections();
+ int active = pool.getNumActiveConnections();
+ if (timeSinceLastActive > connectionCleanupPeriodMs ||
+ active < MIN_ACTIVE_RATIO * total) {
+ // Remove and close 1 connection
+ List<ConnectionContext> conns = pool.removeConnections(1);
+ for (ConnectionContext conn : conns) {
+ conn.close();
+ }
+ LOG.debug("Removed connection {} used {} seconds ago. " +
+ "Pool has {}/{} connections", pool.getConnectionPoolId(),
+ TimeUnit.MILLISECONDS.toSeconds(timeSinceLastActive),
+ pool.getNumConnections(), pool.getMaxSize());
+ }
+ }
+ }
+
+ /**
+ * Removes stale connections not accessed recently from the pool. This is
+ * invoked periodically.
+ */
+ private class CleanupTask implements Runnable {
+
+ @Override
+ public void run() {
+ long currentTime = Time.now();
+ List<ConnectionPoolId> toRemove = new LinkedList<>();
+
+ // Look for stale pools
+ readLock.lock();
+ try {
+ for (Entry<ConnectionPoolId, ConnectionPool> entry : pools.entrySet()) {
+ ConnectionPool pool = entry.getValue();
+ long lastTimeActive = pool.getLastActiveTime();
+ boolean isStale =
+ currentTime > (lastTimeActive + poolCleanupPeriodMs);
+ if (lastTimeActive > 0 && isStale) {
+ // Remove this pool
+ LOG.debug("Closing and removing stale pool {}", pool);
+ pool.close();
+ ConnectionPoolId poolId = entry.getKey();
+ toRemove.add(poolId);
+ } else {
+ // Keep this pool but clean connections inside
+ LOG.debug("Cleaning up {}", pool);
+ cleanup(pool);
+ }
+ }
+ } finally {
+ readLock.unlock();
+ }
+
+ // Remove stale pools
+ if (!toRemove.isEmpty()) {
+ writeLock.lock();
+ try {
+ for (ConnectionPoolId poolId : toRemove) {
+ pools.remove(poolId);
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+ }
+ }
+
+ /**
+ * Thread that creates connections asynchronously.
+ */
+ private static class ConnectionCreator extends Thread {
+ /** If the creator is running. */
+ private boolean running = true;
+ /** Queue to push work to. */
+ private BlockingQueue<ConnectionPool> queue;
+
+ ConnectionCreator(BlockingQueue<ConnectionPool> blockingQueue) {
+ super("Connection creator");
+ this.queue = blockingQueue;
+ }
+
+ @Override
+ public void run() {
+ while (this.running) {
+ try {
+ ConnectionPool pool = this.queue.take();
+ try {
+ int total = pool.getNumConnections();
+ int active = pool.getNumActiveConnections();
+ if (pool.getNumConnections() < pool.getMaxSize() &&
+ active >= MIN_ACTIVE_RATIO * total) {
+ ConnectionContext conn = pool.newConnection();
+ pool.addConnection(conn);
+ } else {
+ LOG.debug("Cannot add more than {} connections to {}",
+ pool.getMaxSize(), pool);
+ }
+ } catch (IOException e) {
+ LOG.error("Cannot create a new connection", e);
+ }
+ } catch (InterruptedException e) {
+ LOG.error("The connection creator was interrupted");
+ this.running = false;
+ }
+ }
+ }
+
+ /**
+ * Stop this connection creator.
+ */
+ public void shutdown() {
+ this.running = false;
+ this.interrupt();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java
new file mode 100644
index 0000000..5af8a86
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java
@@ -0,0 +1,337 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import javax.net.SocketFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.NameNodeProxiesClient.ProxyAndInfo;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB;
+import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.io.retry.RetryUtils;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SaslRpcServer;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Time;
+import org.eclipse.jetty.util.ajax.JSON;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Maintains a pool of connections for each User (including tokens) + NN. The
+ * RPC client maintains a single socket, to achieve throughput similar to a NN,
+ * each request is multiplexed across multiple sockets/connections from a
+ * pool.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class ConnectionPool {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ConnectionPool.class);
+
+
+ /** Configuration settings for the connection pool. */
+ private final Configuration conf;
+
+ /** Identifier for this connection pool. */
+ private final ConnectionPoolId connectionPoolId;
+ /** Namenode this pool connects to. */
+ private final String namenodeAddress;
+ /** User for this connections. */
+ private final UserGroupInformation ugi;
+
+ /** Pool of connections. We mimic a COW array. */
+ private volatile List<ConnectionContext> connections = new ArrayList<>();
+ /** Connection index for round-robin. */
+ private final AtomicInteger clientIndex = new AtomicInteger(0);
+
+ /** Min number of connections per user. */
+ private final int minSize;
+ /** Max number of connections per user. */
+ private final int maxSize;
+
+ /** The last time a connection was active. */
+ private volatile long lastActiveTime = 0;
+
+
+ protected ConnectionPool(Configuration config, String address,
+ UserGroupInformation user, int minPoolSize, int maxPoolSize)
+ throws IOException {
+
+ this.conf = config;
+
+ // Connection pool target
+ this.ugi = user;
+ this.namenodeAddress = address;
+ this.connectionPoolId =
+ new ConnectionPoolId(this.ugi, this.namenodeAddress);
+
+ // Set configuration parameters for the pool
+ this.minSize = minPoolSize;
+ this.maxSize = maxPoolSize;
+
+ // Add minimum connections to the pool
+ for (int i=0; i<this.minSize; i++) {
+ ConnectionContext newConnection = newConnection();
+ this.connections.add(newConnection);
+ }
+ LOG.debug("Created connection pool \"{}\" with {} connections",
+ this.connectionPoolId, this.minSize);
+ }
+
+ /**
+ * Get the maximum number of connections allowed in this pool.
+ *
+ * @return Maximum number of connections.
+ */
+ protected int getMaxSize() {
+ return this.maxSize;
+ }
+
+ /**
+ * Get the minimum number of connections in this pool.
+ *
+ * @return Minimum number of connections.
+ */
+ protected int getMinSize() {
+ return this.minSize;
+ }
+
+ /**
+ * Get the connection pool identifier.
+ *
+ * @return Connection pool identifier.
+ */
+ protected ConnectionPoolId getConnectionPoolId() {
+ return this.connectionPoolId;
+ }
+
+ /**
+ * Return the next connection round-robin.
+ *
+ * @return Connection context.
+ */
+ protected ConnectionContext getConnection() {
+
+ this.lastActiveTime = Time.now();
+
+ // Get a connection from the pool following round-robin
+ ConnectionContext conn = null;
+ List<ConnectionContext> tmpConnections = this.connections;
+ int size = tmpConnections.size();
+ int threadIndex = this.clientIndex.getAndIncrement();
+ for (int i=0; i<size; i++) {
+ int index = (threadIndex + i) % size;
+ conn = tmpConnections.get(index);
+ if (conn != null && conn.isUsable()) {
+ return conn;
+ }
+ }
+
+ // We return a connection even if it's active
+ return conn;
+ }
+
+ /**
+ * Add a connection to the current pool. It uses a Copy-On-Write approach.
+ *
+ * @param conn New connection to add to the pool.
+ */
+ public synchronized void addConnection(ConnectionContext conn) {
+ List<ConnectionContext> tmpConnections = new ArrayList<>(this.connections);
+ tmpConnections.add(conn);
+ this.connections = tmpConnections;
+
+ this.lastActiveTime = Time.now();
+ }
+
+ /**
+ * Remove connections from the current pool.
+ *
+ * @param num Number of connections to remove.
+ * @return Removed connections.
+ */
+ public synchronized List<ConnectionContext> removeConnections(int num) {
+ List<ConnectionContext> removed = new LinkedList<>();
+
+ // Remove and close the last connection
+ List<ConnectionContext> tmpConnections = new ArrayList<>();
+ for (int i=0; i<this.connections.size(); i++) {
+ ConnectionContext conn = this.connections.get(i);
+ if (i < this.minSize || i < this.connections.size() - num) {
+ tmpConnections.add(conn);
+ } else {
+ removed.add(conn);
+ }
+ }
+ this.connections = tmpConnections;
+
+ return removed;
+ }
+
+ /**
+ * Close the connection pool.
+ */
+ protected synchronized void close() {
+ long timeSinceLastActive = TimeUnit.MILLISECONDS.toSeconds(
+ Time.now() - getLastActiveTime());
+ LOG.debug("Shutting down connection pool \"{}\" used {} seconds ago",
+ this.connectionPoolId, timeSinceLastActive);
+
+ for (ConnectionContext connection : this.connections) {
+ connection.close();
+ }
+ this.connections.clear();
+ }
+
+ /**
+ * Number of connections in the pool.
+ *
+ * @return Number of connections.
+ */
+ protected int getNumConnections() {
+ return this.connections.size();
+ }
+
+ /**
+ * Number of active connections in the pool.
+ *
+ * @return Number of active connections.
+ */
+ protected int getNumActiveConnections() {
+ int ret = 0;
+
+ List<ConnectionContext> tmpConnections = this.connections;
+ for (ConnectionContext conn : tmpConnections) {
+ if (conn.isActive()) {
+ ret++;
+ }
+ }
+ return ret;
+ }
+
+ /**
+ * Get the last time the connection pool was used.
+ *
+ * @return Last time the connection pool was used.
+ */
+ protected long getLastActiveTime() {
+ return this.lastActiveTime;
+ }
+
+ @Override
+ public String toString() {
+ return this.connectionPoolId.toString();
+ }
+
+ /**
+ * JSON representation of the connection pool.
+ *
+ * @return String representation of the JSON.
+ */
+ public String getJSON() {
+ final Map<String, String> info = new LinkedHashMap<>();
+ info.put("active", Integer.toString(getNumActiveConnections()));
+ info.put("total", Integer.toString(getNumConnections()));
+ if (LOG.isDebugEnabled()) {
+ List<ConnectionContext> tmpConnections = this.connections;
+ for (int i=0; i<tmpConnections.size(); i++) {
+ ConnectionContext connection = tmpConnections.get(i);
+ info.put(i + " active", Boolean.toString(connection.isActive()));
+ info.put(i + " closed", Boolean.toString(connection.isClosed()));
+ }
+ }
+ return JSON.toString(info);
+ }
+
+ /**
+ * Create a new proxy wrapper for a client NN connection.
+ * @return Proxy for the target ClientProtocol that contains the user's
+ * security context.
+ * @throws IOException
+ */
+ public ConnectionContext newConnection() throws IOException {
+ return newConnection(this.conf, this.namenodeAddress, this.ugi);
+ }
+
+ /**
+ * Creates a proxy wrapper for a client NN connection. Each proxy contains
+ * context for a single user/security context. To maximize throughput it is
+ * recommended to use multiple connection per user+server, allowing multiple
+ * writes and reads to be dispatched in parallel.
+ *
+ * @param conf Configuration for the connection.
+ * @param nnAddress Address of server supporting the ClientProtocol.
+ * @param ugi User context.
+ * @return Proxy for the target ClientProtocol that contains the user's
+ * security context.
+ * @throws IOException If it cannot be created.
+ */
+ protected static ConnectionContext newConnection(Configuration conf,
+ String nnAddress, UserGroupInformation ugi)
+ throws IOException {
+ RPC.setProtocolEngine(
+ conf, ClientNamenodeProtocolPB.class, ProtobufRpcEngine.class);
+
+ final RetryPolicy defaultPolicy = RetryUtils.getDefaultRetryPolicy(
+ conf,
+ HdfsClientConfigKeys.Retry.POLICY_ENABLED_KEY,
+ HdfsClientConfigKeys.Retry.POLICY_ENABLED_DEFAULT,
+ HdfsClientConfigKeys.Retry.POLICY_SPEC_KEY,
+ HdfsClientConfigKeys.Retry.POLICY_SPEC_DEFAULT,
+ HdfsConstants.SAFEMODE_EXCEPTION_CLASS_NAME);
+
+ SocketFactory factory = SocketFactory.getDefault();
+ if (UserGroupInformation.isSecurityEnabled()) {
+ SaslRpcServer.init(conf);
+ }
+ InetSocketAddress socket = NetUtils.createSocketAddr(nnAddress);
+ final long version = RPC.getProtocolVersion(ClientNamenodeProtocolPB.class);
+ ClientNamenodeProtocolPB proxy = RPC.getProtocolProxy(
+ ClientNamenodeProtocolPB.class, version, socket, ugi, conf,
+ factory, RPC.getRpcTimeout(conf), defaultPolicy, null).getProxy();
+ ClientProtocol client = new ClientNamenodeProtocolTranslatorPB(proxy);
+ Text dtService = SecurityUtil.buildTokenService(socket);
+
+ ProxyAndInfo<ClientProtocol> clientProxy =
+ new ProxyAndInfo<ClientProtocol>(client, dtService, socket);
+ ConnectionContext connection = new ConnectionContext(clientProxy);
+ return connection;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java
new file mode 100644
index 0000000..6e1ee9a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+
+/**
+ * Identifier for a connection for a user to a namenode.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class ConnectionPoolId implements Comparable<ConnectionPoolId> {
+
+ /** Namenode identifier. */
+ private final String nnId;
+ /** Information about the user. */
+ private final UserGroupInformation ugi;
+
+ /**
+ * New connection pool identifier.
+ *
+ * @param ugi Information of the user issuing the request.
+ * @param nnId Namenode address with port.
+ */
+ public ConnectionPoolId(final UserGroupInformation ugi, final String nnId) {
+ this.nnId = nnId;
+ this.ugi = ugi;
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = new HashCodeBuilder(17, 31)
+ .append(this.nnId)
+ .append(this.ugi.toString())
+ .append(this.getTokenIds())
+ .toHashCode();
+ return hash;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof ConnectionPoolId) {
+ ConnectionPoolId other = (ConnectionPoolId) o;
+ if (!this.nnId.equals(other.nnId)) {
+ return false;
+ }
+ if (!this.ugi.toString().equals(other.ugi.toString())) {
+ return false;
+ }
+ String thisTokens = this.getTokenIds().toString();
+ String otherTokens = other.getTokenIds().toString();
+ return thisTokens.equals(otherTokens);
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return this.ugi + " " + this.getTokenIds() + "->" + this.nnId;
+ }
+
+ @Override
+ public int compareTo(ConnectionPoolId other) {
+ int ret = this.nnId.compareTo(other.nnId);
+ if (ret == 0) {
+ ret = this.ugi.toString().compareTo(other.ugi.toString());
+ }
+ if (ret == 0) {
+ String thisTokens = this.getTokenIds().toString();
+ String otherTokens = other.getTokenIds().toString();
+ ret = thisTokens.compareTo(otherTokens);
+ }
+ return ret;
+ }
+
+ @VisibleForTesting
+ UserGroupInformation getUgi() {
+ return this.ugi;
+ }
+
+ /**
+ * Get the token identifiers for this connection.
+ * @return List with the token identifiers.
+ */
+ private List<String> getTokenIds() {
+ List<String> tokenIds = new ArrayList<>();
+ Collection<Token<? extends TokenIdentifier>> tokens = this.ugi.getTokens();
+ for (Token<? extends TokenIdentifier> token : tokens) {
+ byte[] tokenIdBytes = token.getIdentifier();
+ String tokenId = Arrays.toString(tokenIdBytes);
+ tokenIds.add(tokenId);
+ }
+ Collections.sort(tokenIds);
+ return tokenIds;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/DFSRouter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/DFSRouter.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/DFSRouter.java
new file mode 100644
index 0000000..a2ac258
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/DFSRouter.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.apache.hadoop.util.ExitUtil.terminate;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.service.CompositeService.CompositeServiceShutdownHook;
+import org.apache.hadoop.util.ShutdownHookManager;
+import org.apache.hadoop.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Tool to start the {@link Router} for Router-based federation.
+ */
+public final class DFSRouter {
+
+ private static final Logger LOG = LoggerFactory.getLogger(DFSRouter.class);
+
+
+ /** Usage string for help message. */
+ private static final String USAGE = "Usage: hdfs dfsrouter";
+
+ /** Priority of the Router shutdown hook. */
+ public static final int SHUTDOWN_HOOK_PRIORITY = 30;
+
+
+ private DFSRouter() {
+ // This is just a class to trigger the Router
+ }
+
+ /**
+ * Main run loop for the router.
+ *
+ * @param argv parameters.
+ */
+ public static void main(String[] argv) {
+ if (DFSUtil.parseHelpArgument(argv, USAGE, System.out, true)) {
+ System.exit(0);
+ }
+
+ try {
+ StringUtils.startupShutdownMessage(Router.class, argv, LOG);
+
+ Router router = new Router();
+
+ ShutdownHookManager.get().addShutdownHook(
+ new CompositeServiceShutdownHook(router), SHUTDOWN_HOOK_PRIORITY);
+
+ Configuration conf = new HdfsConfiguration();
+ router.init(conf);
+ router.start();
+ } catch (Throwable e) {
+ LOG.error("Failed to start router", e);
+ terminate(1, e);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ErasureCoding.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ErasureCoding.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ErasureCoding.java
new file mode 100644
index 0000000..d2b2d50
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ErasureCoding.java
@@ -0,0 +1,198 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.apache.hadoop.hdfs.server.federation.router.RouterRpcServer.merge;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse;
+import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
+
+/**
+ * Module that implements all the RPC calls in
+ * {@link org.apache.hadoop.hdfs.protocol.ClientProtocol} related to
+ * Erasure Coding in the {@link RouterRpcServer}.
+ */
+public class ErasureCoding {
+
+ /** RPC server to receive client calls. */
+ private final RouterRpcServer rpcServer;
+ /** RPC clients to connect to the Namenodes. */
+ private final RouterRpcClient rpcClient;
+ /** Interface to identify the active NN for a nameservice or blockpool ID. */
+ private final ActiveNamenodeResolver namenodeResolver;
+
+
+ public ErasureCoding(RouterRpcServer server) {
+ this.rpcServer = server;
+ this.rpcClient = this.rpcServer.getRPCClient();
+ this.namenodeResolver = this.rpcClient.getNamenodeResolver();
+ }
+
+ public ErasureCodingPolicyInfo[] getErasureCodingPolicies()
+ throws IOException {
+ rpcServer.checkOperation(OperationCategory.READ);
+
+ RemoteMethod method = new RemoteMethod("getErasureCodingPolicies");
+ Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, ErasureCodingPolicyInfo[]> ret =
+ rpcClient.invokeConcurrent(
+ nss, method, true, false, ErasureCodingPolicyInfo[].class);
+ return merge(ret, ErasureCodingPolicyInfo.class);
+ }
+
+ public Map<String, String> getErasureCodingCodecs() throws IOException {
+ rpcServer.checkOperation(OperationCategory.READ);
+
+ RemoteMethod method = new RemoteMethod("getErasureCodingCodecs");
+ Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ @SuppressWarnings("rawtypes")
+ Map<FederationNamespaceInfo, Map> retCodecs =
+ rpcClient.invokeConcurrent(
+ nss, method, true, false, Map.class);
+
+ Map<String, String> ret = new HashMap<>();
+ Object obj = retCodecs;
+ @SuppressWarnings("unchecked")
+ Map<FederationNamespaceInfo, Map<String, String>> results =
+ (Map<FederationNamespaceInfo, Map<String, String>>)obj;
+ Collection<Map<String, String>> allCodecs = results.values();
+ for (Map<String, String> codecs : allCodecs) {
+ ret.putAll(codecs);
+ }
+
+ return ret;
+ }
+
+ public AddErasureCodingPolicyResponse[] addErasureCodingPolicies(
+ ErasureCodingPolicy[] policies) throws IOException {
+ rpcServer.checkOperation(OperationCategory.WRITE);
+
+ RemoteMethod method = new RemoteMethod("addErasureCodingPolicies",
+ new Class<?>[] {ErasureCodingPolicy[].class}, new Object[] {policies});
+ Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, AddErasureCodingPolicyResponse[]> ret =
+ rpcClient.invokeConcurrent(
+ nss, method, true, false, AddErasureCodingPolicyResponse[].class);
+
+ return merge(ret, AddErasureCodingPolicyResponse.class);
+ }
+
+ public void removeErasureCodingPolicy(String ecPolicyName)
+ throws IOException {
+ rpcServer.checkOperation(OperationCategory.WRITE);
+
+ RemoteMethod method = new RemoteMethod("removeErasureCodingPolicy",
+ new Class<?>[] {String.class}, ecPolicyName);
+ Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ rpcClient.invokeConcurrent(nss, method, true, false);
+ }
+
+ public void disableErasureCodingPolicy(String ecPolicyName)
+ throws IOException {
+ rpcServer.checkOperation(OperationCategory.WRITE);
+
+ RemoteMethod method = new RemoteMethod("disableErasureCodingPolicy",
+ new Class<?>[] {String.class}, ecPolicyName);
+ Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ rpcClient.invokeConcurrent(nss, method, true, false);
+ }
+
+ public void enableErasureCodingPolicy(String ecPolicyName)
+ throws IOException {
+ rpcServer.checkOperation(OperationCategory.WRITE);
+
+ RemoteMethod method = new RemoteMethod("enableErasureCodingPolicy",
+ new Class<?>[] {String.class}, ecPolicyName);
+ Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ rpcClient.invokeConcurrent(nss, method, true, false);
+ }
+
+ public ErasureCodingPolicy getErasureCodingPolicy(String src)
+ throws IOException {
+ rpcServer.checkOperation(OperationCategory.READ);
+
+ final List<RemoteLocation> locations =
+ rpcServer.getLocationsForPath(src, true);
+ RemoteMethod remoteMethod = new RemoteMethod("getErasureCodingPolicy",
+ new Class<?>[] {String.class}, new RemoteParam());
+ ErasureCodingPolicy ret = rpcClient.invokeSequential(
+ locations, remoteMethod, null, null);
+ return ret;
+ }
+
+ public void setErasureCodingPolicy(String src, String ecPolicyName)
+ throws IOException {
+ rpcServer.checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations =
+ rpcServer.getLocationsForPath(src, true);
+ RemoteMethod remoteMethod = new RemoteMethod("setErasureCodingPolicy",
+ new Class<?>[] {String.class, String.class},
+ new RemoteParam(), ecPolicyName);
+ rpcClient.invokeSequential(locations, remoteMethod, null, null);
+ }
+
+ public void unsetErasureCodingPolicy(String src) throws IOException {
+ rpcServer.checkOperation(OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations =
+ rpcServer.getLocationsForPath(src, true);
+ RemoteMethod remoteMethod = new RemoteMethod("unsetErasureCodingPolicy",
+ new Class<?>[] {String.class}, new RemoteParam());
+ rpcClient.invokeSequential(locations, remoteMethod, null, null);
+ }
+
+ public ECBlockGroupStats getECBlockGroupStats() throws IOException {
+ rpcServer.checkOperation(OperationCategory.READ);
+
+ RemoteMethod method = new RemoteMethod("getECBlockGroupStats");
+ Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ Map<FederationNamespaceInfo, ECBlockGroupStats> allStats =
+ rpcClient.invokeConcurrent(
+ nss, method, true, false, ECBlockGroupStats.class);
+
+ // Merge the stats from all the namespaces
+ long lowRedundancyBlockGroups = 0;
+ long corruptBlockGroups = 0;
+ long missingBlockGroups = 0;
+ long bytesInFutureBlockGroups = 0;
+ long pendingDeletionBlocks = 0;
+ for (ECBlockGroupStats stats : allStats.values()) {
+ lowRedundancyBlockGroups += stats.getLowRedundancyBlockGroups();
+ corruptBlockGroups += stats.getCorruptBlockGroups();
+ missingBlockGroups += stats.getMissingBlockGroups();
+ bytesInFutureBlockGroups += stats.getBytesInFutureBlockGroups();
+ pendingDeletionBlocks += stats.getPendingDeletionBlocks();
+ }
+ return new ECBlockGroupStats(lowRedundancyBlockGroups, corruptBlockGroups,
+ missingBlockGroups, bytesInFutureBlockGroups, pendingDeletionBlocks);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
new file mode 100644
index 0000000..f8c7a9b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
@@ -0,0 +1,208 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.lang.reflect.Constructor;
+import java.net.URL;
+import java.net.URLConnection;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.util.VersionInfo;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utilities for managing HDFS federation.
+ */
+public final class FederationUtil {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(FederationUtil.class);
+
+ private FederationUtil() {
+ // Utility Class
+ }
+
+ /**
+ * Get a JMX data from a web endpoint.
+ *
+ * @param beanQuery JMX bean.
+ * @param webAddress Web address of the JMX endpoint.
+ * @return JSON with the JMX data
+ */
+ public static JSONArray getJmx(String beanQuery, String webAddress) {
+ JSONArray ret = null;
+ BufferedReader reader = null;
+ try {
+ String host = webAddress;
+ int port = -1;
+ if (webAddress.indexOf(":") > 0) {
+ String[] webAddressSplit = webAddress.split(":");
+ host = webAddressSplit[0];
+ port = Integer.parseInt(webAddressSplit[1]);
+ }
+ URL jmxURL = new URL("http", host, port, "/jmx?qry=" + beanQuery);
+ URLConnection conn = jmxURL.openConnection();
+ conn.setConnectTimeout(5 * 1000);
+ conn.setReadTimeout(5 * 1000);
+ InputStream in = conn.getInputStream();
+ InputStreamReader isr = new InputStreamReader(in, "UTF-8");
+ reader = new BufferedReader(isr);
+
+ StringBuilder sb = new StringBuilder();
+ String line = null;
+ while ((line = reader.readLine()) != null) {
+ sb.append(line);
+ }
+ String jmxOutput = sb.toString();
+
+ // Parse JSON
+ JSONObject json = new JSONObject(jmxOutput);
+ ret = json.getJSONArray("beans");
+ } catch (IOException e) {
+ LOG.error("Cannot read JMX bean {} from server {}: {}",
+ beanQuery, webAddress, e.getMessage());
+ } catch (JSONException e) {
+ LOG.error("Cannot parse JMX output for {} from server {}: {}",
+ beanQuery, webAddress, e.getMessage());
+ } catch (Exception e) {
+ LOG.error("Cannot parse JMX output for {} from server {}: {}",
+ beanQuery, webAddress, e);
+ } finally {
+ if (reader != null) {
+ try {
+ reader.close();
+ } catch (IOException e) {
+ LOG.error("Problem closing {}", webAddress, e);
+ }
+ }
+ }
+ return ret;
+ }
+
+ /**
+ * Fetch the Hadoop version string for this jar.
+ *
+ * @return Hadoop version string, e.g., 3.0.1.
+ */
+ public static String getVersion() {
+ return VersionInfo.getVersion();
+ }
+
+ /**
+ * Fetch the build/compile information for this jar.
+ *
+ * @return String Compilation info.
+ */
+ public static String getCompileInfo() {
+ return VersionInfo.getDate() + " by " + VersionInfo.getUser() + " from "
+ + VersionInfo.getBranch();
+ }
+
+ /**
+ * Create an instance of an interface with a constructor using a context.
+ *
+ * @param conf Configuration for the class names.
+ * @param context Context object to pass to the instance.
+ * @param contextClass Type of the context passed to the constructor.
+ * @param clazz Class of the object to return.
+ * @return New instance of the specified class that implements the desired
+ * interface and a single parameter constructor containing a
+ * StateStore reference.
+ */
+ private static <T, R> T newInstance(final Configuration conf,
+ final R context, final Class<R> contextClass, final Class<T> clazz) {
+ try {
+ if (contextClass == null) {
+ // Default constructor if no context
+ Constructor<T> constructor = clazz.getConstructor();
+ return constructor.newInstance();
+ } else {
+ // Constructor with context
+ Constructor<T> constructor = clazz.getConstructor(
+ Configuration.class, contextClass);
+ return constructor.newInstance(conf, context);
+ }
+ } catch (ReflectiveOperationException e) {
+ LOG.error("Could not instantiate: {}", clazz.getSimpleName(), e);
+ return null;
+ }
+ }
+
+ /**
+ * Creates an instance of a FileSubclusterResolver from the configuration.
+ *
+ * @param conf Configuration that defines the file resolver class.
+ * @param router Router service.
+ * @return New file subcluster resolver.
+ */
+ public static FileSubclusterResolver newFileSubclusterResolver(
+ Configuration conf, Router router) {
+ Class<? extends FileSubclusterResolver> clazz = conf.getClass(
+ RBFConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
+ RBFConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS_DEFAULT,
+ FileSubclusterResolver.class);
+ return newInstance(conf, router, Router.class, clazz);
+ }
+
+ /**
+ * Creates an instance of an ActiveNamenodeResolver from the configuration.
+ *
+ * @param conf Configuration that defines the namenode resolver class.
+ * @param stateStore State store passed to class constructor.
+ * @return New active namenode resolver.
+ */
+ public static ActiveNamenodeResolver newActiveNamenodeResolver(
+ Configuration conf, StateStoreService stateStore) {
+ Class<? extends ActiveNamenodeResolver> clazz = conf.getClass(
+ RBFConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS,
+ RBFConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS_DEFAULT,
+ ActiveNamenodeResolver.class);
+ return newInstance(conf, stateStore, StateStoreService.class, clazz);
+ }
+
+ /**
+ * Check if the given path is the child of parent path.
+ * @param path Path to be check.
+ * @param parent Parent path.
+ * @return True if parent path is parent entry for given path.
+ */
+ public static boolean isParentEntry(final String path, final String parent) {
+ if (!path.startsWith(parent)) {
+ return false;
+ }
+
+ if (path.equals(parent)) {
+ return true;
+ }
+
+ return path.charAt(parent.length()) == Path.SEPARATOR_CHAR
+ || parent.equals(Path.SEPARATOR);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSafemodeService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSafemodeService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSafemodeService.java
deleted file mode 100644
index 56aab0a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSafemodeService.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import static org.apache.hadoop.util.Time.now;
-
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Service to periodically check if the {@link org.apache.hadoop.hdfs.server.
- * federation.store.StateStoreService StateStoreService} cached information in
- * the {@link Router} is up to date. This is for performance and removes the
- * {@link org.apache.hadoop.hdfs.server.federation.store.StateStoreService
- * StateStoreService} from the critical path in common operations.
- */
-public class RouterSafemodeService extends PeriodicService {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(RouterSafemodeService.class);
-
- /** Router to manage safe mode. */
- private final Router router;
-
- /** Interval in ms to wait post startup before allowing RPC requests. */
- private long startupInterval;
- /** Interval in ms after which the State Store cache is too stale. */
- private long staleInterval;
- /** Start time in ms of this service. */
- private long startupTime;
-
- /** The time the Router enters safe mode in milliseconds. */
- private long enterSafeModeTime = now();
-
-
- /**
- * Create a new Cache update service.
- *
- * @param router Router containing the cache.
- */
- public RouterSafemodeService(Router router) {
- super(RouterSafemodeService.class.getSimpleName());
- this.router = router;
- }
-
- /**
- * Enter safe mode.
- */
- private void enter() {
- LOG.info("Entering safe mode");
- enterSafeModeTime = now();
- RouterRpcServer rpcServer = router.getRpcServer();
- rpcServer.setSafeMode(true);
- router.updateRouterState(RouterServiceState.SAFEMODE);
- }
-
- /**
- * Leave safe mode.
- */
- private void leave() {
- // Cache recently updated, leave safemode
- long timeInSafemode = now() - enterSafeModeTime;
- LOG.info("Leaving safe mode after {} milliseconds", timeInSafemode);
- RouterMetrics routerMetrics = router.getRouterMetrics();
- if (routerMetrics == null) {
- LOG.error("The Router metrics are not enabled");
- } else {
- routerMetrics.setSafeModeTime(timeInSafemode);
- }
- RouterRpcServer rpcServer = router.getRpcServer();
- rpcServer.setSafeMode(false);
- router.updateRouterState(RouterServiceState.RUNNING);
- }
-
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
-
- // Use same interval as cache update service
- this.setIntervalMs(conf.getTimeDuration(
- DFSConfigKeys.DFS_ROUTER_CACHE_TIME_TO_LIVE_MS,
- DFSConfigKeys.DFS_ROUTER_CACHE_TIME_TO_LIVE_MS_DEFAULT,
- TimeUnit.MILLISECONDS));
-
- this.startupInterval = conf.getTimeDuration(
- DFSConfigKeys.DFS_ROUTER_SAFEMODE_EXTENSION,
- DFSConfigKeys.DFS_ROUTER_SAFEMODE_EXTENSION_DEFAULT,
- TimeUnit.MILLISECONDS);
- LOG.info("Leave startup safe mode after {} ms", this.startupInterval);
-
- this.staleInterval = conf.getTimeDuration(
- DFSConfigKeys.DFS_ROUTER_SAFEMODE_EXPIRATION,
- DFSConfigKeys.DFS_ROUTER_SAFEMODE_EXPIRATION_DEFAULT,
- TimeUnit.MILLISECONDS);
- LOG.info("Enter safe mode after {} ms without reaching the State Store",
- this.staleInterval);
-
- this.startupTime = Time.now();
-
- // Initializing the RPC server in safe mode, it will disable it later
- enter();
-
- super.serviceInit(conf);
- }
-
- @Override
- public void periodicInvoke() {
- long now = Time.now();
- long delta = now - startupTime;
- if (delta < startupInterval) {
- LOG.info("Delaying safemode exit for {} milliseconds...",
- this.startupInterval - delta);
- return;
- }
- RouterRpcServer rpcServer = router.getRpcServer();
- StateStoreService stateStore = router.getStateStore();
- long cacheUpdateTime = stateStore.getCacheUpdateTime();
- boolean isCacheStale = (now - cacheUpdateTime) > this.staleInterval;
-
- // Always update to indicate our cache was updated
- if (isCacheStale) {
- if (!rpcServer.isInSafeMode()) {
- enter();
- }
- } else if (rpcServer.isInSafeMode()) {
- // Cache recently updated, leave safe mode
- leave();
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterServiceState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterServiceState.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterServiceState.java
deleted file mode 100644
index 3accbe9..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterServiceState.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-/**
- * States of the Router.
- */
-public enum RouterServiceState {
- UNINITIALIZED,
- INITIALIZING,
- SAFEMODE,
- RUNNING,
- STOPPING,
- SHUTDOWN,
- EXPIRED;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterStateManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterStateManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterStateManager.java
deleted file mode 100644
index 527600c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterStateManager.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeResponse;
-
-/**
- * Interface of managing the Router state.
- */
-public interface RouterStateManager {
- /**
- * Enter safe mode and change Router state to RouterServiceState#SAFEMODE.
- */
- EnterSafeModeResponse enterSafeMode(EnterSafeModeRequest request)
- throws IOException;
-
- /**
- * Leave safe mode and change Router state to RouterServiceState#RUNNING.
- */
- LeaveSafeModeResponse leaveSafeMode(LeaveSafeModeRequest request)
- throws IOException;
-
- /**
- * Verify if current Router state is safe mode.
- */
- GetSafeModeResponse getSafeMode(GetSafeModeRequest request)
- throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/package-info.java
deleted file mode 100644
index 327f39b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/package-info.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * The router package includes the core services for a HDFS federation router.
- * The {@link Router} acts as a transparent proxy in front of a cluster of
- * multiple NameNodes and nameservices. The {@link RouterRpcServer} exposes the
- * NameNode clientProtocol and is the primary contact point for DFS clients in a
- * federated cluster.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/CachedRecordStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/CachedRecordStore.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/CachedRecordStore.java
deleted file mode 100644
index cdd4449..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/CachedRecordStore.java
+++ /dev/null
@@ -1,242 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import org.apache.hadoop.hdfs.server.federation.metrics.StateStoreMetrics;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
-import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
-import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Record store that takes care of caching the records in memory.
- *
- * @param <R> Record to store by this interface.
- */
-public abstract class CachedRecordStore<R extends BaseRecord>
- extends RecordStore<R> implements StateStoreCache {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(CachedRecordStore.class);
-
-
- /** Prevent loading the cache more than once every 500 ms. */
- private static final long MIN_UPDATE_MS = 500;
-
-
- /** Cached entries. */
- private List<R> records = new ArrayList<>();
-
- /** Time stamp of the cached entries. */
- private long timestamp = -1;
-
- /** If the cache is initialized. */
- private boolean initialized = false;
-
- /** Last time the cache was updated. */
- private long lastUpdate = -1;
-
- /** Lock to access the memory cache. */
- private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
- private final Lock readLock = readWriteLock.readLock();
- private final Lock writeLock = readWriteLock.writeLock();
-
- /** If it should override the expired values when loading the cache. */
- private boolean override = false;
-
-
- /**
- * Create a new cached record store.
- *
- * @param clazz Class of the record to store.
- * @param driver State Store driver.
- */
- protected CachedRecordStore(Class<R> clazz, StateStoreDriver driver) {
- this(clazz, driver, false);
- }
-
- /**
- * Create a new cached record store.
- *
- * @param clazz Class of the record to store.
- * @param driver State Store driver.
- * @param over If the entries should be override if they expire
- */
- protected CachedRecordStore(
- Class<R> clazz, StateStoreDriver driver, boolean over) {
- super(clazz, driver);
-
- this.override = over;
- }
-
- /**
- * Check that the cache of the State Store information is available.
- *
- * @throws StateStoreUnavailableException If the cache is not initialized.
- */
- private void checkCacheAvailable() throws StateStoreUnavailableException {
- if (!this.initialized) {
- throw new StateStoreUnavailableException(
- "Cached State Store not initialized, " +
- getRecordClass().getSimpleName() + " records not valid");
- }
- }
-
- @Override
- public boolean loadCache(boolean force) throws IOException {
- // Prevent loading the cache too frequently
- if (force || isUpdateTime()) {
- List<R> newRecords = null;
- long t = -1;
- try {
- QueryResult<R> result = getDriver().get(getRecordClass());
- newRecords = result.getRecords();
- t = result.getTimestamp();
-
- // If we have any expired record, update the State Store
- if (this.override) {
- overrideExpiredRecords(result);
- }
- } catch (IOException e) {
- LOG.error("Cannot get \"{}\" records from the State Store",
- getRecordClass().getSimpleName());
- this.initialized = false;
- return false;
- }
-
- // Update cache atomically
- writeLock.lock();
- try {
- this.records.clear();
- this.records.addAll(newRecords);
- this.timestamp = t;
- this.initialized = true;
- } finally {
- writeLock.unlock();
- }
-
- // Update the metrics for the cache State Store size
- StateStoreMetrics metrics = getDriver().getMetrics();
- if (metrics != null) {
- String recordName = getRecordClass().getSimpleName();
- metrics.setCacheSize(recordName, this.records.size());
- }
-
- lastUpdate = Time.monotonicNow();
- }
- return true;
- }
-
- /**
- * Check if it's time to update the cache. Update it it was never updated.
- *
- * @return If it's time to update this cache.
- */
- private boolean isUpdateTime() {
- return Time.monotonicNow() - lastUpdate > MIN_UPDATE_MS;
- }
-
- /**
- * Updates the state store with any record overrides we detected, such as an
- * expired state.
- *
- * @param query RecordQueryResult containing the data to be inspected.
- * @throws IOException
- */
- public void overrideExpiredRecords(QueryResult<R> query) throws IOException {
- List<R> commitRecords = new ArrayList<>();
- List<R> newRecords = query.getRecords();
- long currentDriverTime = query.getTimestamp();
- if (newRecords == null || currentDriverTime <= 0) {
- LOG.error("Cannot check overrides for record");
- return;
- }
- for (R record : newRecords) {
- if (record.checkExpired(currentDriverTime)) {
- String recordName = StateStoreUtils.getRecordName(record.getClass());
- LOG.info("Override State Store record {}: {}", recordName, record);
- commitRecords.add(record);
- }
- }
- if (commitRecords.size() > 0) {
- getDriver().putAll(commitRecords, true, false);
- }
- }
-
- /**
- * Updates the state store with any record overrides we detected, such as an
- * expired state.
- *
- * @param record Record record to be updated.
- * @throws IOException
- */
- public void overrideExpiredRecord(R record) throws IOException {
- List<R> newRecords = Collections.singletonList(record);
- long time = getDriver().getTime();
- QueryResult<R> query = new QueryResult<>(newRecords, time);
- overrideExpiredRecords(query);
- }
-
- /**
- * Get all the cached records.
- *
- * @return Copy of the cached records.
- * @throws StateStoreUnavailableException If the State store is not available.
- */
- public List<R> getCachedRecords() throws StateStoreUnavailableException {
- checkCacheAvailable();
-
- List<R> ret = new LinkedList<R>();
- this.readLock.lock();
- try {
- ret.addAll(this.records);
- } finally {
- this.readLock.unlock();
- }
- return ret;
- }
-
- /**
- * Get all the cached records and the time stamp of the cache.
- *
- * @return Copy of the cached records and the time stamp.
- * @throws StateStoreUnavailableException If the State store is not available.
- */
- protected QueryResult<R> getCachedRecordsAndTimeStamp()
- throws StateStoreUnavailableException {
- checkCacheAvailable();
-
- this.readLock.lock();
- try {
- return new QueryResult<R>(this.records, this.timestamp);
- } finally {
- this.readLock.unlock();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MembershipStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MembershipStore.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MembershipStore.java
deleted file mode 100644
index 3e8ba6b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MembershipStore.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateNamenodeRegistrationRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateNamenodeRegistrationResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
-
-/**
- * Management API for NameNode registrations stored in
- * {@link org.apache.hadoop.hdfs.server.federation.store.records.MembershipState
- * MembershipState} records. The {@link org.apache.hadoop.hdfs.server.
- * federation.router.RouterHeartbeatService RouterHeartbeatService} periodically
- * polls each NN to update the NameNode metadata(addresses, operational) and HA
- * state(active, standby). Each NameNode may be polled by multiple
- * {@link org.apache.hadoop.hdfs.server.federation.router.Router Router}
- * instances.
- * <p>
- * Once fetched from the
- * {@link org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver
- * StateStoreDriver}, NameNode registrations are cached until the next query.
- * The fetched registration data is aggregated using a quorum to determine the
- * best/most accurate state for each NameNode. The cache is periodically updated
- * by the @{link StateStoreCacheUpdateService}.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public abstract class MembershipStore
- extends CachedRecordStore<MembershipState> {
-
- protected MembershipStore(StateStoreDriver driver) {
- super(MembershipState.class, driver, true);
- }
-
- /**
- * Inserts or updates a namenode membership entry into the table.
- *
- * @param request Fully populated NamenodeHeartbeatRequest request.
- * @return True if successful, false otherwise.
- * @throws StateStoreUnavailableException Throws exception if the data store
- * is not initialized.
- * @throws IOException if the data store could not be queried or the query is
- * invalid.
- */
- public abstract NamenodeHeartbeatResponse namenodeHeartbeat(
- NamenodeHeartbeatRequest request) throws IOException;
-
- /**
- * Queries for a single cached registration entry matching the given
- * parameters. Possible keys are the names of data structure elements Possible
- * values are matching SQL "LIKE" targets.
- *
- * @param request Fully populated GetNamenodeRegistrationsRequest request.
- * @return Single matching FederationMembershipStateEntry or null if not found
- * or more than one entry matches.
- * @throws StateStoreUnavailableException Throws exception if the data store
- * is not initialized.
- * @throws IOException if the data store could not be queried or the query is
- * invalid.
- */
- public abstract GetNamenodeRegistrationsResponse getNamenodeRegistrations(
- GetNamenodeRegistrationsRequest request) throws IOException;
-
- /**
- * Get the expired registrations from the registration cache.
- *
- * @return Expired registrations or zero-length list if none are found.
- * @throws StateStoreUnavailableException Throws exception if the data store
- * is not initialized.
- * @throws IOException if the data store could not be queried or the query is
- * invalid.
- */
- public abstract GetNamenodeRegistrationsResponse
- getExpiredNamenodeRegistrations(GetNamenodeRegistrationsRequest request)
- throws IOException;
-
- /**
- * Retrieves a list of registered nameservices and their associated info.
- *
- * @param request
- * @return Collection of information for each registered nameservice.
- * @throws IOException if the data store could not be queried or the query is
- * invalid.
- */
- public abstract GetNamespaceInfoResponse getNamespaceInfo(
- GetNamespaceInfoRequest request) throws IOException;
-
- /**
- * Overrides a cached namenode state with an updated state.
- *
- * @param request Fully populated OverrideNamenodeRegistrationRequest request.
- * @return OverrideNamenodeRegistrationResponse
- * @throws StateStoreUnavailableException if the data store is not
- * initialized.
- * @throws IOException if the data store could not be queried or the query is
- * invalid.
- */
- public abstract UpdateNamenodeRegistrationResponse updateNamenodeRegistration(
- UpdateNamenodeRegistrationRequest request) throws IOException;
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MountTableStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MountTableStore.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MountTableStore.java
deleted file mode 100644
index b439659..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MountTableStore.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-
-/**
- * Management API for the HDFS mount table information stored in
- * {@link org.apache.hadoop.hdfs.server.federation.store.records.MountTable
- * MountTable} records. The mount table contains entries that map a particular
- * global namespace path one or more HDFS nameservices (NN) + target path. It is
- * possible to map mount locations for root folders, directories or individual
- * files.
- * <p>
- * Once fetched from the
- * {@link org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver
- * StateStoreDriver}, MountTable records are cached in a tree for faster access.
- * Each path in the global namespace is mapped to a nameserivce ID and local
- * path upon request. The cache is periodically updated by the @{link
- * StateStoreCacheUpdateService}.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public abstract class MountTableStore extends CachedRecordStore<MountTable>
- implements MountTableManager {
-
- public MountTableStore(StateStoreDriver driver) {
- super(MountTable.class, driver);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RecordStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RecordStore.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RecordStore.java
deleted file mode 100644
index 53a8b82..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RecordStore.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store;
-
-import java.lang.reflect.Constructor;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
-import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
-
-/**
- * Store records in the State Store. Subclasses provide interfaces to operate on
- * those records.
- *
- * @param <R> Record to store by this interface.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public abstract class RecordStore<R extends BaseRecord> {
-
- private static final Log LOG = LogFactory.getLog(RecordStore.class);
-
-
- /** Class of the record stored in this State Store. */
- private final Class<R> recordClass;
-
- /** State store driver backed by persistent storage. */
- private final StateStoreDriver driver;
-
-
- /**
- * Create a new store for records.
- *
- * @param clazz Class of the record to store.
- * @param stateStoreDriver Driver for the State Store.
- */
- protected RecordStore(Class<R> clazz, StateStoreDriver stateStoreDriver) {
- this.recordClass = clazz;
- this.driver = stateStoreDriver;
- }
-
- /**
- * Report a required record to the data store. The data store uses this to
- * create/maintain storage for the record.
- *
- * @return The class of the required record or null if no record is required
- * for this interface.
- */
- public Class<R> getRecordClass() {
- return this.recordClass;
- }
-
- /**
- * Get the State Store driver.
- *
- * @return State Store driver.
- */
- protected StateStoreDriver getDriver() {
- return this.driver;
- }
-
- /**
- * Build a state store API implementation interface.
- *
- * @param clazz The specific interface implementation to create
- * @param driver The {@link StateStoreDriver} implementation in use.
- * @return An initialized instance of the specified state store API
- * implementation.
- */
- public static <T extends RecordStore<?>> T newInstance(
- final Class<T> clazz, final StateStoreDriver driver) {
-
- try {
- Constructor<T> constructor = clazz.getConstructor(StateStoreDriver.class);
- T recordStore = constructor.newInstance(driver);
- return recordStore;
- } catch (Exception e) {
- LOG.error("Cannot create new instance for " + clazz, e);
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RouterStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RouterStore.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RouterStore.java
deleted file mode 100644
index c6a0dad..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RouterStore.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationsRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationsResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
-
-/**
- * Management API for
- * {@link org.apache.hadoop.hdfs.server.federation.store.records.RouterState
- * RouterState} records in the state store. Accesses the data store via the
- * {@link org.apache.hadoop.hdfs.server.federation.store.driver.
- * StateStoreDriver StateStoreDriver} interface. No data is cached.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public abstract class RouterStore extends CachedRecordStore<RouterState> {
-
- public RouterStore(StateStoreDriver driver) {
- super(RouterState.class, driver, true);
- }
-
- /**
- * Fetches the current router state object.
- *
- * @param request Fully populated request object.
- * @return The matching router record or null if none exists.
- * @throws IOException Throws exception if unable to query the data store or
- * if more than one matching record is found.
- */
- public abstract GetRouterRegistrationResponse getRouterRegistration(
- GetRouterRegistrationRequest request) throws IOException;
-
- /**
- * Fetches all router status objects.
- *
- * @param request Fully populated request object.
- * @return List of Router records present in the data store.
- * @throws IOException Throws exception if unable to query the data store
- */
- public abstract GetRouterRegistrationsResponse getRouterRegistrations(
- GetRouterRegistrationsRequest request) throws IOException;
-
- /**
- * Update the state of this router in the State Store.
- *
- * @param request Fully populated request object.
- * @return True if the update was successfully recorded, false otherwise.
- * @throws IOException Throws exception if unable to query the data store
- */
- public abstract RouterHeartbeatResponse routerHeartbeat(
- RouterHeartbeatRequest request) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreCache.java
deleted file mode 100644
index 83fc501..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreCache.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store;
-
-import java.io.IOException;
-
-/**
- * Interface for a cached copy of the State Store.
- */
-public interface StateStoreCache {
-
- /**
- * Load the cache from the State Store. Called by the cache update service
- * when the data has been reloaded.
- *
- * @param force If we force the load.
- * @return If the cache was loaded successfully.
- * @throws IOException If there was an error loading the cache.
- */
- boolean loadCache(boolean force) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreCacheUpdateService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreCacheUpdateService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreCacheUpdateService.java
deleted file mode 100644
index 9bcbc1e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreCacheUpdateService.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store;
-
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.server.federation.router.PeriodicService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Service to periodically update the {@link StateStoreService}
- * cached information in the
- * {@link org.apache.hadoop.hdfs.server.federation.router.Router Router}.
- * This is for performance and removes the State Store from the critical path
- * in common operations.
- */
-public class StateStoreCacheUpdateService extends PeriodicService {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(StateStoreCacheUpdateService.class);
-
- /** The service that manages the State Store connection. */
- private final StateStoreService stateStore;
-
-
- /**
- * Create a new Cache update service.
- *
- * @param stateStore Implementation of the state store
- */
- public StateStoreCacheUpdateService(StateStoreService stateStore) {
- super(StateStoreCacheUpdateService.class.getSimpleName());
- this.stateStore = stateStore;
- }
-
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
-
- this.setIntervalMs(conf.getTimeDuration(
- DFSConfigKeys.DFS_ROUTER_CACHE_TIME_TO_LIVE_MS,
- DFSConfigKeys.DFS_ROUTER_CACHE_TIME_TO_LIVE_MS_DEFAULT,
- TimeUnit.MILLISECONDS));
-
- super.serviceInit(conf);
- }
-
- @Override
- public void periodicInvoke() {
- LOG.debug("Updating State Store cache");
- stateStore.refreshCaches();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreConnectionMonitorService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreConnectionMonitorService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreConnectionMonitorService.java
deleted file mode 100644
index 4d279c5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreConnectionMonitorService.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.server.federation.router.PeriodicService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Service to periodically monitor the connection of the StateStore
- * {@link StateStoreService} data store and to re-open the connection
- * to the data store if required.
- */
-public class StateStoreConnectionMonitorService extends PeriodicService {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(StateStoreConnectionMonitorService.class);
-
- /** Service that maintains the State Store connection. */
- private final StateStoreService stateStore;
-
-
- /**
- * Create a new service to monitor the connectivity of the state store driver.
- *
- * @param store Instance of the state store to be monitored.
- */
- public StateStoreConnectionMonitorService(StateStoreService store) {
- super(StateStoreConnectionMonitorService.class.getSimpleName());
- this.stateStore = store;
- }
-
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
- this.setIntervalMs(conf.getLong(
- DFSConfigKeys.FEDERATION_STORE_CONNECTION_TEST_MS,
- DFSConfigKeys.FEDERATION_STORE_CONNECTION_TEST_MS_DEFAULT));
-
- super.serviceInit(conf);
- }
-
- @Override
- public void periodicInvoke() {
- LOG.debug("Checking state store connection");
- if (!stateStore.isDriverReady()) {
- LOG.info("Attempting to open state store driver.");
- stateStore.loadDriver();
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
deleted file mode 100644
index aa730ae..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
+++ /dev/null
@@ -1,450 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import javax.management.NotCompliantMBeanException;
-import javax.management.ObjectName;
-import javax.management.StandardMBean;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.server.federation.metrics.StateStoreMBean;
-import org.apache.hadoop.hdfs.server.federation.metrics.StateStoreMetrics;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
-import org.apache.hadoop.hdfs.server.federation.store.impl.MembershipStoreImpl;
-import org.apache.hadoop.hdfs.server.federation.store.impl.MountTableStoreImpl;
-import org.apache.hadoop.hdfs.server.federation.store.impl.RouterStoreImpl;
-import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
-import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
-import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
-import org.apache.hadoop.metrics2.MetricsException;
-import org.apache.hadoop.metrics2.util.MBeans;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.annotations.VisibleForTesting;
-
-/**
- * A service to initialize a
- * {@link org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver
- * StateStoreDriver} and maintain the connection to the data store. There are
- * multiple state store driver connections supported:
- * <ul>
- * <li>File
- * {@link org.apache.hadoop.hdfs.server.federation.store.driver.impl.
- * StateStoreFileImpl StateStoreFileImpl}
- * <li>ZooKeeper
- * {@link org.apache.hadoop.hdfs.server.federation.store.driver.impl.
- * StateStoreZooKeeperImpl StateStoreZooKeeperImpl}
- * </ul>
- * <p>
- * The service also supports the dynamic registration of record stores like:
- * <ul>
- * <li>{@link MembershipStore}: state of the Namenodes in the
- * federation.
- * <li>{@link MountTableStore}: Mount table between to subclusters.
- * See {@link org.apache.hadoop.fs.viewfs.ViewFs ViewFs}.
- * <li>{@link RebalancerStore}: Log of the rebalancing operations.
- * <li>{@link RouterStore}: Router state in the federation.
- * <li>{@link TokenStore}: Tokens in the federation.
- * </ul>
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class StateStoreService extends CompositeService {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(StateStoreService.class);
-
-
- /** State Store configuration. */
- private Configuration conf;
-
- /** Identifier for the service. */
- private String identifier;
-
- /** Driver for the back end connection. */
- private StateStoreDriver driver;
-
- /** Service to maintain data store connection. */
- private StateStoreConnectionMonitorService monitorService;
-
- /** StateStore metrics. */
- private StateStoreMetrics metrics;
-
- /** Supported record stores. */
- private final Map<
- Class<? extends BaseRecord>, RecordStore<? extends BaseRecord>>
- recordStores;
-
- /** Service to maintain State Store caches. */
- private StateStoreCacheUpdateService cacheUpdater;
- /** Time the cache was last successfully updated. */
- private long cacheLastUpdateTime;
- /** List of internal caches to update. */
- private final List<StateStoreCache> cachesToUpdateInternal;
- /** List of external caches to update. */
- private final List<StateStoreCache> cachesToUpdateExternal;
-
-
- public StateStoreService() {
- super(StateStoreService.class.getName());
-
- // Records and stores supported by this implementation
- this.recordStores = new HashMap<>();
-
- // Caches to maintain
- this.cachesToUpdateInternal = new ArrayList<>();
- this.cachesToUpdateExternal = new ArrayList<>();
- }
-
- /**
- * Initialize the State Store and the connection to the backend.
- *
- * @param config Configuration for the State Store.
- * @throws IOException
- */
- @Override
- protected void serviceInit(Configuration config) throws Exception {
- this.conf = config;
-
- // Create implementation of State Store
- Class<? extends StateStoreDriver> driverClass = this.conf.getClass(
- DFSConfigKeys.FEDERATION_STORE_DRIVER_CLASS,
- DFSConfigKeys.FEDERATION_STORE_DRIVER_CLASS_DEFAULT,
- StateStoreDriver.class);
- this.driver = ReflectionUtils.newInstance(driverClass, this.conf);
-
- if (this.driver == null) {
- throw new IOException("Cannot create driver for the State Store");
- }
-
- // Add supported record stores
- addRecordStore(MembershipStoreImpl.class);
- addRecordStore(MountTableStoreImpl.class);
- addRecordStore(RouterStoreImpl.class);
-
- // Check the connection to the State Store periodically
- this.monitorService = new StateStoreConnectionMonitorService(this);
- this.addService(monitorService);
-
- // Set expirations intervals for each record
- MembershipState.setExpirationMs(conf.getLong(
- DFSConfigKeys.FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS,
- DFSConfigKeys.FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS_DEFAULT));
-
- RouterState.setExpirationMs(conf.getTimeDuration(
- DFSConfigKeys.FEDERATION_STORE_ROUTER_EXPIRATION_MS,
- DFSConfigKeys.FEDERATION_STORE_ROUTER_EXPIRATION_MS_DEFAULT,
- TimeUnit.MILLISECONDS));
-
- // Cache update service
- this.cacheUpdater = new StateStoreCacheUpdateService(this);
- addService(this.cacheUpdater);
-
- // Create metrics for the State Store
- this.metrics = StateStoreMetrics.create(conf);
-
- // Adding JMX interface
- try {
- StandardMBean bean = new StandardMBean(metrics, StateStoreMBean.class);
- ObjectName registeredObject =
- MBeans.register("Router", "StateStore", bean);
- LOG.info("Registered StateStoreMBean: {}", registeredObject);
- } catch (NotCompliantMBeanException e) {
- throw new RuntimeException("Bad StateStoreMBean setup", e);
- } catch (MetricsException e) {
- LOG.info("Failed to register State Store bean {}", e.getMessage());
- }
-
- super.serviceInit(this.conf);
- }
-
- @Override
- protected void serviceStart() throws Exception {
- loadDriver();
- super.serviceStart();
- }
-
- @Override
- protected void serviceStop() throws Exception {
- closeDriver();
-
- if (metrics != null) {
- metrics.shutdown();
- metrics = null;
- }
-
- super.serviceStop();
- }
-
- /**
- * Add a record store to the State Store. It includes adding the store, the
- * supported record and the cache management.
- *
- * @param clazz Class of the record store to track.
- * @return New record store.
- * @throws ReflectiveOperationException
- */
- private <T extends RecordStore<?>> void addRecordStore(
- final Class<T> clazz) throws ReflectiveOperationException {
-
- assert this.getServiceState() == STATE.INITED :
- "Cannot add record to the State Store once started";
-
- T recordStore = RecordStore.newInstance(clazz, this.getDriver());
- Class<? extends BaseRecord> recordClass = recordStore.getRecordClass();
- this.recordStores.put(recordClass, recordStore);
-
- // Subscribe for cache updates
- if (recordStore instanceof StateStoreCache) {
- StateStoreCache cachedRecordStore = (StateStoreCache) recordStore;
- this.cachesToUpdateInternal.add(cachedRecordStore);
- }
- }
-
- /**
- * Get the record store in this State Store for a given interface.
- *
- * @param recordStoreClass Class of the record store.
- * @return Registered record store or null if not found.
- */
- public <T extends RecordStore<?>> T getRegisteredRecordStore(
- final Class<T> recordStoreClass) {
- for (RecordStore<? extends BaseRecord> recordStore :
- this.recordStores.values()) {
- if (recordStoreClass.isInstance(recordStore)) {
- @SuppressWarnings("unchecked")
- T recordStoreChecked = (T) recordStore;
- return recordStoreChecked;
- }
- }
- return null;
- }
-
- /**
- * List of records supported by this State Store.
- *
- * @return List of supported record classes.
- */
- public Collection<Class<? extends BaseRecord>> getSupportedRecords() {
- return this.recordStores.keySet();
- }
-
- /**
- * Load the State Store driver. If successful, refresh cached data tables.
- */
- public void loadDriver() {
- synchronized (this.driver) {
- if (!isDriverReady()) {
- String driverName = this.driver.getClass().getSimpleName();
- if (this.driver.init(
- conf, getIdentifier(), getSupportedRecords(), metrics)) {
- LOG.info("Connection to the State Store driver {} is open and ready",
- driverName);
- this.refreshCaches();
- } else {
- LOG.error("Cannot initialize State Store driver {}", driverName);
- }
- }
- }
- }
-
- /**
- * Check if the driver is ready to be used.
- *
- * @return If the driver is ready.
- */
- public boolean isDriverReady() {
- return this.driver.isDriverReady();
- }
-
- /**
- * Manually shuts down the driver.
- *
- * @throws Exception If the driver cannot be closed.
- */
- @VisibleForTesting
- public void closeDriver() throws Exception {
- if (this.driver != null) {
- this.driver.close();
- }
- }
-
- /**
- * Get the state store driver.
- *
- * @return State store driver.
- */
- public StateStoreDriver getDriver() {
- return this.driver;
- }
-
- /**
- * Fetch a unique identifier for this state store instance. Typically it is
- * the address of the router.
- *
- * @return Unique identifier for this store.
- */
- public String getIdentifier() {
- return this.identifier;
- }
-
- /**
- * Set a unique synchronization identifier for this store.
- *
- * @param id Unique identifier, typically the router's RPC address.
- */
- public void setIdentifier(String id) {
- this.identifier = id;
- }
-
- //
- // Cached state store data
- //
- /**
- * The last time the state store cache was fully updated.
- *
- * @return Timestamp.
- */
- public long getCacheUpdateTime() {
- return this.cacheLastUpdateTime;
- }
-
- /**
- * Stops the cache update service.
- */
- @VisibleForTesting
- public void stopCacheUpdateService() {
- if (this.cacheUpdater != null) {
- this.cacheUpdater.stop();
- removeService(this.cacheUpdater);
- this.cacheUpdater = null;
- }
- }
-
- /**
- * Register a cached record store for automatic periodic cache updates.
- *
- * @param client Client to the state store.
- */
- public void registerCacheExternal(StateStoreCache client) {
- this.cachesToUpdateExternal.add(client);
- }
-
- /**
- * Refresh the cache with information from the State Store. Called
- * periodically by the CacheUpdateService to maintain data caches and
- * versions.
- */
- public void refreshCaches() {
- refreshCaches(false);
- }
-
- /**
- * Refresh the cache with information from the State Store. Called
- * periodically by the CacheUpdateService to maintain data caches and
- * versions.
- * @param force If we force the refresh.
- */
- public void refreshCaches(boolean force) {
- boolean success = true;
- if (isDriverReady()) {
- List<StateStoreCache> cachesToUpdate = new LinkedList<>();
- cachesToUpdate.addAll(cachesToUpdateInternal);
- cachesToUpdate.addAll(cachesToUpdateExternal);
- for (StateStoreCache cachedStore : cachesToUpdate) {
- String cacheName = cachedStore.getClass().getSimpleName();
- boolean result = false;
- try {
- result = cachedStore.loadCache(force);
- } catch (IOException e) {
- LOG.error("Error updating cache for {}", cacheName, e);
- result = false;
- }
- if (!result) {
- success = false;
- LOG.error("Cache update failed for cache {}", cacheName);
- }
- }
- } else {
- success = false;
- LOG.info("Skipping State Store cache update, driver is not ready.");
- }
- if (success) {
- // Uses local time, not driver time.
- this.cacheLastUpdateTime = Time.now();
- }
- }
-
- /**
- * Update the cache for a specific record store.
- *
- * @param clazz Class of the record store.
- * @return If the cached was loaded.
- * @throws IOException if the cache update failed.
- */
- public boolean loadCache(final Class<?> clazz) throws IOException {
- return loadCache(clazz, false);
- }
-
- /**
- * Update the cache for a specific record store.
- *
- * @param clazz Class of the record store.
- * @param force Force the update ignoring cached periods.
- * @return If the cached was loaded.
- * @throws IOException if the cache update failed.
- */
- public boolean loadCache(Class<?> clazz, boolean force) throws IOException {
- List<StateStoreCache> cachesToUpdate =
- new LinkedList<StateStoreCache>();
- cachesToUpdate.addAll(this.cachesToUpdateInternal);
- cachesToUpdate.addAll(this.cachesToUpdateExternal);
- for (StateStoreCache cachedStore : cachesToUpdate) {
- if (clazz.isInstance(cachedStore)) {
- return cachedStore.loadCache(force);
- }
- }
- throw new IOException("Registered cache was not found for " + clazz);
- }
-
- /**
- * Get the metrics for the State Store.
- *
- * @return State Store metrics.
- */
- public StateStoreMetrics getMetrics() {
- return metrics;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUnavailableException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUnavailableException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUnavailableException.java
deleted file mode 100644
index 4e6f8c8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUnavailableException.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store;
-
-import java.io.IOException;
-
-/**
- * Thrown when the state store is not reachable or available. Cached APIs and
- * queries may succeed. Client should retry again later.
- */
-public class StateStoreUnavailableException extends IOException {
-
- private static final long serialVersionUID = 1L;
-
- public StateStoreUnavailableException(String msg) {
- super(msg);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUtils.java
deleted file mode 100644
index 0a36619..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUtils.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
-import org.apache.hadoop.hdfs.server.federation.store.records.Query;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Set of utility functions used to work with the State Store.
- */
-public final class StateStoreUtils {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(StateStoreUtils.class);
-
-
- private StateStoreUtils() {
- // Utility class
- }
-
- /**
- * Get the base class for a record class. If we get an implementation of a
- * record we will return the real parent record class.
- *
- * @param clazz Class of the data record to check.
- * @return Base class for the record.
- */
- @SuppressWarnings("unchecked")
- public static <T extends BaseRecord>
- Class<? extends BaseRecord> getRecordClass(final Class<T> clazz) {
-
- // We ignore the Impl classes and go to the super class
- Class<? extends BaseRecord> actualClazz = clazz;
- while (actualClazz.getSimpleName().endsWith("Impl")) {
- actualClazz = (Class<? extends BaseRecord>) actualClazz.getSuperclass();
- }
-
- // Check if we went too far
- if (actualClazz.equals(BaseRecord.class)) {
- LOG.error("We went too far ({}) with {}", actualClazz, clazz);
- actualClazz = clazz;
- }
- return actualClazz;
- }
-
- /**
- * Get the base class for a record. If we get an implementation of a record we
- * will return the real parent record class.
- *
- * @param record Record to check its main class.
- * @return Base class for the record.
- */
- public static <T extends BaseRecord>
- Class<? extends BaseRecord> getRecordClass(final T record) {
- return getRecordClass(record.getClass());
- }
-
- /**
- * Get the base class name for a record. If we get an implementation of a
- * record we will return the real parent record class.
- *
- * @param clazz Class of the data record to check.
- * @return Name of the base class for the record.
- */
- public static <T extends BaseRecord> String getRecordName(
- final Class<T> clazz) {
- return getRecordClass(clazz).getSimpleName();
- }
-
- /**
- * Filters a list of records to find all records matching the query.
- *
- * @param query Map of field names and objects to use to filter results.
- * @param records List of data records to filter.
- * @return List of all records matching the query (or empty list if none
- * match), null if the data set could not be filtered.
- */
- public static <T extends BaseRecord> List<T> filterMultiple(
- final Query<T> query, final Iterable<T> records) {
-
- List<T> matchingList = new ArrayList<>();
- for (T record : records) {
- if (query.matches(record)) {
- matchingList.add(record);
- }
- }
- return matchingList;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java
deleted file mode 100644
index d595a97..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java
+++ /dev/null
@@ -1,203 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.driver;
-
-import java.net.InetAddress;
-import java.util.Collection;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.server.federation.metrics.StateStoreMetrics;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreUnavailableException;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils;
-import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Driver class for an implementation of a {@link StateStoreService}
- * provider. Driver implementations will extend this class and implement some of
- * the default methods.
- */
-public abstract class StateStoreDriver implements StateStoreRecordOperations {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(StateStoreDriver.class);
-
-
- /** State Store configuration. */
- private Configuration conf;
-
- /** Identifier for the driver. */
- private String identifier;
-
- /** State Store metrics. */
- private StateStoreMetrics metrics;
-
-
- /**
- * Initialize the state store connection.
- *
- * @param config Configuration for the driver.
- * @param id Identifier for the driver.
- * @param records Records that are supported.
- * @return If initialized and ready, false if failed to initialize driver.
- */
- public boolean init(final Configuration config, final String id,
- final Collection<Class<? extends BaseRecord>> records,
- final StateStoreMetrics stateStoreMetrics) {
-
- this.conf = config;
- this.identifier = id;
- this.metrics = stateStoreMetrics;
-
- if (this.identifier == null) {
- LOG.warn("The identifier for the State Store connection is not set");
- }
-
- boolean success = initDriver();
- if (!success) {
- LOG.error("Cannot initialize driver for {}", getDriverName());
- return false;
- }
-
- for (Class<? extends BaseRecord> cls : records) {
- String recordString = StateStoreUtils.getRecordName(cls);
- if (!initRecordStorage(recordString, cls)) {
- LOG.error("Cannot initialize record store for {}", cls.getSimpleName());
- return false;
- }
- }
- return true;
- }
-
- /**
- * Get the State Store configuration.
- *
- * @return Configuration for the State Store.
- */
- protected Configuration getConf() {
- return this.conf;
- }
-
- /**
- * Gets a unique identifier for the running task/process. Typically the
- * router address.
- *
- * @return Unique identifier for the running task.
- */
- public String getIdentifier() {
- return this.identifier;
- }
-
- /**
- * Get the metrics for the State Store.
- *
- * @return State Store metrics.
- */
- public StateStoreMetrics getMetrics() {
- return this.metrics;
- }
-
- /**
- * Prepare the driver to access data storage.
- *
- * @return True if the driver was successfully initialized. If false is
- * returned, the state store will periodically attempt to
- * re-initialize the driver and the router will remain in safe mode
- * until the driver is initialized.
- */
- public abstract boolean initDriver();
-
- /**
- * Initialize storage for a single record class.
- *
- * @param className String reference of the record class to initialize,
- * used to construct paths and file names for the record.
- * Determined by configuration settings for the specific
- * driver.
- * @param clazz Record type corresponding to the provided name.
- * @return True if successful, false otherwise.
- */
- public abstract <T extends BaseRecord> boolean initRecordStorage(
- String className, Class<T> clazz);
-
- /**
- * Check if the driver is currently running and the data store connection is
- * valid.
- *
- * @return True if the driver is initialized and the data store is ready.
- */
- public abstract boolean isDriverReady();
-
- /**
- * Check if the driver is ready to be used and throw an exception otherwise.
- *
- * @throws StateStoreUnavailableException If the driver is not ready.
- */
- public void verifyDriverReady() throws StateStoreUnavailableException {
- if (!isDriverReady()) {
- String driverName = getDriverName();
- String hostname = getHostname();
- throw new StateStoreUnavailableException("State Store driver " +
- driverName + " in " + hostname + " is not ready.");
- }
- }
-
- /**
- * Close the State Store driver connection.
- */
- public abstract void close() throws Exception;
-
- /**
- * Returns the current time synchronization from the underlying store.
- * Override for stores that supply a current date. The data store driver is
- * responsible for maintaining the official synchronization time/date for all
- * distributed components.
- *
- * @return Current time stamp, used for all synchronization dates.
- */
- public long getTime() {
- return Time.now();
- }
-
- /**
- * Get the name of the driver implementation for debugging.
- *
- * @return Name of the driver implementation.
- */
- private String getDriverName() {
- return this.getClass().getSimpleName();
- }
-
- /**
- * Get the host name of the machine running the driver for debugging.
- *
- * @return Host name of the machine running the driver.
- */
- private String getHostname() {
- String hostname = "Unknown";
- try {
- hostname = InetAddress.getLocalHost().getHostName();
- } catch (Exception e) {
- LOG.error("Cannot get local address", e);
- }
- return hostname;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
deleted file mode 100644
index 443d46e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.driver;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
-import org.apache.hadoop.hdfs.server.federation.store.records.Query;
-import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
-import org.apache.hadoop.io.retry.AtMostOnce;
-import org.apache.hadoop.io.retry.Idempotent;
-
-/**
- * Operations for a driver to manage records in the State Store.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public interface StateStoreRecordOperations {
-
- /**
- * Get all records of the requested record class from the data store. To use
- * the default implementations in this class, getAll must return new instances
- * of the records on each call. It is recommended to override the default
- * implementations for better performance.
- *
- * @param clazz Class of record to fetch.
- * @return List of all records that match the clazz.
- * @throws IOException Throws exception if unable to query the data store.
- */
- @Idempotent
- <T extends BaseRecord> QueryResult<T> get(Class<T> clazz) throws IOException;
-
- /**
- * Get a single record from the store that matches the query.
- *
- * @param clazz Class of record to fetch.
- * @param query Query to filter results.
- * @return A single record matching the query. Null if there are no matching
- * records or more than one matching record in the store.
- * @throws IOException If multiple records match or if the data store cannot
- * be queried.
- */
- @Idempotent
- <T extends BaseRecord> T get(Class<T> clazz, Query<T> query)
- throws IOException;
-
- /**
- * Get multiple records from the store that match a query. This method
- * assumes the underlying driver does not support filtering. If the driver
- * supports filtering it should overwrite this method.
- *
- * @param clazz Class of record to fetch.
- * @param query Query to filter results.
- * @return Records of type clazz that match the query or empty list if none
- * are found.
- * @throws IOException Throws exception if unable to query the data store.
- */
- @Idempotent
- <T extends BaseRecord> List<T> getMultiple(
- Class<T> clazz, Query<T> query) throws IOException;
-
- /**
- * Creates a single record. Optionally updates an existing record with same
- * primary key.
- *
- * @param record The record to insert or update.
- * @param allowUpdate True if update of exiting record is allowed.
- * @param errorIfExists True if an error should be returned when inserting
- * an existing record. Only used if allowUpdate = false.
- * @return True if the operation was successful.
- *
- * @throws IOException Throws exception if unable to query the data store.
- */
- @AtMostOnce
- <T extends BaseRecord> boolean put(
- T record, boolean allowUpdate, boolean errorIfExists) throws IOException;
-
- /**
- * Creates multiple records. Optionally updates existing records that have
- * the same primary key.
- *
- * @param records List of data records to update or create. All records must
- * be of class clazz.
- * @param clazz Record class of records.
- * @param allowUpdate True if update of exiting record is allowed.
- * @param errorIfExists True if an error should be returned when inserting
- * an existing record. Only used if allowUpdate = false.
- * @return true if all operations were successful.
- *
- * @throws IOException Throws exception if unable to query the data store.
- */
- @AtMostOnce
- <T extends BaseRecord> boolean putAll(
- List<T> records, boolean allowUpdate, boolean errorIfExists)
- throws IOException;
-
- /**
- * Remove a single record.
- *
- * @param record Record to be removed.
- * @return true If the record was successfully removed. False if the record
- * could not be removed or not stored.
- * @throws IOException Throws exception if unable to query the data store.
- */
- @AtMostOnce
- <T extends BaseRecord> boolean remove(T record) throws IOException;
-
- /**
- * Remove all records of this class from the store.
- *
- * @param clazz Class of records to remove.
- * @return True if successful.
- * @throws IOException Throws exception if unable to query the data store.
- */
- @AtMostOnce
- <T extends BaseRecord> boolean removeAll(Class<T> clazz) throws IOException;
-
- /**
- * Remove multiple records of a specific class that match a query. Requires
- * the getAll implementation to fetch fresh records on each call.
- *
- * @param query Query to filter what to remove.
- * @return The number of records removed.
- * @throws IOException Throws exception if unable to query the data store.
- */
- @AtMostOnce
- <T extends BaseRecord> int remove(Class<T> clazz, Query<T> query)
- throws IOException;
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreSerializer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreSerializer.java
deleted file mode 100644
index 8540405..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreSerializer.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.driver;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
-import org.apache.hadoop.util.ReflectionUtils;
-
-/**
- * Serializer to store and retrieve data in the State Store.
- */
-public abstract class StateStoreSerializer {
-
- /** Singleton for the serializer instance. */
- private static StateStoreSerializer defaultSerializer;
-
- /**
- * Get the default serializer based.
- * @return Singleton serializer.
- */
- public static StateStoreSerializer getSerializer() {
- return getSerializer(null);
- }
-
- /**
- * Get a serializer based on the provided configuration.
- * @param conf Configuration. Default if null.
- * @return Singleton serializer.
- */
- public static StateStoreSerializer getSerializer(Configuration conf) {
- if (conf == null) {
- synchronized (StateStoreSerializer.class) {
- if (defaultSerializer == null) {
- conf = new Configuration();
- defaultSerializer = newSerializer(conf);
- }
- }
- return defaultSerializer;
- } else {
- return newSerializer(conf);
- }
- }
-
- private static StateStoreSerializer newSerializer(final Configuration conf) {
- Class<? extends StateStoreSerializer> serializerName = conf.getClass(
- DFSConfigKeys.FEDERATION_STORE_SERIALIZER_CLASS,
- DFSConfigKeys.FEDERATION_STORE_SERIALIZER_CLASS_DEFAULT,
- StateStoreSerializer.class);
- return ReflectionUtils.newInstance(serializerName, conf);
- }
-
- /**
- * Create a new record.
- * @param clazz Class of the new record.
- * @return New record.
- */
- public static <T> T newRecord(Class<T> clazz) {
- return getSerializer(null).newRecordInstance(clazz);
- }
-
- /**
- * Create a new record.
- * @param clazz Class of the new record.
- * @return New record.
- */
- public abstract <T> T newRecordInstance(Class<T> clazz);
-
- /**
- * Serialize a record into a byte array.
- * @param record Record to serialize.
- * @return Byte array with the serialized record.
- */
- public abstract byte[] serialize(BaseRecord record);
-
- /**
- * Serialize a record into a string.
- * @param record Record to serialize.
- * @return String with the serialized record.
- */
- public abstract String serializeString(BaseRecord record);
-
- /**
- * Deserialize a bytes array into a record.
- * @param byteArray Byte array to deserialize.
- * @param clazz Class of the record.
- * @return New record.
- * @throws IOException If it cannot deserialize the record.
- */
- public abstract <T extends BaseRecord> T deserialize(
- byte[] byteArray, Class<T> clazz) throws IOException;
-
- /**
- * Deserialize a string into a record.
- * @param data String with the data to deserialize.
- * @param clazz Class of the record.
- * @return New record.
- * @throws IOException If it cannot deserialize the record.
- */
- public abstract <T extends BaseRecord> T deserialize(
- String data, Class<T> clazz) throws IOException;
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetMountTableEntriesRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetMountTableEntriesRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetMountTableEntriesRequestPBImpl.java
new file mode 100644
index 0000000..3e0d1a6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetMountTableEntriesRequestPBImpl.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesRequestProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * GetMountTableEntriesRequest.
+ */
+public class GetMountTableEntriesRequestPBImpl
+ extends GetMountTableEntriesRequest implements PBRecord {
+
+ private FederationProtocolPBTranslator<GetMountTableEntriesRequestProto,
+ GetMountTableEntriesRequestProto.Builder,
+ GetMountTableEntriesRequestProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<GetMountTableEntriesRequestProto,
+ GetMountTableEntriesRequestProto.Builder,
+ GetMountTableEntriesRequestProtoOrBuilder>(
+ GetMountTableEntriesRequestProto.class);
+
+ public GetMountTableEntriesRequestPBImpl() {
+ }
+
+ public GetMountTableEntriesRequestPBImpl(
+ GetMountTableEntriesRequestProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public GetMountTableEntriesRequestProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public String getSrcPath() {
+ return this.translator.getProtoOrBuilder().getSrcPath();
+ }
+
+ @Override
+ public void setSrcPath(String path) {
+ this.translator.getBuilder().setSrcPath(path);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetMountTableEntriesResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetMountTableEntriesResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetMountTableEntriesResponsePBImpl.java
new file mode 100644
index 0000000..9d64bc9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetMountTableEntriesResponsePBImpl.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesResponseProtoOrBuilder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProto;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.MountTablePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * GetMountTableEntriesResponse.
+ */
+public class GetMountTableEntriesResponsePBImpl
+ extends GetMountTableEntriesResponse implements PBRecord {
+
+ private FederationProtocolPBTranslator<GetMountTableEntriesResponseProto,
+ GetMountTableEntriesResponseProto.Builder,
+ GetMountTableEntriesResponseProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<GetMountTableEntriesResponseProto,
+ GetMountTableEntriesResponseProto.Builder,
+ GetMountTableEntriesResponseProtoOrBuilder>(
+ GetMountTableEntriesResponseProto.class);
+
+ public GetMountTableEntriesResponsePBImpl() {
+ }
+
+ public GetMountTableEntriesResponsePBImpl(
+ GetMountTableEntriesResponseProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public GetMountTableEntriesResponseProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public List<MountTable> getEntries() throws IOException {
+ List<MountTableRecordProto> entries =
+ this.translator.getProtoOrBuilder().getEntriesList();
+ List<MountTable> ret = new ArrayList<MountTable>();
+ for (MountTableRecordProto entry : entries) {
+ MountTable record = new MountTablePBImpl(entry);
+ ret.add(record);
+ }
+ return ret;
+ }
+
+ @Override
+ public void setEntries(List<MountTable> records) throws IOException {
+ this.translator.getBuilder().clearEntries();
+ for (MountTable entry : records) {
+ if (entry instanceof MountTablePBImpl) {
+ MountTablePBImpl entryPB = (MountTablePBImpl)entry;
+ this.translator.getBuilder().addEntries(entryPB.getProto());
+ }
+ }
+ }
+
+ @Override
+ public long getTimestamp() {
+ return this.translator.getProtoOrBuilder().getTimestamp();
+ }
+
+ @Override
+ public void setTimestamp(long time) {
+ this.translator.getBuilder().setTimestamp(time);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamenodeRegistrationsRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamenodeRegistrationsRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamenodeRegistrationsRequestPBImpl.java
new file mode 100644
index 0000000..4f7fee1
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamenodeRegistrationsRequestPBImpl.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetNamenodeRegistrationsRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetNamenodeRegistrationsRequestProtoOrBuilder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipRecordProto;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.MembershipStatePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * GetNamenodeRegistrationsRequest.
+ */
+public class GetNamenodeRegistrationsRequestPBImpl
+ extends GetNamenodeRegistrationsRequest implements PBRecord {
+
+ private FederationProtocolPBTranslator<GetNamenodeRegistrationsRequestProto,
+ GetNamenodeRegistrationsRequestProto.Builder,
+ GetNamenodeRegistrationsRequestProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<
+ GetNamenodeRegistrationsRequestProto,
+ GetNamenodeRegistrationsRequestProto.Builder,
+ GetNamenodeRegistrationsRequestProtoOrBuilder>(
+ GetNamenodeRegistrationsRequestProto.class);
+
+ public GetNamenodeRegistrationsRequestPBImpl() {
+ }
+
+ public GetNamenodeRegistrationsRequestPBImpl(
+ GetNamenodeRegistrationsRequestProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public GetNamenodeRegistrationsRequestProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public MembershipState getPartialMembership() {
+ GetNamenodeRegistrationsRequestProtoOrBuilder proto =
+ this.translator.getProtoOrBuilder();
+ if (!proto.hasMembership()){
+ return null;
+ }
+ NamenodeMembershipRecordProto memberProto = proto.getMembership();
+ return new MembershipStatePBImpl(memberProto);
+ }
+
+ @Override
+ public void setPartialMembership(MembershipState member) {
+ MembershipStatePBImpl memberPB = (MembershipStatePBImpl)member;
+ this.translator.getBuilder().setMembership(memberPB.getProto());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamenodeRegistrationsResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamenodeRegistrationsResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamenodeRegistrationsResponsePBImpl.java
new file mode 100644
index 0000000..f6be11d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamenodeRegistrationsResponsePBImpl.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetNamenodeRegistrationsResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetNamenodeRegistrationsResponseProtoOrBuilder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipRecordProto;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.MembershipStatePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * GetNamenodeRegistrationsResponse.
+ */
+public class GetNamenodeRegistrationsResponsePBImpl
+ extends GetNamenodeRegistrationsResponse implements PBRecord {
+
+ private FederationProtocolPBTranslator<GetNamenodeRegistrationsResponseProto,
+ GetNamenodeRegistrationsResponseProto.Builder,
+ GetNamenodeRegistrationsResponseProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<
+ GetNamenodeRegistrationsResponseProto,
+ GetNamenodeRegistrationsResponseProto.Builder,
+ GetNamenodeRegistrationsResponseProtoOrBuilder>(
+ GetNamenodeRegistrationsResponseProto.class);
+
+ public GetNamenodeRegistrationsResponsePBImpl() {
+ }
+
+ public GetNamenodeRegistrationsResponsePBImpl(
+ GetNamenodeRegistrationsResponseProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public GetNamenodeRegistrationsResponseProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public List<MembershipState> getNamenodeMemberships()
+ throws IOException {
+
+ List<MembershipState> ret = new ArrayList<MembershipState>();
+ List<NamenodeMembershipRecordProto> memberships =
+ this.translator.getProtoOrBuilder().getNamenodeMembershipsList();
+ for (NamenodeMembershipRecordProto memberProto : memberships) {
+ MembershipState membership = new MembershipStatePBImpl(memberProto);
+ ret.add(membership);
+ }
+
+ return ret;
+ }
+
+ @Override
+ public void setNamenodeMemberships(List<MembershipState> records)
+ throws IOException {
+ for (MembershipState member : records) {
+ if (member instanceof MembershipStatePBImpl) {
+ MembershipStatePBImpl memberPB = (MembershipStatePBImpl)member;
+ this.translator.getBuilder().addNamenodeMemberships(
+ memberPB.getProto());
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamespaceInfoRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamespaceInfoRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamespaceInfoRequestPBImpl.java
new file mode 100644
index 0000000..5f3e186
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamespaceInfoRequestPBImpl.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetNamespaceInfoRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetNamespaceInfoRequestProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetNamespaceInfoRequestProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * GetNamespaceInfoRequest.
+ */
+public class GetNamespaceInfoRequestPBImpl extends GetNamespaceInfoRequest
+ implements PBRecord {
+
+ private FederationProtocolPBTranslator<GetNamespaceInfoRequestProto,
+ Builder, GetNamespaceInfoRequestProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<GetNamespaceInfoRequestProto,
+ Builder, GetNamespaceInfoRequestProtoOrBuilder>(
+ GetNamespaceInfoRequestProto.class);
+
+ public GetNamespaceInfoRequestPBImpl() {
+ }
+
+ @Override
+ public GetNamespaceInfoRequestProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message protocol) {
+ this.translator.setProto(protocol);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamespaceInfoResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamespaceInfoResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamespaceInfoResponsePBImpl.java
new file mode 100644
index 0000000..be1b184
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamespaceInfoResponsePBImpl.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.FederationNamespaceInfoProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetNamespaceInfoResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetNamespaceInfoResponseProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * GetNamespaceInfoResponse.
+ */
+public class GetNamespaceInfoResponsePBImpl
+ extends GetNamespaceInfoResponse implements PBRecord {
+
+ private FederationProtocolPBTranslator<GetNamespaceInfoResponseProto,
+ GetNamespaceInfoResponseProto.Builder,
+ GetNamespaceInfoResponseProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<GetNamespaceInfoResponseProto,
+ GetNamespaceInfoResponseProto.Builder,
+ GetNamespaceInfoResponseProtoOrBuilder>(
+ GetNamespaceInfoResponseProto.class);
+
+ public GetNamespaceInfoResponsePBImpl() {
+ }
+
+ @Override
+ public GetNamespaceInfoResponseProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message protocol) {
+ this.translator.setProto(protocol);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public Set<FederationNamespaceInfo> getNamespaceInfo() {
+
+ Set<FederationNamespaceInfo> ret = new HashSet<FederationNamespaceInfo>();
+ List<FederationNamespaceInfoProto> namespaceList =
+ this.translator.getProtoOrBuilder().getNamespaceInfosList();
+ for (FederationNamespaceInfoProto ns : namespaceList) {
+ FederationNamespaceInfo info = new FederationNamespaceInfo(
+ ns.getBlockPoolId(), ns.getClusterId(), ns.getNameserviceId());
+ ret.add(info);
+ }
+ return ret;
+ }
+
+ @Override
+ public void setNamespaceInfo(Set<FederationNamespaceInfo> namespaceInfo) {
+ int index = 0;
+ for (FederationNamespaceInfo item : namespaceInfo) {
+ FederationNamespaceInfoProto.Builder itemBuilder =
+ FederationNamespaceInfoProto.newBuilder();
+ itemBuilder.setClusterId(item.getClusterId());
+ itemBuilder.setBlockPoolId(item.getBlockPoolId());
+ itemBuilder.setNameserviceId(item.getNameserviceId());
+ this.translator.getBuilder().addNamespaceInfos(index,
+ itemBuilder.build());
+ index++;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationRequestPBImpl.java
new file mode 100644
index 0000000..19bf4ec
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationRequestPBImpl.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationRequestProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationRequestProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * GetRouterRegistrationRequest.
+ */
+public class GetRouterRegistrationRequestPBImpl
+ extends GetRouterRegistrationRequest implements PBRecord {
+
+ private FederationProtocolPBTranslator<GetRouterRegistrationRequestProto,
+ Builder, GetRouterRegistrationRequestProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<GetRouterRegistrationRequestProto,
+ Builder, GetRouterRegistrationRequestProtoOrBuilder>(
+ GetRouterRegistrationRequestProto.class);
+
+ public GetRouterRegistrationRequestPBImpl() {
+ }
+
+ @Override
+ public GetRouterRegistrationRequestProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public String getRouterId() {
+ return this.translator.getProtoOrBuilder().getRouterId();
+ }
+
+ @Override
+ public void setRouterId(String routerId) {
+ this.translator.getBuilder().setRouterId(routerId);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationResponsePBImpl.java
new file mode 100644
index 0000000..ae17e92
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationResponsePBImpl.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationResponseProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationResponseProtoOrBuilder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterRecordProto;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.RouterStatePBImpl;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * GetRouterRegistrationResponse.
+ */
+public class GetRouterRegistrationResponsePBImpl
+ extends GetRouterRegistrationResponse implements PBRecord {
+
+ private FederationProtocolPBTranslator<GetRouterRegistrationResponseProto,
+ Builder, GetRouterRegistrationResponseProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<
+ GetRouterRegistrationResponseProto, Builder,
+ GetRouterRegistrationResponseProtoOrBuilder>(
+ GetRouterRegistrationResponseProto.class);
+
+ public GetRouterRegistrationResponsePBImpl() {
+ }
+
+ @Override
+ public GetRouterRegistrationResponseProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message protocol) {
+ this.translator.setProto(protocol);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public RouterState getRouter() {
+ RouterRecordProto proto = this.translator.getProtoOrBuilder().getRouter();
+ return new RouterStatePBImpl(proto);
+ }
+
+ @Override
+ public void setRouter(RouterState router) {
+ if (router instanceof RouterStatePBImpl) {
+ RouterStatePBImpl routerPB = (RouterStatePBImpl)router;
+ RouterRecordProto routerProto = routerPB.getProto();
+ this.translator.getBuilder().setRouter(routerProto);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationsRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationsRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationsRequestPBImpl.java
new file mode 100644
index 0000000..4b48682
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationsRequestPBImpl.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationsRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationsRequestProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationsRequestProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationsRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * GetRouterRegistrationsRequest.
+ */
+public class GetRouterRegistrationsRequestPBImpl
+ extends GetRouterRegistrationsRequest implements PBRecord {
+
+ private FederationProtocolPBTranslator<GetRouterRegistrationsRequestProto,
+ Builder, GetRouterRegistrationsRequestProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<
+ GetRouterRegistrationsRequestProto, Builder,
+ GetRouterRegistrationsRequestProtoOrBuilder>(
+ GetRouterRegistrationsRequestProto.class);
+
+ public GetRouterRegistrationsRequestPBImpl() {
+ }
+
+ @Override
+ public GetRouterRegistrationsRequestProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationsResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationsResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationsResponsePBImpl.java
new file mode 100644
index 0000000..2d597fb
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationsResponsePBImpl.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationsResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationsResponseProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationsResponseProtoOrBuilder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterRecordProto;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationsResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.RouterStatePBImpl;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * GetRouterRegistrationsResponse.
+ */
+public class GetRouterRegistrationsResponsePBImpl
+ extends GetRouterRegistrationsResponse implements PBRecord {
+
+ private FederationProtocolPBTranslator<GetRouterRegistrationsResponseProto,
+ Builder, GetRouterRegistrationsResponseProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<
+ GetRouterRegistrationsResponseProto, Builder,
+ GetRouterRegistrationsResponseProtoOrBuilder>(
+ GetRouterRegistrationsResponseProto.class);
+
+ public GetRouterRegistrationsResponsePBImpl() {
+
+ }
+
+ @Override
+ public GetRouterRegistrationsResponseProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public List<RouterState> getRouters() throws IOException {
+
+ List<RouterState> ret = new ArrayList<RouterState>();
+ List<RouterRecordProto> memberships =
+ this.translator.getProtoOrBuilder().getRoutersList();
+ for (RouterRecordProto memberProto : memberships) {
+ RouterState membership = new RouterStatePBImpl(memberProto);
+ ret.add(membership);
+ }
+ return ret;
+ }
+
+ @Override
+ public void setRouters(List<RouterState> records) throws IOException {
+
+ this.translator.getBuilder().clearRouters();
+ for (RouterState router : records) {
+ if (router instanceof RouterStatePBImpl) {
+ RouterStatePBImpl routerPB = (RouterStatePBImpl) router;
+ this.translator.getBuilder().addRouters(routerPB.getProto());
+ }
+ }
+ }
+
+ @Override
+ public long getTimestamp() {
+ return this.translator.getProtoOrBuilder().getTimestamp();
+ }
+
+ @Override
+ public void setTimestamp(long time) {
+ this.translator.getBuilder().setTimestamp(time);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetSafeModeRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetSafeModeRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetSafeModeRequestPBImpl.java
new file mode 100644
index 0000000..1a4f63b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetSafeModeRequestPBImpl.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetSafeModeRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetSafeModeRequestProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetSafeModeRequestProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * GetSafeModeRequest.
+ */
+public class GetSafeModeRequestPBImpl extends GetSafeModeRequest
+ implements PBRecord {
+
+ private FederationProtocolPBTranslator<GetSafeModeRequestProto,
+ Builder, GetSafeModeRequestProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<>(GetSafeModeRequestProto.class);
+
+ public GetSafeModeRequestPBImpl() {
+ }
+
+ public GetSafeModeRequestPBImpl(GetSafeModeRequestProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public GetSafeModeRequestProto getProto() {
+ return translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetSafeModeResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetSafeModeResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetSafeModeResponsePBImpl.java
new file mode 100644
index 0000000..77fbeca
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetSafeModeResponsePBImpl.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetSafeModeResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetSafeModeResponseProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetSafeModeResponseProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * GetSafeModeResponse.
+ */
+public class GetSafeModeResponsePBImpl extends GetSafeModeResponse
+ implements PBRecord {
+
+ private FederationProtocolPBTranslator<GetSafeModeResponseProto,
+ Builder, GetSafeModeResponseProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<>(
+ GetSafeModeResponseProto.class);
+
+ public GetSafeModeResponsePBImpl() {
+ }
+
+ public GetSafeModeResponsePBImpl(GetSafeModeResponseProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public GetSafeModeResponseProto getProto() {
+ return translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public boolean isInSafeMode() {
+ return this.translator.getProtoOrBuilder().getIsInSafeMode();
+ }
+
+ @Override
+ public void setSafeMode(boolean isInSafeMode) {
+ this.translator.getBuilder().setIsInSafeMode(isInSafeMode);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/LeaveSafeModeRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/LeaveSafeModeRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/LeaveSafeModeRequestPBImpl.java
new file mode 100644
index 0000000..075099b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/LeaveSafeModeRequestPBImpl.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.LeaveSafeModeRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.LeaveSafeModeRequestProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.LeaveSafeModeRequestProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * LeaveSafeModeRequest.
+ */
+public class LeaveSafeModeRequestPBImpl extends LeaveSafeModeRequest
+ implements PBRecord {
+
+ private FederationProtocolPBTranslator<LeaveSafeModeRequestProto,
+ Builder, LeaveSafeModeRequestProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<>(LeaveSafeModeRequestProto.class);
+
+ public LeaveSafeModeRequestPBImpl() {
+ }
+
+ public LeaveSafeModeRequestPBImpl(LeaveSafeModeRequestProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public LeaveSafeModeRequestProto getProto() {
+ return translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/LeaveSafeModeResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/LeaveSafeModeResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/LeaveSafeModeResponsePBImpl.java
new file mode 100644
index 0000000..79f65fe
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/LeaveSafeModeResponsePBImpl.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.LeaveSafeModeResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.LeaveSafeModeResponseProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.LeaveSafeModeResponseProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * LeaveSafeModeResponse.
+ */
+public class LeaveSafeModeResponsePBImpl extends LeaveSafeModeResponse
+ implements PBRecord {
+
+ private FederationProtocolPBTranslator<LeaveSafeModeResponseProto,
+ Builder, LeaveSafeModeResponseProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<>(
+ LeaveSafeModeResponseProto.class);
+
+ public LeaveSafeModeResponsePBImpl() {
+ }
+
+ public LeaveSafeModeResponsePBImpl(LeaveSafeModeResponseProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public LeaveSafeModeResponseProto getProto() {
+ return translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public boolean getStatus() {
+ return this.translator.getProtoOrBuilder().getStatus();
+ }
+
+ @Override
+ public void setStatus(boolean result) {
+ this.translator.getBuilder().setStatus(result);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/NamenodeHeartbeatRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/NamenodeHeartbeatRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/NamenodeHeartbeatRequestPBImpl.java
new file mode 100644
index 0000000..d1fc73f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/NamenodeHeartbeatRequestPBImpl.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeHeartbeatRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeHeartbeatRequestProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeHeartbeatRequestProtoOrBuilder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipRecordProto;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.MembershipStatePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * NamenodeHeartbeatRequest.
+ */
+public class NamenodeHeartbeatRequestPBImpl
+ extends NamenodeHeartbeatRequest implements PBRecord {
+
+ private FederationProtocolPBTranslator<NamenodeHeartbeatRequestProto, Builder,
+ NamenodeHeartbeatRequestProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<NamenodeHeartbeatRequestProto,
+ Builder,
+ NamenodeHeartbeatRequestProtoOrBuilder>(
+ NamenodeHeartbeatRequestProto.class);
+
+ public NamenodeHeartbeatRequestPBImpl() {
+ }
+
+ @Override
+ public NamenodeHeartbeatRequestProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public MembershipState getNamenodeMembership() throws IOException {
+ NamenodeMembershipRecordProto membershipProto =
+ this.translator.getProtoOrBuilder().getNamenodeMembership();
+ MembershipState membership =
+ StateStoreSerializer.newRecord(MembershipState.class);
+ if (membership instanceof MembershipStatePBImpl) {
+ MembershipStatePBImpl membershipPB = (MembershipStatePBImpl)membership;
+ membershipPB.setProto(membershipProto);
+ return membershipPB;
+ } else {
+ throw new IOException("Cannot get membership from request");
+ }
+ }
+
+ @Override
+ public void setNamenodeMembership(MembershipState membership)
+ throws IOException {
+ if (membership instanceof MembershipStatePBImpl) {
+ MembershipStatePBImpl membershipPB = (MembershipStatePBImpl)membership;
+ NamenodeMembershipRecordProto membershipProto =
+ (NamenodeMembershipRecordProto)membershipPB.getProto();
+ this.translator.getBuilder().setNamenodeMembership(membershipProto);
+ } else {
+ throw new IOException("Cannot set mount table entry");
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/NamenodeHeartbeatResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/NamenodeHeartbeatResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/NamenodeHeartbeatResponsePBImpl.java
new file mode 100644
index 0000000..c243a6f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/NamenodeHeartbeatResponsePBImpl.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeHeartbeatResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeHeartbeatResponseProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * NamenodeHeartbeatResponse.
+ */
+public class NamenodeHeartbeatResponsePBImpl extends NamenodeHeartbeatResponse
+ implements PBRecord {
+
+ private FederationProtocolPBTranslator<NamenodeHeartbeatResponseProto,
+ NamenodeHeartbeatResponseProto.Builder,
+ NamenodeHeartbeatResponseProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<NamenodeHeartbeatResponseProto,
+ NamenodeHeartbeatResponseProto.Builder,
+ NamenodeHeartbeatResponseProtoOrBuilder>(
+ NamenodeHeartbeatResponseProto.class);
+
+ public NamenodeHeartbeatResponsePBImpl() {
+ }
+
+ @Override
+ public NamenodeHeartbeatResponseProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public boolean getResult() {
+ return this.translator.getProtoOrBuilder().getStatus();
+ }
+
+ @Override
+ public void setResult(boolean result) {
+ this.translator.getBuilder().setStatus(result);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntryRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntryRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntryRequestPBImpl.java
new file mode 100644
index 0000000..7f7c998
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntryRequestPBImpl.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryRequestProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * RemoveMountTableEntryRequest.
+ */
+public class RemoveMountTableEntryRequestPBImpl
+ extends RemoveMountTableEntryRequest implements PBRecord {
+
+ private FederationProtocolPBTranslator<RemoveMountTableEntryRequestProto,
+ RemoveMountTableEntryRequestProto.Builder,
+ RemoveMountTableEntryRequestProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<RemoveMountTableEntryRequestProto,
+ RemoveMountTableEntryRequestProto.Builder,
+ RemoveMountTableEntryRequestProtoOrBuilder>(
+ RemoveMountTableEntryRequestProto.class);
+
+ public RemoveMountTableEntryRequestPBImpl() {
+ }
+
+ public RemoveMountTableEntryRequestPBImpl(
+ RemoveMountTableEntryRequestProto proto) {
+ this.setProto(proto);
+ }
+
+ @Override
+ public RemoveMountTableEntryRequestProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public String getSrcPath() {
+ return this.translator.getProtoOrBuilder().getSrcPath();
+ }
+
+ @Override
+ public void setSrcPath(String path) {
+ this.translator.getBuilder().setSrcPath(path);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntryResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntryResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntryResponsePBImpl.java
new file mode 100644
index 0000000..0c943ac
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntryResponsePBImpl.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryResponseProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryResponseProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * RemoveMountTableEntryResponse.
+ */
+public class RemoveMountTableEntryResponsePBImpl
+ extends RemoveMountTableEntryResponse implements PBRecord {
+
+ private FederationProtocolPBTranslator<RemoveMountTableEntryResponseProto,
+ Builder, RemoveMountTableEntryResponseProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<RemoveMountTableEntryResponseProto,
+ RemoveMountTableEntryResponseProto.Builder,
+ RemoveMountTableEntryResponseProtoOrBuilder>(
+ RemoveMountTableEntryResponseProto.class);
+
+ public RemoveMountTableEntryResponsePBImpl() {
+ }
+
+ public RemoveMountTableEntryResponsePBImpl(
+ RemoveMountTableEntryResponseProto proto) {
+ this.setProto(proto);
+ }
+
+ @Override
+ public RemoveMountTableEntryResponseProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public boolean getStatus() {
+ return this.translator.getProtoOrBuilder().getStatus();
+ }
+
+ @Override
+ public void setStatus(boolean result) {
+ this.translator.getBuilder().setStatus(result);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RouterHeartbeatRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RouterHeartbeatRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RouterHeartbeatRequestPBImpl.java
new file mode 100644
index 0000000..cc2be04
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RouterHeartbeatRequestPBImpl.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterHeartbeatRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterHeartbeatRequestProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterHeartbeatRequestProtoOrBuilder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterRecordProto;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.RouterStatePBImpl;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * RouterHeartbeatRequest.
+ */
+public class RouterHeartbeatRequestPBImpl extends RouterHeartbeatRequest
+ implements PBRecord {
+
+ private FederationProtocolPBTranslator<RouterHeartbeatRequestProto, Builder,
+ RouterHeartbeatRequestProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<RouterHeartbeatRequestProto,
+ Builder, RouterHeartbeatRequestProtoOrBuilder>(
+ RouterHeartbeatRequestProto.class);
+
+ public RouterHeartbeatRequestPBImpl() {
+ }
+
+ @Override
+ public RouterHeartbeatRequestProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public RouterState getRouter() throws IOException {
+ RouterRecordProto routerProto =
+ this.translator.getProtoOrBuilder().getRouter();
+ return new RouterStatePBImpl(routerProto);
+ }
+
+ @Override
+ public void setRouter(RouterState routerState) {
+ if (routerState instanceof RouterStatePBImpl) {
+ RouterStatePBImpl routerStatePB = (RouterStatePBImpl)routerState;
+ this.translator.getBuilder().setRouter(routerStatePB.getProto());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RouterHeartbeatResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RouterHeartbeatResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RouterHeartbeatResponsePBImpl.java
new file mode 100644
index 0000000..ac534fd
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RouterHeartbeatResponsePBImpl.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterHeartbeatResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterHeartbeatResponseProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterHeartbeatResponseProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * RouterHeartbeatResponse.
+ */
+public class RouterHeartbeatResponsePBImpl extends RouterHeartbeatResponse
+ implements PBRecord {
+
+ private FederationProtocolPBTranslator<RouterHeartbeatResponseProto, Builder,
+ RouterHeartbeatResponseProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<RouterHeartbeatResponseProto,
+ Builder, RouterHeartbeatResponseProtoOrBuilder>(
+ RouterHeartbeatResponseProto.class);
+
+ public RouterHeartbeatResponsePBImpl() {
+ }
+
+ @Override
+ public RouterHeartbeatResponseProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public boolean getStatus() {
+ return this.translator.getProtoOrBuilder().getStatus();
+ }
+
+ @Override
+ public void setStatus(boolean result) {
+ this.translator.getBuilder().setStatus(result);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateMountTableEntryRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateMountTableEntryRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateMountTableEntryRequestPBImpl.java
new file mode 100644
index 0000000..621bb3a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateMountTableEntryRequestPBImpl.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryRequestProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.MountTablePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * UpdateMountTableEntryRequest.
+ */
+public class UpdateMountTableEntryRequestPBImpl
+ extends UpdateMountTableEntryRequest implements PBRecord {
+
+ private FederationProtocolPBTranslator<UpdateMountTableEntryRequestProto,
+ UpdateMountTableEntryRequestProto.Builder,
+ UpdateMountTableEntryRequestProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<UpdateMountTableEntryRequestProto,
+ UpdateMountTableEntryRequestProto.Builder,
+ UpdateMountTableEntryRequestProtoOrBuilder>(
+ UpdateMountTableEntryRequestProto.class);
+
+ public UpdateMountTableEntryRequestPBImpl() {
+ }
+
+ public UpdateMountTableEntryRequestPBImpl(
+ UpdateMountTableEntryRequestProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public UpdateMountTableEntryRequestProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public MountTable getEntry() throws IOException {
+ MountTableRecordProto statsProto =
+ this.translator.getProtoOrBuilder().getEntry();
+ MountTable stats = StateStoreSerializer.newRecord(MountTable.class);
+ if (stats instanceof MountTablePBImpl) {
+ MountTablePBImpl entryPB = (MountTablePBImpl)stats;
+ entryPB.setProto(statsProto);
+ return entryPB;
+ } else {
+ throw new IOException("Cannot get stats for the membership");
+ }
+ }
+
+ @Override
+ public void setEntry(MountTable mount) throws IOException {
+ if (mount instanceof MountTablePBImpl) {
+ MountTablePBImpl mountPB = (MountTablePBImpl)mount;
+ MountTableRecordProto mountProto =
+ (MountTableRecordProto)mountPB.getProto();
+ this.translator.getBuilder().setEntry(mountProto);
+ } else {
+ throw new IOException("Cannot set mount table entry");
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateMountTableEntryResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateMountTableEntryResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateMountTableEntryResponsePBImpl.java
new file mode 100644
index 0000000..5d566d6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateMountTableEntryResponsePBImpl.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryResponseProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the state store API object
+ * UpdateMountTableEntryResponse.
+ */
+public class UpdateMountTableEntryResponsePBImpl
+ extends UpdateMountTableEntryResponse implements PBRecord {
+
+ private FederationProtocolPBTranslator<UpdateMountTableEntryResponseProto,
+ UpdateMountTableEntryResponseProto.Builder,
+ UpdateMountTableEntryResponseProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<UpdateMountTableEntryResponseProto,
+ UpdateMountTableEntryResponseProto.Builder,
+ UpdateMountTableEntryResponseProtoOrBuilder>(
+ UpdateMountTableEntryResponseProto.class);
+
+ public UpdateMountTableEntryResponsePBImpl() {
+ }
+
+ public UpdateMountTableEntryResponsePBImpl(
+ UpdateMountTableEntryResponseProto proto) {
+ this.setProto(proto);
+ }
+
+ @Override
+ public UpdateMountTableEntryResponseProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public boolean getStatus() {
+ return this.translator.getProtoOrBuilder().getStatus();
+ }
+
+ @Override
+ public void setStatus(boolean result) {
+ this.translator.getBuilder().setStatus(result);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetrics.java
deleted file mode 100644
index 851538a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetrics.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import static org.apache.hadoop.metrics2.impl.MsInfo.ProcessName;
-import static org.apache.hadoop.metrics2.impl.MsInfo.SessionId;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.metrics2.MetricsSystem;
-import org.apache.hadoop.metrics2.annotation.Metric;
-import org.apache.hadoop.metrics2.annotation.Metrics;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.lib.MetricsRegistry;
-import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
-import org.apache.hadoop.metrics2.source.JvmMetrics;
-
-/**
- * This class is for maintaining the various Router activity statistics
- * and publishing them through the metrics interfaces.
- */
-@Metrics(name="RouterActivity", about="Router metrics", context="dfs")
-public class RouterMetrics {
-
- private final MetricsRegistry registry = new MetricsRegistry("router");
-
- @Metric("Duration in SafeMode at startup in msec")
- private MutableGaugeInt safeModeTime;
-
- private JvmMetrics jvmMetrics = null;
-
- RouterMetrics(
- String processName, String sessionId, final JvmMetrics jvmMetrics) {
- this.jvmMetrics = jvmMetrics;
- registry.tag(ProcessName, processName).tag(SessionId, sessionId);
- }
-
- public static RouterMetrics create(Configuration conf) {
- String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY);
- String processName = "Router";
- MetricsSystem ms = DefaultMetricsSystem.instance();
- JvmMetrics jm = JvmMetrics.create(processName, sessionId, ms);
-
- return ms.register(new RouterMetrics(processName, sessionId, jm));
- }
-
- public JvmMetrics getJvmMetrics() {
- return jvmMetrics;
- }
-
- public void shutdown() {
- DefaultMetricsSystem.shutdown();
- }
-
- public void setSafeModeTime(long elapsed) {
- safeModeTime.set((int) elapsed);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetricsService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetricsService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetricsService.java
deleted file mode 100644
index f4debce..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetricsService.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.server.federation.metrics.FederationMetrics;
-import org.apache.hadoop.hdfs.server.federation.metrics.NamenodeBeanMetrics;
-import org.apache.hadoop.metrics2.source.JvmMetrics;
-import org.apache.hadoop.service.AbstractService;
-
-/**
- * Service to manage the metrics of the Router.
- */
-public class RouterMetricsService extends AbstractService {
-
- /** Router for this metrics. */
- private final Router router;
-
- /** Router metrics. */
- private RouterMetrics routerMetrics;
- /** Federation metrics. */
- private FederationMetrics federationMetrics;
- /** Namenode mock metrics. */
- private NamenodeBeanMetrics nnMetrics;
-
-
- public RouterMetricsService(final Router router) {
- super(RouterMetricsService.class.getName());
- this.router = router;
- }
-
- @Override
- protected void serviceInit(Configuration configuration) throws Exception {
- this.routerMetrics = RouterMetrics.create(configuration);
- }
-
- @Override
- protected void serviceStart() throws Exception {
- // Wrapper for all the FSNamesystem JMX interfaces
- this.nnMetrics = new NamenodeBeanMetrics(this.router);
-
- // Federation MBean JMX interface
- this.federationMetrics = new FederationMetrics(this.router);
- }
-
- @Override
- protected void serviceStop() throws Exception {
- // Remove JMX interfaces
- if (this.federationMetrics != null) {
- this.federationMetrics.close();
- }
-
- // Remove Namenode JMX interfaces
- if (this.nnMetrics != null) {
- this.nnMetrics.close();
- }
-
- // Shutdown metrics
- if (this.routerMetrics != null) {
- this.routerMetrics.shutdown();
- }
- }
-
- /**
- * Get the metrics system for the Router.
- *
- * @return Router metrics.
- */
- public RouterMetrics getRouterMetrics() {
- return this.routerMetrics;
- }
-
- /**
- * Get the federation metrics.
- *
- * @return Federation metrics.
- */
- public FederationMetrics getFederationMetrics() {
- return this.federationMetrics;
- }
-
- /**
- * Get the JVM metrics for the Router.
- *
- * @return JVM metrics.
- */
- public JvmMetrics getJvmMetrics() {
- if (this.routerMetrics == null) {
- return null;
- }
- return this.routerMetrics.getJvmMetrics();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterPermissionChecker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterPermissionChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterPermissionChecker.java
deleted file mode 100644
index 9d81dce..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterPermissionChecker.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-import org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker;
-import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.security.UserGroupInformation;
-
-/**
- * Class that helps in checking permissions in Router-based federation.
- */
-public class RouterPermissionChecker extends FSPermissionChecker {
- static final Log LOG = LogFactory.getLog(RouterPermissionChecker.class);
-
- /** Mount table default permission. */
- public static final short MOUNT_TABLE_PERMISSION_DEFAULT = 00755;
-
- public RouterPermissionChecker(String routerOwner, String supergroup,
- UserGroupInformation callerUgi) {
- super(routerOwner, supergroup, callerUgi, null);
- }
-
- /**
- * Whether a mount table entry can be accessed by the current context.
- *
- * @param mountTable
- * MountTable being accessed
- * @param access
- * type of action being performed on the cache pool
- * @throws AccessControlException
- * if mount table cannot be accessed
- */
- public void checkPermission(MountTable mountTable, FsAction access)
- throws AccessControlException {
- if (isSuperUser()) {
- return;
- }
-
- FsPermission mode = mountTable.getMode();
- if (getUser().equals(mountTable.getOwnerName())
- && mode.getUserAction().implies(access)) {
- return;
- }
-
- if (isMemberOfGroup(mountTable.getGroupName())
- && mode.getGroupAction().implies(access)) {
- return;
- }
-
- if (!getUser().equals(mountTable.getOwnerName())
- && !isMemberOfGroup(mountTable.getGroupName())
- && mode.getOtherAction().implies(access)) {
- return;
- }
-
- throw new AccessControlException(
- "Permission denied while accessing mount table "
- + mountTable.getSourcePath()
- + ": user " + getUser() + " does not have " + access.toString()
- + " permissions.");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaManager.java
deleted file mode 100644
index 0df34fc..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaManager.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import static org.apache.hadoop.hdfs.server.federation.router.FederationUtil.isParentEntry;
-
-import java.util.HashSet;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-
-/**
- * Router quota manager in Router. The manager maintains
- * {@link RouterQuotaUsage} cache of mount tables and do management
- * for the quota caches.
- */
-public class RouterQuotaManager {
- /** Quota usage <MountTable Path, Aggregated QuotaUsage> cache. */
- private TreeMap<String, RouterQuotaUsage> cache;
-
- /** Lock to access the quota cache. */
- private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
- private final Lock readLock = readWriteLock.readLock();
- private final Lock writeLock = readWriteLock.writeLock();
-
- public RouterQuotaManager() {
- this.cache = new TreeMap<>();
- }
-
- /**
- * Get all the mount quota paths.
- */
- public Set<String> getAll() {
- readLock.lock();
- try {
- return this.cache.keySet();
- } finally {
- readLock.unlock();
- }
- }
-
- /**
- * Get the nearest ancestor's quota usage, and meanwhile its quota was set.
- * @param path The path being written.
- * @return RouterQuotaUsage Quota usage.
- */
- public RouterQuotaUsage getQuotaUsage(String path) {
- readLock.lock();
- try {
- RouterQuotaUsage quotaUsage = this.cache.get(path);
- if (quotaUsage != null && isQuotaSet(quotaUsage)) {
- return quotaUsage;
- }
-
- // If not found, look for its parent path usage value.
- int pos = path.lastIndexOf(Path.SEPARATOR);
- if (pos != -1) {
- String parentPath = path.substring(0, pos);
- return getQuotaUsage(parentPath);
- }
- } finally {
- readLock.unlock();
- }
-
- return null;
- }
-
- /**
- * Get children paths (can including itself) under specified federation path.
- * @param parentPath
- * @return Set<String> Children path set.
- */
- public Set<String> getPaths(String parentPath) {
- readLock.lock();
- try {
- String from = parentPath;
- String to = parentPath + Character.MAX_VALUE;
- SortedMap<String, RouterQuotaUsage> subMap = this.cache.subMap(from, to);
-
- Set<String> validPaths = new HashSet<>();
- if (subMap != null) {
- for (String path : subMap.keySet()) {
- if (isParentEntry(path, parentPath)) {
- validPaths.add(path);
- }
- }
- }
- return validPaths;
- } finally {
- readLock.unlock();
- }
- }
-
- /**
- * Put new entity into cache.
- * @param path Mount table path.
- * @param quotaUsage Corresponding cache value.
- */
- public void put(String path, RouterQuotaUsage quotaUsage) {
- writeLock.lock();
- try {
- this.cache.put(path, quotaUsage);
- } finally {
- writeLock.unlock();
- }
- }
-
- /**
- * Remove the entity from cache.
- * @param path Mount table path.
- */
- public void remove(String path) {
- writeLock.lock();
- try {
- this.cache.remove(path);
- } finally {
- writeLock.unlock();
- }
- }
-
- /**
- * Clean up the cache.
- */
- public void clear() {
- writeLock.lock();
- try {
- this.cache.clear();
- } finally {
- writeLock.unlock();
- }
- }
-
- /**
- * Check if the quota was set.
- * @param quota RouterQuotaUsage set in mount table.
- */
- public boolean isQuotaSet(RouterQuotaUsage quota) {
- if (quota != null) {
- long nsQuota = quota.getQuota();
- long ssQuota = quota.getSpaceQuota();
-
- // once nsQuota or ssQuota was set, this mount table is quota set
- if (nsQuota != HdfsConstants.QUOTA_DONT_SET
- || ssQuota != HdfsConstants.QUOTA_DONT_SET) {
- return true;
- }
- }
-
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java
deleted file mode 100644
index 80abc11..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java
+++ /dev/null
@@ -1,228 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.QuotaUsage;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.server.federation.store.MountTableStore;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Service to periodically update the {@link RouterQuotaUsage}
- * cached information in the {@link Router} and update corresponding
- * mount table in State Store.
- */
-public class RouterQuotaUpdateService extends PeriodicService {
- private static final Logger LOG =
- LoggerFactory.getLogger(RouterQuotaUpdateService.class);
-
- private MountTableStore mountTableStore;
- private RouterRpcServer rpcServer;
- /** Router using this Service. */
- private final Router router;
- /** Router Quota manager. */
- private RouterQuotaManager quotaManager;
-
- public RouterQuotaUpdateService(final Router router) throws IOException {
- super(RouterQuotaUpdateService.class.getName());
- this.router = router;
- this.rpcServer = router.getRpcServer();
- this.quotaManager = router.getQuotaManager();
-
- if (this.quotaManager == null) {
- throw new IOException("Router quota manager is not initialized.");
- }
- }
-
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
- this.setIntervalMs(conf.getTimeDuration(
- DFSConfigKeys.DFS_ROUTER_QUOTA_CACHE_UPATE_INTERVAL,
- DFSConfigKeys.DFS_ROUTER_QUOTA_CACHE_UPATE_INTERVAL_DEFAULT,
- TimeUnit.MILLISECONDS));
-
- super.serviceInit(conf);
- }
-
- @Override
- protected void periodicInvoke() {
- LOG.debug("Start to update quota cache.");
- try {
- List<MountTable> updateMountTables = new LinkedList<>();
- List<MountTable> mountTables = getQuotaSetMountTables();
- for (MountTable entry : mountTables) {
- String src = entry.getSourcePath();
- RouterQuotaUsage oldQuota = entry.getQuota();
- long nsQuota = oldQuota.getQuota();
- long ssQuota = oldQuota.getSpaceQuota();
- // Call RouterRpcServer#getQuotaUsage for getting current quota usage.
- QuotaUsage currentQuotaUsage = this.rpcServer.getQuotaModule()
- .getQuotaUsage(src);
- // If quota is not set in some subclusters under federation path,
- // set quota for this path.
- if (currentQuotaUsage.getQuota() == HdfsConstants.QUOTA_DONT_SET) {
- this.rpcServer.setQuota(src, nsQuota, ssQuota, null);
- }
-
- RouterQuotaUsage newQuota = generateNewQuota(oldQuota,
- currentQuotaUsage);
- this.quotaManager.put(src, newQuota);
- entry.setQuota(newQuota);
-
- // only update mount tables which quota was changed
- if (!oldQuota.equals(newQuota)) {
- updateMountTables.add(entry);
-
- LOG.debug(
- "Update quota usage entity of path: {}, nsCount: {},"
- + " nsQuota: {}, ssCount: {}, ssQuota: {}.",
- src, newQuota.getFileAndDirectoryCount(),
- newQuota.getQuota(), newQuota.getSpaceConsumed(),
- newQuota.getSpaceQuota());
- }
- }
-
- updateMountTableEntries(updateMountTables);
- } catch (IOException e) {
- LOG.error("Quota cache updated error.", e);
- }
- }
-
- /**
- * Get mount table store management interface.
- * @return MountTableStore instance.
- * @throws IOException
- */
- private MountTableStore getMountTableStore() throws IOException {
- if (this.mountTableStore == null) {
- this.mountTableStore = router.getStateStore().getRegisteredRecordStore(
- MountTableStore.class);
- if (this.mountTableStore == null) {
- throw new IOException("Mount table state store is not available.");
- }
- }
- return this.mountTableStore;
- }
-
- /**
- * Get all the existing mount tables.
- * @return List of mount tables.
- * @throws IOException
- */
- private List<MountTable> getMountTableEntries() throws IOException {
- // scan mount tables from root path
- GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest
- .newInstance("/");
- GetMountTableEntriesResponse getResponse = getMountTableStore()
- .getMountTableEntries(getRequest);
- return getResponse.getEntries();
- }
-
- /**
- * Get mount tables which quota was set.
- * During this time, the quota usage cache will also be updated by
- * quota manager:
- * 1. Stale paths (entries) will be removed.
- * 2. Existing entries will be override and updated.
- * @return List of mount tables which quota was set.
- * @throws IOException
- */
- private List<MountTable> getQuotaSetMountTables() throws IOException {
- List<MountTable> mountTables = getMountTableEntries();
- Set<String> stalePaths = new HashSet<>();
- for (String path : this.quotaManager.getAll()) {
- stalePaths.add(path);
- }
-
- List<MountTable> neededMountTables = new LinkedList<>();
- for (MountTable entry : mountTables) {
- // select mount tables which is quota set
- if (isQuotaSet(entry)) {
- neededMountTables.add(entry);
- }
-
- // update mount table entries info in quota cache
- String src = entry.getSourcePath();
- this.quotaManager.put(src, entry.getQuota());
- stalePaths.remove(src);
- }
-
- // remove stale paths that currently cached
- for (String stalePath : stalePaths) {
- this.quotaManager.remove(stalePath);
- }
-
- return neededMountTables;
- }
-
- /**
- * Check if the quota was set in given MountTable.
- * @param mountTable Mount table entry.
- */
- private boolean isQuotaSet(MountTable mountTable) {
- if (mountTable != null) {
- return this.quotaManager.isQuotaSet(mountTable.getQuota());
- }
- return false;
- }
-
- /**
- * Generate a new quota based on old quota and current quota usage value.
- * @param oldQuota Old quota stored in State Store.
- * @param currentQuotaUsage Current quota usage value queried from
- * subcluster.
- * @return A new RouterQuotaUsage.
- */
- private RouterQuotaUsage generateNewQuota(RouterQuotaUsage oldQuota,
- QuotaUsage currentQuotaUsage) {
- RouterQuotaUsage newQuota = new RouterQuotaUsage.Builder()
- .fileAndDirectoryCount(currentQuotaUsage.getFileAndDirectoryCount())
- .quota(oldQuota.getQuota())
- .spaceConsumed(currentQuotaUsage.getSpaceConsumed())
- .spaceQuota(oldQuota.getSpaceQuota()).build();
- return newQuota;
- }
-
- /**
- * Write out updated mount table entries into State Store.
- * @param updateMountTables Mount tables to be updated.
- * @throws IOException
- */
- private void updateMountTableEntries(List<MountTable> updateMountTables)
- throws IOException {
- for (MountTable entry : updateMountTables) {
- UpdateMountTableEntryRequest updateRequest = UpdateMountTableEntryRequest
- .newInstance(entry);
- getMountTableStore().updateMountTableEntry(updateRequest);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUsage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUsage.java
deleted file mode 100644
index eedd80f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUsage.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import org.apache.hadoop.fs.QuotaUsage;
-import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
-import org.apache.hadoop.hdfs.server.namenode.DirectoryWithQuotaFeature;
-import org.apache.hadoop.hdfs.server.namenode.Quota;
-import org.apache.hadoop.util.StringUtils;
-
-/**
- * The subclass of {@link QuotaUsage} used in Router-based federation.
- */
-public final class RouterQuotaUsage extends QuotaUsage {
-
- /** Default quota usage count. */
- public static final long QUOTA_USAGE_COUNT_DEFAULT = 0;
-
- private RouterQuotaUsage(Builder builder) {
- super(builder);
- }
-
- /** Build the instance based on the builder. */
- public static class Builder extends QuotaUsage.Builder {
-
- public RouterQuotaUsage build() {
- return new RouterQuotaUsage(this);
- }
-
- @Override
- public Builder fileAndDirectoryCount(long count) {
- super.fileAndDirectoryCount(count);
- return this;
- }
-
- @Override
- public Builder quota(long quota) {
- super.quota(quota);
- return this;
- }
-
- @Override
- public Builder spaceConsumed(long spaceConsumed) {
- super.spaceConsumed(spaceConsumed);
- return this;
- }
-
- @Override
- public Builder spaceQuota(long spaceQuota) {
- super.spaceQuota(spaceQuota);
- return this;
- }
- }
-
- /**
- * Verify if namespace quota is violated once quota is set. Relevant
- * method {@link DirectoryWithQuotaFeature#verifyNamespaceQuota}.
- * @throws NSQuotaExceededException
- */
- public void verifyNamespaceQuota() throws NSQuotaExceededException {
- if (Quota.isViolated(getQuota(), getFileAndDirectoryCount())) {
- throw new NSQuotaExceededException(getQuota(),
- getFileAndDirectoryCount());
- }
- }
-
- /**
- * Verify if storage space quota is violated once quota is set. Relevant
- * method {@link DirectoryWithQuotaFeature#verifyStoragespaceQuota}.
- * @throws DSQuotaExceededException
- */
- public void verifyStoragespaceQuota() throws DSQuotaExceededException {
- if (Quota.isViolated(getSpaceQuota(), getSpaceConsumed())) {
- throw new DSQuotaExceededException(getSpaceQuota(), getSpaceConsumed());
- }
- }
-
- @Override
- public String toString() {
- String nsQuota = String.valueOf(getQuota());
- String nsCount = String.valueOf(getFileAndDirectoryCount());
- if (getQuota() == HdfsConstants.QUOTA_DONT_SET) {
- nsQuota = "-";
- nsCount = "-";
- }
-
- String ssQuota = StringUtils.byteDesc(getSpaceQuota());
- String ssCount = StringUtils.byteDesc(getSpaceConsumed());
- if (getSpaceQuota() == HdfsConstants.QUOTA_DONT_SET) {
- ssQuota = "-";
- ssCount = "-";
- }
-
- StringBuilder str = new StringBuilder();
- str.append("[NsQuota: ").append(nsQuota).append("/")
- .append(nsCount);
- str.append(", SsQuota: ").append(ssQuota)
- .append("/").append(ssCount)
- .append("]");
- return str.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
deleted file mode 100644
index d3b7947..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
+++ /dev/null
@@ -1,1022 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.net.InetSocketAddress;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CancellationException;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.TimeUnit;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.NameNodeProxiesClient.ProxyAndInfo;
-import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
-import org.apache.hadoop.hdfs.protocol.ClientProtocol;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
-import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
-import org.apache.hadoop.io.retry.RetryPolicies;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.io.retry.RetryPolicy.RetryAction.RetryDecision;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.ipc.StandbyException;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-
-/**
- * A client proxy for Router -> NN communication using the NN ClientProtocol.
- * <p>
- * Provides routers to invoke remote ClientProtocol methods and handle
- * retries/failover.
- * <ul>
- * <li>invokeSingle Make a single request to a single namespace
- * <li>invokeSequential Make a sequential series of requests to multiple
- * ordered namespaces until a condition is met.
- * <li>invokeConcurrent Make concurrent requests to multiple namespaces and
- * return all of the results.
- * </ul>
- * Also maintains a cached pool of connections to NNs. Connections are managed
- * by the ConnectionManager and are unique to each user + NN. The size of the
- * connection pool can be configured. Larger pools allow for more simultaneous
- * requests to a single NN from a single user.
- */
-public class RouterRpcClient {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(RouterRpcClient.class);
-
-
- /** Router identifier. */
- private final String routerId;
-
- /** Interface to identify the active NN for a nameservice or blockpool ID. */
- private final ActiveNamenodeResolver namenodeResolver;
-
- /** Connection pool to the Namenodes per user for performance. */
- private final ConnectionManager connectionManager;
- /** Service to run asynchronous calls. */
- private final ExecutorService executorService;
- /** Retry policy for router -> NN communication. */
- private final RetryPolicy retryPolicy;
- /** Optional perf monitor. */
- private final RouterRpcMonitor rpcMonitor;
-
- /** Pattern to parse a stack trace line. */
- private static final Pattern STACK_TRACE_PATTERN =
- Pattern.compile("\\tat (.*)\\.(.*)\\((.*):(\\d*)\\)");
-
-
- /**
- * Create a router RPC client to manage remote procedure calls to NNs.
- *
- * @param conf Hdfs Configuation.
- * @param resolver A NN resolver to determine the currently active NN in HA.
- * @param monitor Optional performance monitor.
- */
- public RouterRpcClient(Configuration conf, String identifier,
- ActiveNamenodeResolver resolver, RouterRpcMonitor monitor) {
- this.routerId = identifier;
-
- this.namenodeResolver = resolver;
-
- this.connectionManager = new ConnectionManager(conf);
- this.connectionManager.start();
-
- int numThreads = conf.getInt(
- DFSConfigKeys.DFS_ROUTER_CLIENT_THREADS_SIZE,
- DFSConfigKeys.DFS_ROUTER_CLIENT_THREADS_SIZE_DEFAULT);
- ThreadFactory threadFactory = new ThreadFactoryBuilder()
- .setNameFormat("RPC Router Client-%d")
- .build();
- this.executorService = Executors.newFixedThreadPool(
- numThreads, threadFactory);
-
- this.rpcMonitor = monitor;
-
- int maxFailoverAttempts = conf.getInt(
- HdfsClientConfigKeys.Failover.MAX_ATTEMPTS_KEY,
- HdfsClientConfigKeys.Failover.MAX_ATTEMPTS_DEFAULT);
- int maxRetryAttempts = conf.getInt(
- DFSConfigKeys.DFS_ROUTER_CLIENT_MAX_ATTEMPTS,
- DFSConfigKeys.DFS_ROUTER_CLIENT_MAX_ATTEMPTS_DEFAULT);
- int failoverSleepBaseMillis = conf.getInt(
- HdfsClientConfigKeys.Failover.SLEEPTIME_BASE_KEY,
- HdfsClientConfigKeys.Failover.SLEEPTIME_BASE_DEFAULT);
- int failoverSleepMaxMillis = conf.getInt(
- HdfsClientConfigKeys.Failover.SLEEPTIME_MAX_KEY,
- HdfsClientConfigKeys.Failover.SLEEPTIME_MAX_DEFAULT);
- this.retryPolicy = RetryPolicies.failoverOnNetworkException(
- RetryPolicies.TRY_ONCE_THEN_FAIL, maxFailoverAttempts, maxRetryAttempts,
- failoverSleepBaseMillis, failoverSleepMaxMillis);
- }
-
- /**
- * Get the active namenode resolver used by this client.
- * @return Active namenode resolver.
- */
- public ActiveNamenodeResolver getNamenodeResolver() {
- return this.namenodeResolver;
- }
-
- /**
- * Shutdown the client.
- */
- public void shutdown() {
- if (this.connectionManager != null) {
- this.connectionManager.close();
- }
- if (this.executorService != null) {
- this.executorService.shutdownNow();
- }
- }
-
- /**
- * Total number of available sockets between the router and NNs.
- *
- * @return Number of namenode clients.
- */
- public int getNumConnections() {
- return this.connectionManager.getNumConnections();
- }
-
- /**
- * Total number of available sockets between the router and NNs.
- *
- * @return Number of namenode clients.
- */
- public int getNumActiveConnections() {
- return this.connectionManager.getNumActiveConnections();
- }
-
- /**
- * Total number of open connection pools to a NN. Each connection pool.
- * represents one user + one NN.
- *
- * @return Number of connection pools.
- */
- public int getNumConnectionPools() {
- return this.connectionManager.getNumConnectionPools();
- }
-
- /**
- * Number of connections between the router and NNs being created sockets.
- *
- * @return Number of connections waiting to be created.
- */
- public int getNumCreatingConnections() {
- return this.connectionManager.getNumCreatingConnections();
- }
-
- /**
- * JSON representation of the connection pool.
- *
- * @return String representation of the JSON.
- */
- public String getJSON() {
- return this.connectionManager.getJSON();
- }
-
- /**
- * Get ClientProtocol proxy client for a NameNode. Each combination of user +
- * NN must use a unique proxy client. Previously created clients are cached
- * and stored in a connection pool by the ConnectionManager.
- *
- * @param ugi User group information.
- * @param nsId Nameservice identifier.
- * @param rpcAddress ClientProtocol RPC server address of the NN.
- * @return ConnectionContext containing a ClientProtocol proxy client for the
- * NN + current user.
- * @throws IOException If we cannot get a connection to the NameNode.
- */
- private ConnectionContext getConnection(
- UserGroupInformation ugi, String nsId, String rpcAddress)
- throws IOException {
- ConnectionContext connection = null;
- try {
- // Each proxy holds the UGI info for the current user when it is created.
- // This cache does not scale very well, one entry per user per namenode,
- // and may need to be adjusted and/or selectively pruned. The cache is
- // important due to the excessive overhead of creating a new proxy wrapper
- // for each individual request.
-
- // TODO Add tokens from the federated UGI
- connection = this.connectionManager.getConnection(ugi, rpcAddress);
- LOG.debug("User {} NN {} is using connection {}",
- ugi.getUserName(), rpcAddress, connection);
- } catch (Exception ex) {
- LOG.error("Cannot open NN client to address: {}", rpcAddress, ex);
- }
-
- if (connection == null) {
- throw new IOException("Cannot get a connection to " + rpcAddress);
- }
- return connection;
- }
-
- /**
- * Convert an exception to an IOException.
- *
- * For a non-IOException, wrap it with IOException. For a RemoteException,
- * unwrap it. For an IOException which is not a RemoteException, return it.
- *
- * @param e Exception to convert into an exception.
- * @return Created IO exception.
- */
- private static IOException toIOException(Exception e) {
- if (e instanceof RemoteException) {
- return ((RemoteException) e).unwrapRemoteException();
- }
- if (e instanceof IOException) {
- return (IOException)e;
- }
- return new IOException(e);
- }
-
- /**
- * If we should retry the RPC call.
- *
- * @param ioe IOException reported.
- * @param retryCount Number of retries.
- * @param nsId Nameservice ID.
- * @return Retry decision.
- * @throws IOException Original exception if the retry policy generates one
- * or IOException for no available namenodes.
- */
- private RetryDecision shouldRetry(final IOException ioe, final int retryCount,
- final String nsId) throws IOException {
- // check for the case of cluster unavailable state
- if (isClusterUnAvailable(nsId)) {
- // we allow to retry once if cluster is unavailable
- if (retryCount == 0) {
- return RetryDecision.RETRY;
- } else {
- throw new IOException("No namenode available under nameservice " + nsId,
- ioe);
- }
- }
-
- try {
- final RetryPolicy.RetryAction a =
- this.retryPolicy.shouldRetry(ioe, retryCount, 0, true);
- return a.action;
- } catch (Exception ex) {
- LOG.error("Re-throwing API exception, no more retries", ex);
- throw toIOException(ex);
- }
- }
-
- /**
- * Invokes a method against the ClientProtocol proxy server. If a standby
- * exception is generated by the call to the client, retries using the
- * alternate server.
- *
- * Re-throws exceptions generated by the remote RPC call as either
- * RemoteException or IOException.
- *
- * @param ugi User group information.
- * @param namenodes A prioritized list of namenodes within the same
- * nameservice.
- * @param method Remote ClientProtcol method to invoke.
- * @param params Variable list of parameters matching the method.
- * @return The result of invoking the method.
- * @throws IOException
- */
- private Object invokeMethod(
- final UserGroupInformation ugi,
- final List<? extends FederationNamenodeContext> namenodes,
- final Method method, final Object... params) throws IOException {
-
- if (namenodes == null || namenodes.isEmpty()) {
- throw new IOException("No namenodes to invoke " + method.getName() +
- " with params " + Arrays.toString(params) + " from " + this.routerId);
- }
-
- Object ret = null;
- if (rpcMonitor != null) {
- rpcMonitor.proxyOp();
- }
- boolean failover = false;
- Map<FederationNamenodeContext, IOException> ioes = new LinkedHashMap<>();
- for (FederationNamenodeContext namenode : namenodes) {
- ConnectionContext connection = null;
- try {
- String nsId = namenode.getNameserviceId();
- String rpcAddress = namenode.getRpcAddress();
- connection = this.getConnection(ugi, nsId, rpcAddress);
- ProxyAndInfo<ClientProtocol> client = connection.getClient();
- ClientProtocol proxy = client.getProxy();
- ret = invoke(nsId, 0, method, proxy, params);
- if (failover) {
- // Success on alternate server, update
- InetSocketAddress address = client.getAddress();
- namenodeResolver.updateActiveNamenode(nsId, address);
- }
- if (this.rpcMonitor != null) {
- this.rpcMonitor.proxyOpComplete(true);
- }
- return ret;
- } catch (IOException ioe) {
- ioes.put(namenode, ioe);
- if (ioe instanceof StandbyException) {
- // Fail over indicated by retry policy and/or NN
- if (this.rpcMonitor != null) {
- this.rpcMonitor.proxyOpFailureStandby();
- }
- failover = true;
- } else if (ioe instanceof RemoteException) {
- if (this.rpcMonitor != null) {
- this.rpcMonitor.proxyOpComplete(true);
- }
- // RemoteException returned by NN
- throw (RemoteException) ioe;
- } else {
- // Other communication error, this is a failure
- // Communication retries are handled by the retry policy
- if (this.rpcMonitor != null) {
- this.rpcMonitor.proxyOpFailureCommunicate();
- this.rpcMonitor.proxyOpComplete(false);
- }
- throw ioe;
- }
- } finally {
- if (connection != null) {
- connection.release();
- }
- }
- }
- if (this.rpcMonitor != null) {
- this.rpcMonitor.proxyOpComplete(false);
- }
-
- // All namenodes were unavailable or in standby
- String msg = "No namenode available to invoke " + method.getName() + " " +
- Arrays.toString(params);
- LOG.error(msg);
- for (Entry<FederationNamenodeContext, IOException> entry :
- ioes.entrySet()) {
- FederationNamenodeContext namenode = entry.getKey();
- String nsId = namenode.getNameserviceId();
- String nnId = namenode.getNamenodeId();
- String addr = namenode.getRpcAddress();
- IOException ioe = entry.getValue();
- if (ioe instanceof StandbyException) {
- LOG.error("{} {} at {} is in Standby", nsId, nnId, addr);
- } else {
- LOG.error("{} {} at {} error: \"{}\"",
- nsId, nnId, addr, ioe.getMessage());
- }
- }
- throw new StandbyException(msg);
- }
-
- /**
- * Invokes a method on the designated object. Catches exceptions specific to
- * the invocation.
- *
- * Re-throws exceptions generated by the remote RPC call as either
- * RemoteException or IOException.
- *
- * @param nsId Identifier for the namespace
- * @param retryCount Current retry times
- * @param method Method to invoke
- * @param obj Target object for the method
- * @param params Variable parameters
- * @return Response from the remote server
- * @throws IOException
- * @throws InterruptedException
- */
- private Object invoke(String nsId, int retryCount, final Method method,
- final Object obj, final Object... params) throws IOException {
- try {
- return method.invoke(obj, params);
- } catch (IllegalAccessException e) {
- LOG.error("Unexpected exception while proxying API", e);
- return null;
- } catch (IllegalArgumentException e) {
- LOG.error("Unexpected exception while proxying API", e);
- return null;
- } catch (InvocationTargetException e) {
- Throwable cause = e.getCause();
- if (cause instanceof IOException) {
- IOException ioe = (IOException) cause;
-
- // Check if we should retry.
- RetryDecision decision = shouldRetry(ioe, retryCount, nsId);
- if (decision == RetryDecision.RETRY) {
- if (this.rpcMonitor != null) {
- this.rpcMonitor.proxyOpRetries();
- }
-
- // retry
- return invoke(nsId, ++retryCount, method, obj, params);
- } else if (decision == RetryDecision.FAILOVER_AND_RETRY) {
- // failover, invoker looks for standby exceptions for failover.
- if (ioe instanceof StandbyException) {
- throw ioe;
- } else {
- throw new StandbyException(ioe.getMessage());
- }
- } else {
- if (ioe instanceof RemoteException) {
- RemoteException re = (RemoteException) ioe;
- ioe = re.unwrapRemoteException();
- ioe = getCleanException(ioe);
- }
- throw ioe;
- }
- } else {
- throw new IOException(e);
- }
- }
- }
-
- /**
- * Check if the cluster of given nameservice id is available.
- * @param nsId nameservice ID.
- * @return
- * @throws IOException
- */
- private boolean isClusterUnAvailable(String nsId) throws IOException {
- List<? extends FederationNamenodeContext> nnState = this.namenodeResolver
- .getNamenodesForNameserviceId(nsId);
-
- if (nnState != null) {
- for (FederationNamenodeContext nnContext : nnState) {
- // Once we find one NN is in active state, we assume this
- // cluster is available.
- if (nnContext.getState() == FederationNamenodeServiceState.ACTIVE) {
- return false;
- }
- }
- }
-
- return true;
- }
-
- /**
- * Get a clean copy of the exception. Sometimes the exceptions returned by the
- * server contain the full stack trace in the message.
- *
- * @param ioe Exception to clean up.
- * @return Copy of the original exception with a clean message.
- */
- private static IOException getCleanException(IOException ioe) {
- IOException ret = null;
-
- String msg = ioe.getMessage();
- Throwable cause = ioe.getCause();
- StackTraceElement[] stackTrace = ioe.getStackTrace();
-
- // Clean the message by removing the stack trace
- int index = msg.indexOf("\n");
- if (index > 0) {
- String[] msgSplit = msg.split("\n");
- msg = msgSplit[0];
-
- // Parse stack trace from the message
- List<StackTraceElement> elements = new LinkedList<>();
- for (int i=1; i<msgSplit.length; i++) {
- String line = msgSplit[i];
- Matcher matcher = STACK_TRACE_PATTERN.matcher(line);
- if (matcher.find()) {
- String declaringClass = matcher.group(1);
- String methodName = matcher.group(2);
- String fileName = matcher.group(3);
- int lineNumber = Integer.parseInt(matcher.group(4));
- StackTraceElement element = new StackTraceElement(
- declaringClass, methodName, fileName, lineNumber);
- elements.add(element);
- }
- }
- stackTrace = elements.toArray(new StackTraceElement[elements.size()]);
- }
-
- // Create the new output exception
- if (ioe instanceof RemoteException) {
- RemoteException re = (RemoteException)ioe;
- ret = new RemoteException(re.getClassName(), msg);
- } else {
- // Try the simple constructor and initialize the fields
- Class<? extends IOException> ioeClass = ioe.getClass();
- try {
- Constructor<? extends IOException> constructor =
- ioeClass.getDeclaredConstructor(String.class);
- ret = constructor.newInstance(msg);
- } catch (ReflectiveOperationException e) {
- // If there are errors, just use the input one
- LOG.error("Could not create exception {}", ioeClass.getSimpleName(), e);
- ret = ioe;
- }
- }
- if (ret != null) {
- ret.initCause(cause);
- ret.setStackTrace(stackTrace);
- }
-
- return ret;
- }
-
- /**
- * Invokes a ClientProtocol method. Determines the target nameservice via a
- * provided block.
- *
- * Re-throws exceptions generated by the remote RPC call as either
- * RemoteException or IOException.
- *
- * @param block Block used to determine appropriate nameservice.
- * @param method The remote method and parameters to invoke.
- * @return The result of invoking the method.
- * @throws IOException
- */
- public Object invokeSingle(final ExtendedBlock block, RemoteMethod method)
- throws IOException {
- String bpId = block.getBlockPoolId();
- return invokeSingleBlockPool(bpId, method);
- }
-
- /**
- * Invokes a ClientProtocol method. Determines the target nameservice using
- * the block pool id.
- *
- * Re-throws exceptions generated by the remote RPC call as either
- * RemoteException or IOException.
- *
- * @param bpId Block pool identifier.
- * @param method The remote method and parameters to invoke.
- * @return The result of invoking the method.
- * @throws IOException
- */
- public Object invokeSingleBlockPool(final String bpId, RemoteMethod method)
- throws IOException {
- String nsId = getNameserviceForBlockPoolId(bpId);
- return invokeSingle(nsId, method);
- }
-
- /**
- * Invokes a ClientProtocol method against the specified namespace.
- *
- * Re-throws exceptions generated by the remote RPC call as either
- * RemoteException or IOException.
- *
- * @param nsId Target namespace for the method.
- * @param method The remote method and parameters to invoke.
- * @return The result of invoking the method.
- * @throws IOException
- */
- public Object invokeSingle(final String nsId, RemoteMethod method)
- throws IOException {
- UserGroupInformation ugi = RouterRpcServer.getRemoteUser();
- List<? extends FederationNamenodeContext> nns =
- getNamenodesForNameservice(nsId);
- RemoteLocationContext loc = new RemoteLocation(nsId, "/");
- return invokeMethod(ugi, nns, method.getMethod(), method.getParams(loc));
- }
-
- /**
- * Invokes a single proxy call for a single location.
- *
- * Re-throws exceptions generated by the remote RPC call as either
- * RemoteException or IOException.
- *
- * @param location RemoteLocation to invoke.
- * @param remoteMethod The remote method and parameters to invoke.
- * @return The result of invoking the method if successful.
- * @throws IOException
- */
- public Object invokeSingle(final RemoteLocationContext location,
- RemoteMethod remoteMethod) throws IOException {
- List<RemoteLocationContext> locations = Collections.singletonList(location);
- return invokeSequential(locations, remoteMethod);
- }
-
- /**
- * Invokes sequential proxy calls to different locations. Continues to invoke
- * calls until a call returns without throwing a remote exception.
- *
- * @param locations List of locations/nameservices to call concurrently.
- * @param remoteMethod The remote method and parameters to invoke.
- * @return The result of the first successful call, or if no calls are
- * successful, the result of the last RPC call executed.
- * @throws IOException if the success condition is not met and one of the RPC
- * calls generated a remote exception.
- */
- public Object invokeSequential(
- final List<? extends RemoteLocationContext> locations,
- final RemoteMethod remoteMethod) throws IOException {
- return invokeSequential(locations, remoteMethod, null, null);
- }
-
- /**
- * Invokes sequential proxy calls to different locations. Continues to invoke
- * calls until the success condition is met, or until all locations have been
- * attempted.
- *
- * The success condition may be specified by:
- * <ul>
- * <li>An expected result class
- * <li>An expected result value
- * </ul>
- *
- * If no expected result class/values are specified, the success condition is
- * a call that does not throw a remote exception.
- *
- * @param locations List of locations/nameservices to call concurrently.
- * @param remoteMethod The remote method and parameters to invoke.
- * @param expectedResultClass In order to be considered a positive result, the
- * return type must be of this class.
- * @param expectedResultValue In order to be considered a positive result, the
- * return value must equal the value of this object.
- * @return The result of the first successful call, or if no calls are
- * successful, the result of the first RPC call executed.
- * @throws IOException if the success condition is not met, return the first
- * remote exception generated.
- */
- public <T> T invokeSequential(
- final List<? extends RemoteLocationContext> locations,
- final RemoteMethod remoteMethod, Class<T> expectedResultClass,
- Object expectedResultValue) throws IOException {
-
- final UserGroupInformation ugi = RouterRpcServer.getRemoteUser();
- final Method m = remoteMethod.getMethod();
- IOException firstThrownException = null;
- IOException lastThrownException = null;
- Object firstResult = null;
- // Invoke in priority order
- for (final RemoteLocationContext loc : locations) {
- String ns = loc.getNameserviceId();
- List<? extends FederationNamenodeContext> namenodes =
- getNamenodesForNameservice(ns);
- try {
- Object[] params = remoteMethod.getParams(loc);
- Object result = invokeMethod(ugi, namenodes, m, params);
- // Check if the result is what we expected
- if (isExpectedClass(expectedResultClass, result) &&
- isExpectedValue(expectedResultValue, result)) {
- // Valid result, stop here
- @SuppressWarnings("unchecked")
- T ret = (T)result;
- return ret;
- }
- if (firstResult == null) {
- firstResult = result;
- }
- } catch (IOException ioe) {
- // Record it and move on
- lastThrownException = (IOException) ioe;
- if (firstThrownException == null) {
- firstThrownException = lastThrownException;
- }
- } catch (Exception e) {
- // Unusual error, ClientProtocol calls always use IOException (or
- // RemoteException). Re-wrap in IOException for compatibility with
- // ClientProtcol.
- LOG.error("Unexpected exception {} proxying {} to {}",
- e.getClass(), m.getName(), ns, e);
- lastThrownException = new IOException(
- "Unexpected exception proxying API " + e.getMessage(), e);
- if (firstThrownException == null) {
- firstThrownException = lastThrownException;
- }
- }
- }
-
- if (firstThrownException != null) {
- // re-throw the last exception thrown for compatibility
- throw firstThrownException;
- }
- // Return the last result, whether it is the value we are looking for or a
- @SuppressWarnings("unchecked")
- T ret = (T)firstResult;
- return ret;
- }
-
- /**
- * Checks if a result matches the required result class.
- *
- * @param expectedClass Required result class, null to skip the check.
- * @param clazz The result to check.
- * @return True if the result is an instance of the required class or if the
- * expected class is null.
- */
- private static boolean isExpectedClass(Class<?> expectedClass, Object clazz) {
- if (expectedClass == null) {
- return true;
- } else if (clazz == null) {
- return false;
- } else {
- return expectedClass.isInstance(clazz);
- }
- }
-
- /**
- * Checks if a result matches the expected value.
- *
- * @param expectedValue The expected value, null to skip the check.
- * @param value The result to check.
- * @return True if the result is equals to the expected value or if the
- * expected value is null.
- */
- private static boolean isExpectedValue(Object expectedValue, Object value) {
- if (expectedValue == null) {
- return true;
- } else if (value == null) {
- return false;
- } else {
- return value.equals(expectedValue);
- }
- }
-
- /**
- * Invoke multiple concurrent proxy calls to different clients. Returns an
- * array of results.
- *
- * Re-throws exceptions generated by the remote RPC call as either
- * RemoteException or IOException.
- *
- * @param <T> The type of the remote location.
- * @param locations List of remote locations to call concurrently.
- * @param method The remote method and parameters to invoke.
- * @param requireResponse If true an exception will be thrown if all calls do
- * not complete. If false exceptions are ignored and all data results
- * successfully received are returned.
- * @param standby If the requests should go to the standby namenodes too.
- * @throws IOException If all the calls throw an exception.
- */
- public <T extends RemoteLocationContext, R> void invokeConcurrent(
- final Collection<T> locations, final RemoteMethod method,
- boolean requireResponse, boolean standby) throws IOException {
- invokeConcurrent(locations, method, requireResponse, standby, void.class);
- }
-
- /**
- * Invokes multiple concurrent proxy calls to different clients. Returns an
- * array of results.
- *
- * Re-throws exceptions generated by the remote RPC call as either
- * RemoteException or IOException.
- *
- * @param <T> The type of the remote location.
- * @param <R> The type of the remote method return.
- * @param locations List of remote locations to call concurrently.
- * @param method The remote method and parameters to invoke.
- * @param requireResponse If true an exception will be thrown if all calls do
- * not complete. If false exceptions are ignored and all data results
- * successfully received are returned.
- * @param standby If the requests should go to the standby namenodes too.
- * @param clazz Type of the remote return type.
- * @return Result of invoking the method per subcluster: nsId -> result.
- * @throws IOException If requiredResponse=true and any of the calls throw an
- * exception.
- */
- public <T extends RemoteLocationContext, R> Map<T, R> invokeConcurrent(
- final Collection<T> locations, final RemoteMethod method,
- boolean requireResponse, boolean standby, Class<R> clazz)
- throws IOException {
- return invokeConcurrent(
- locations, method, requireResponse, standby, -1, clazz);
- }
-
- /**
- * Invokes multiple concurrent proxy calls to different clients. Returns an
- * array of results.
- *
- * Re-throws exceptions generated by the remote RPC call as either
- * RemoteException or IOException.
- *
- * @param <T> The type of the remote location.
- * @param <R> The type of the remote method return.
- * @param locations List of remote locations to call concurrently.
- * @param method The remote method and parameters to invoke.
- * @param requireResponse If true an exception will be thrown if all calls do
- * not complete. If false exceptions are ignored and all data results
- * successfully received are returned.
- * @param standby If the requests should go to the standby namenodes too.
- * @param timeOutMs Timeout for each individual call.
- * @param clazz Type of the remote return type.
- * @return Result of invoking the method per subcluster: nsId -> result.
- * @throws IOException If requiredResponse=true and any of the calls throw an
- * exception.
- */
- @SuppressWarnings("unchecked")
- public <T extends RemoteLocationContext, R> Map<T, R> invokeConcurrent(
- final Collection<T> locations, final RemoteMethod method,
- boolean requireResponse, boolean standby, long timeOutMs, Class<R> clazz)
- throws IOException {
-
- final UserGroupInformation ugi = RouterRpcServer.getRemoteUser();
- final Method m = method.getMethod();
-
- if (locations.size() == 1) {
- // Shortcut, just one call
- T location = locations.iterator().next();
- String ns = location.getNameserviceId();
- final List<? extends FederationNamenodeContext> namenodes =
- getNamenodesForNameservice(ns);
- Object[] paramList = method.getParams(location);
- Object result = invokeMethod(ugi, namenodes, m, paramList);
- return Collections.singletonMap(location, clazz.cast(result));
- }
-
- List<T> orderedLocations = new LinkedList<>();
- Set<Callable<Object>> callables = new HashSet<>();
- for (final T location : locations) {
- String nsId = location.getNameserviceId();
- final List<? extends FederationNamenodeContext> namenodes =
- getNamenodesForNameservice(nsId);
- final Object[] paramList = method.getParams(location);
- if (standby) {
- // Call the objectGetter to all NNs (including standby)
- for (final FederationNamenodeContext nn : namenodes) {
- String nnId = nn.getNamenodeId();
- final List<FederationNamenodeContext> nnList =
- Collections.singletonList(nn);
- T nnLocation = location;
- if (location instanceof RemoteLocation) {
- nnLocation = (T)new RemoteLocation(nsId, nnId, location.getDest());
- }
- orderedLocations.add(nnLocation);
- callables.add(new Callable<Object>() {
- public Object call() throws Exception {
- return invokeMethod(ugi, nnList, m, paramList);
- }
- });
- }
- } else {
- // Call the objectGetter in order of nameservices in the NS list
- orderedLocations.add(location);
- callables.add(new Callable<Object>() {
- public Object call() throws Exception {
- return invokeMethod(ugi, namenodes, m, paramList);
- }
- });
- }
- }
-
- if (rpcMonitor != null) {
- rpcMonitor.proxyOp();
- }
-
- try {
- List<Future<Object>> futures = null;
- if (timeOutMs > 0) {
- futures = executorService.invokeAll(
- callables, timeOutMs, TimeUnit.MILLISECONDS);
- } else {
- futures = executorService.invokeAll(callables);
- }
- Map<T, R> results = new TreeMap<>();
- Map<T, IOException> exceptions = new TreeMap<>();
- for (int i=0; i<futures.size(); i++) {
- T location = orderedLocations.get(i);
- try {
- Future<Object> future = futures.get(i);
- Object result = future.get();
- results.put(location, clazz.cast(result));
- } catch (CancellationException ce) {
- T loc = orderedLocations.get(i);
- String msg =
- "Invocation to \"" + loc + "\" for \"" + method + "\" timed out";
- LOG.error(msg);
- IOException ioe = new IOException(msg);
- exceptions.put(location, ioe);
- } catch (ExecutionException ex) {
- Throwable cause = ex.getCause();
- LOG.debug("Canot execute {} in {}: {}",
- m.getName(), location, cause.getMessage());
-
- // Convert into IOException if needed
- IOException ioe = null;
- if (cause instanceof IOException) {
- ioe = (IOException) cause;
- } else {
- ioe = new IOException("Unhandled exception while proxying API " +
- m.getName() + ": " + cause.getMessage(), cause);
- }
-
- // Response from all servers required, use this error.
- if (requireResponse) {
- throw ioe;
- }
-
- // Store the exceptions
- exceptions.put(location, ioe);
- }
- }
-
- // Throw the exception for the first location if there are no results
- if (results.isEmpty()) {
- T location = orderedLocations.get(0);
- IOException ioe = exceptions.get(location);
- if (ioe != null) {
- throw ioe;
- }
- }
-
- return results;
- } catch (InterruptedException ex) {
- LOG.error("Unexpected error while invoking API: {}", ex.getMessage());
- throw new IOException(
- "Unexpected error while invoking API " + ex.getMessage(), ex);
- }
- }
-
- /**
- * Get a prioritized list of NNs that share the same nameservice ID (in the
- * same namespace). NNs that are reported as ACTIVE will be first in the list.
- *
- * @param nsId The nameservice ID for the namespace.
- * @return A prioritized list of NNs to use for communication.
- * @throws IOException If a NN cannot be located for the nameservice ID.
- */
- private List<? extends FederationNamenodeContext> getNamenodesForNameservice(
- final String nsId) throws IOException {
-
- final List<? extends FederationNamenodeContext> namenodes =
- namenodeResolver.getNamenodesForNameserviceId(nsId);
-
- if (namenodes == null || namenodes.isEmpty()) {
- throw new IOException("Cannot locate a registered namenode for " + nsId +
- " from " + this.routerId);
- }
- return namenodes;
- }
-
- /**
- * Get a prioritized list of NNs that share the same block pool ID (in the
- * same namespace). NNs that are reported as ACTIVE will be first in the list.
- *
- * @param bpId The blockpool ID for the namespace.
- * @return A prioritized list of NNs to use for communication.
- * @throws IOException If a NN cannot be located for the block pool ID.
- */
- private List<? extends FederationNamenodeContext> getNamenodesForBlockPoolId(
- final String bpId) throws IOException {
-
- List<? extends FederationNamenodeContext> namenodes =
- namenodeResolver.getNamenodesForBlockPoolId(bpId);
-
- if (namenodes == null || namenodes.isEmpty()) {
- throw new IOException("Cannot locate a registered namenode for " + bpId +
- " from " + this.routerId);
- }
- return namenodes;
- }
-
- /**
- * Get the nameservice identifier for a block pool.
- *
- * @param bpId Identifier of the block pool.
- * @return Nameservice identifier.
- * @throws IOException If a NN cannot be located for the block pool ID.
- */
- private String getNameserviceForBlockPoolId(final String bpId)
- throws IOException {
- List<? extends FederationNamenodeContext> namenodes =
- getNamenodesForBlockPoolId(bpId);
- FederationNamenodeContext namenode = namenodes.get(0);
- return namenode.getNameserviceId();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcMonitor.java
deleted file mode 100644
index df9aa11..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcMonitor.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCMetrics;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
-
-/**
- * Metrics and monitoring interface for the router RPC server. Allows pluggable
- * diagnostics and monitoring services to be attached.
- */
-public interface RouterRpcMonitor {
-
- /**
- * Initialize the monitor.
- * @param conf Configuration for the monitor.
- * @param server RPC server.
- * @param store State Store.
- */
- void init(
- Configuration conf, RouterRpcServer server, StateStoreService store);
-
- /**
- * Get Router RPC metrics info.
- * @return The instance of FederationRPCMetrics.
- */
- FederationRPCMetrics getRPCMetrics();
-
- /**
- * Close the monitor.
- */
- void close();
-
- /**
- * Start processing an operation on the Router.
- */
- void startOp();
-
- /**
- * Start proxying an operation to the Namenode.
- * @return Id of the thread doing the proxying.
- */
- long proxyOp();
-
- /**
- * Mark a proxy operation as completed.
- * @param success If the operation was successful.
- */
- void proxyOpComplete(boolean success);
-
- /**
- * Failed to proxy an operation to a Namenode because it was in standby.
- */
- void proxyOpFailureStandby();
-
- /**
- * Failed to proxy an operation to a Namenode because of an unexpected
- * exception.
- */
- void proxyOpFailureCommunicate();
-
- /**
- * Failed to proxy an operation because it is not implemented.
- */
- void proxyOpNotImplemented();
-
- /**
- * Retry to proxy an operation to a Namenode because of an unexpected
- * exception.
- */
- void proxyOpRetries();
-
- /**
- * If the Router cannot contact the State Store in an operation.
- */
- void routerFailureStateStore();
-
- /**
- * If the Router is in safe mode.
- */
- void routerFailureSafemode();
-
- /**
- * If a path is locked.
- */
- void routerFailureLocked();
-
- /**
- * If a path is in a read only mount point.
- */
- void routerFailureReadOnly();
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetMountTableEntriesRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetMountTableEntriesRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetMountTableEntriesRequestPBImpl.java
deleted file mode 100644
index 3e0d1a6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetMountTableEntriesRequestPBImpl.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesRequestProtoOrBuilder;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * GetMountTableEntriesRequest.
- */
-public class GetMountTableEntriesRequestPBImpl
- extends GetMountTableEntriesRequest implements PBRecord {
-
- private FederationProtocolPBTranslator<GetMountTableEntriesRequestProto,
- GetMountTableEntriesRequestProto.Builder,
- GetMountTableEntriesRequestProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<GetMountTableEntriesRequestProto,
- GetMountTableEntriesRequestProto.Builder,
- GetMountTableEntriesRequestProtoOrBuilder>(
- GetMountTableEntriesRequestProto.class);
-
- public GetMountTableEntriesRequestPBImpl() {
- }
-
- public GetMountTableEntriesRequestPBImpl(
- GetMountTableEntriesRequestProto proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public GetMountTableEntriesRequestProto getProto() {
- return this.translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public String getSrcPath() {
- return this.translator.getProtoOrBuilder().getSrcPath();
- }
-
- @Override
- public void setSrcPath(String path) {
- this.translator.getBuilder().setSrcPath(path);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetMountTableEntriesResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetMountTableEntriesResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetMountTableEntriesResponsePBImpl.java
deleted file mode 100644
index 9d64bc9..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetMountTableEntriesResponsePBImpl.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesResponseProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesResponseProtoOrBuilder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProto;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.MountTablePBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * GetMountTableEntriesResponse.
- */
-public class GetMountTableEntriesResponsePBImpl
- extends GetMountTableEntriesResponse implements PBRecord {
-
- private FederationProtocolPBTranslator<GetMountTableEntriesResponseProto,
- GetMountTableEntriesResponseProto.Builder,
- GetMountTableEntriesResponseProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<GetMountTableEntriesResponseProto,
- GetMountTableEntriesResponseProto.Builder,
- GetMountTableEntriesResponseProtoOrBuilder>(
- GetMountTableEntriesResponseProto.class);
-
- public GetMountTableEntriesResponsePBImpl() {
- }
-
- public GetMountTableEntriesResponsePBImpl(
- GetMountTableEntriesResponseProto proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public GetMountTableEntriesResponseProto getProto() {
- return this.translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public List<MountTable> getEntries() throws IOException {
- List<MountTableRecordProto> entries =
- this.translator.getProtoOrBuilder().getEntriesList();
- List<MountTable> ret = new ArrayList<MountTable>();
- for (MountTableRecordProto entry : entries) {
- MountTable record = new MountTablePBImpl(entry);
- ret.add(record);
- }
- return ret;
- }
-
- @Override
- public void setEntries(List<MountTable> records) throws IOException {
- this.translator.getBuilder().clearEntries();
- for (MountTable entry : records) {
- if (entry instanceof MountTablePBImpl) {
- MountTablePBImpl entryPB = (MountTablePBImpl)entry;
- this.translator.getBuilder().addEntries(entryPB.getProto());
- }
- }
- }
-
- @Override
- public long getTimestamp() {
- return this.translator.getProtoOrBuilder().getTimestamp();
- }
-
- @Override
- public void setTimestamp(long time) {
- this.translator.getBuilder().setTimestamp(time);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamenodeRegistrationsRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamenodeRegistrationsRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamenodeRegistrationsRequestPBImpl.java
deleted file mode 100644
index 4f7fee1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamenodeRegistrationsRequestPBImpl.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetNamenodeRegistrationsRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetNamenodeRegistrationsRequestProtoOrBuilder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipRecordProto;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsRequest;
-import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.MembershipStatePBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * GetNamenodeRegistrationsRequest.
- */
-public class GetNamenodeRegistrationsRequestPBImpl
- extends GetNamenodeRegistrationsRequest implements PBRecord {
-
- private FederationProtocolPBTranslator<GetNamenodeRegistrationsRequestProto,
- GetNamenodeRegistrationsRequestProto.Builder,
- GetNamenodeRegistrationsRequestProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<
- GetNamenodeRegistrationsRequestProto,
- GetNamenodeRegistrationsRequestProto.Builder,
- GetNamenodeRegistrationsRequestProtoOrBuilder>(
- GetNamenodeRegistrationsRequestProto.class);
-
- public GetNamenodeRegistrationsRequestPBImpl() {
- }
-
- public GetNamenodeRegistrationsRequestPBImpl(
- GetNamenodeRegistrationsRequestProto proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public GetNamenodeRegistrationsRequestProto getProto() {
- return this.translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public MembershipState getPartialMembership() {
- GetNamenodeRegistrationsRequestProtoOrBuilder proto =
- this.translator.getProtoOrBuilder();
- if (!proto.hasMembership()){
- return null;
- }
- NamenodeMembershipRecordProto memberProto = proto.getMembership();
- return new MembershipStatePBImpl(memberProto);
- }
-
- @Override
- public void setPartialMembership(MembershipState member) {
- MembershipStatePBImpl memberPB = (MembershipStatePBImpl)member;
- this.translator.getBuilder().setMembership(memberPB.getProto());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamenodeRegistrationsResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamenodeRegistrationsResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamenodeRegistrationsResponsePBImpl.java
deleted file mode 100644
index f6be11d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamenodeRegistrationsResponsePBImpl.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetNamenodeRegistrationsResponseProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetNamenodeRegistrationsResponseProtoOrBuilder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipRecordProto;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.MembershipStatePBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * GetNamenodeRegistrationsResponse.
- */
-public class GetNamenodeRegistrationsResponsePBImpl
- extends GetNamenodeRegistrationsResponse implements PBRecord {
-
- private FederationProtocolPBTranslator<GetNamenodeRegistrationsResponseProto,
- GetNamenodeRegistrationsResponseProto.Builder,
- GetNamenodeRegistrationsResponseProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<
- GetNamenodeRegistrationsResponseProto,
- GetNamenodeRegistrationsResponseProto.Builder,
- GetNamenodeRegistrationsResponseProtoOrBuilder>(
- GetNamenodeRegistrationsResponseProto.class);
-
- public GetNamenodeRegistrationsResponsePBImpl() {
- }
-
- public GetNamenodeRegistrationsResponsePBImpl(
- GetNamenodeRegistrationsResponseProto proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public GetNamenodeRegistrationsResponseProto getProto() {
- return this.translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public List<MembershipState> getNamenodeMemberships()
- throws IOException {
-
- List<MembershipState> ret = new ArrayList<MembershipState>();
- List<NamenodeMembershipRecordProto> memberships =
- this.translator.getProtoOrBuilder().getNamenodeMembershipsList();
- for (NamenodeMembershipRecordProto memberProto : memberships) {
- MembershipState membership = new MembershipStatePBImpl(memberProto);
- ret.add(membership);
- }
-
- return ret;
- }
-
- @Override
- public void setNamenodeMemberships(List<MembershipState> records)
- throws IOException {
- for (MembershipState member : records) {
- if (member instanceof MembershipStatePBImpl) {
- MembershipStatePBImpl memberPB = (MembershipStatePBImpl)member;
- this.translator.getBuilder().addNamenodeMemberships(
- memberPB.getProto());
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamespaceInfoRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamespaceInfoRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamespaceInfoRequestPBImpl.java
deleted file mode 100644
index 5f3e186..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamespaceInfoRequestPBImpl.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetNamespaceInfoRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetNamespaceInfoRequestProto.Builder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetNamespaceInfoRequestProtoOrBuilder;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoRequest;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * GetNamespaceInfoRequest.
- */
-public class GetNamespaceInfoRequestPBImpl extends GetNamespaceInfoRequest
- implements PBRecord {
-
- private FederationProtocolPBTranslator<GetNamespaceInfoRequestProto,
- Builder, GetNamespaceInfoRequestProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<GetNamespaceInfoRequestProto,
- Builder, GetNamespaceInfoRequestProtoOrBuilder>(
- GetNamespaceInfoRequestProto.class);
-
- public GetNamespaceInfoRequestPBImpl() {
- }
-
- @Override
- public GetNamespaceInfoRequestProto getProto() {
- return this.translator.build();
- }
-
- @Override
- public void setProto(Message protocol) {
- this.translator.setProto(protocol);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamespaceInfoResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamespaceInfoResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamespaceInfoResponsePBImpl.java
deleted file mode 100644
index be1b184..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetNamespaceInfoResponsePBImpl.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.FederationNamespaceInfoProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetNamespaceInfoResponseProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetNamespaceInfoResponseProtoOrBuilder;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * GetNamespaceInfoResponse.
- */
-public class GetNamespaceInfoResponsePBImpl
- extends GetNamespaceInfoResponse implements PBRecord {
-
- private FederationProtocolPBTranslator<GetNamespaceInfoResponseProto,
- GetNamespaceInfoResponseProto.Builder,
- GetNamespaceInfoResponseProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<GetNamespaceInfoResponseProto,
- GetNamespaceInfoResponseProto.Builder,
- GetNamespaceInfoResponseProtoOrBuilder>(
- GetNamespaceInfoResponseProto.class);
-
- public GetNamespaceInfoResponsePBImpl() {
- }
-
- @Override
- public GetNamespaceInfoResponseProto getProto() {
- return this.translator.build();
- }
-
- @Override
- public void setProto(Message protocol) {
- this.translator.setProto(protocol);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public Set<FederationNamespaceInfo> getNamespaceInfo() {
-
- Set<FederationNamespaceInfo> ret = new HashSet<FederationNamespaceInfo>();
- List<FederationNamespaceInfoProto> namespaceList =
- this.translator.getProtoOrBuilder().getNamespaceInfosList();
- for (FederationNamespaceInfoProto ns : namespaceList) {
- FederationNamespaceInfo info = new FederationNamespaceInfo(
- ns.getBlockPoolId(), ns.getClusterId(), ns.getNameserviceId());
- ret.add(info);
- }
- return ret;
- }
-
- @Override
- public void setNamespaceInfo(Set<FederationNamespaceInfo> namespaceInfo) {
- int index = 0;
- for (FederationNamespaceInfo item : namespaceInfo) {
- FederationNamespaceInfoProto.Builder itemBuilder =
- FederationNamespaceInfoProto.newBuilder();
- itemBuilder.setClusterId(item.getClusterId());
- itemBuilder.setBlockPoolId(item.getBlockPoolId());
- itemBuilder.setNameserviceId(item.getNameserviceId());
- this.translator.getBuilder().addNamespaceInfos(index,
- itemBuilder.build());
- index++;
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationRequestPBImpl.java
deleted file mode 100644
index 19bf4ec..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationRequestPBImpl.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationRequestProto.Builder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationRequestProtoOrBuilder;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationRequest;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * GetRouterRegistrationRequest.
- */
-public class GetRouterRegistrationRequestPBImpl
- extends GetRouterRegistrationRequest implements PBRecord {
-
- private FederationProtocolPBTranslator<GetRouterRegistrationRequestProto,
- Builder, GetRouterRegistrationRequestProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<GetRouterRegistrationRequestProto,
- Builder, GetRouterRegistrationRequestProtoOrBuilder>(
- GetRouterRegistrationRequestProto.class);
-
- public GetRouterRegistrationRequestPBImpl() {
- }
-
- @Override
- public GetRouterRegistrationRequestProto getProto() {
- return this.translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public String getRouterId() {
- return this.translator.getProtoOrBuilder().getRouterId();
- }
-
- @Override
- public void setRouterId(String routerId) {
- this.translator.getBuilder().setRouterId(routerId);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationResponsePBImpl.java
deleted file mode 100644
index ae17e92..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationResponsePBImpl.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationResponseProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationResponseProto.Builder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationResponseProtoOrBuilder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterRecordProto;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.RouterStatePBImpl;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * GetRouterRegistrationResponse.
- */
-public class GetRouterRegistrationResponsePBImpl
- extends GetRouterRegistrationResponse implements PBRecord {
-
- private FederationProtocolPBTranslator<GetRouterRegistrationResponseProto,
- Builder, GetRouterRegistrationResponseProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<
- GetRouterRegistrationResponseProto, Builder,
- GetRouterRegistrationResponseProtoOrBuilder>(
- GetRouterRegistrationResponseProto.class);
-
- public GetRouterRegistrationResponsePBImpl() {
- }
-
- @Override
- public GetRouterRegistrationResponseProto getProto() {
- return this.translator.build();
- }
-
- @Override
- public void setProto(Message protocol) {
- this.translator.setProto(protocol);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public RouterState getRouter() {
- RouterRecordProto proto = this.translator.getProtoOrBuilder().getRouter();
- return new RouterStatePBImpl(proto);
- }
-
- @Override
- public void setRouter(RouterState router) {
- if (router instanceof RouterStatePBImpl) {
- RouterStatePBImpl routerPB = (RouterStatePBImpl)router;
- RouterRecordProto routerProto = routerPB.getProto();
- this.translator.getBuilder().setRouter(routerProto);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationsRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationsRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationsRequestPBImpl.java
deleted file mode 100644
index 4b48682..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationsRequestPBImpl.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationsRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationsRequestProto.Builder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationsRequestProtoOrBuilder;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationsRequest;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * GetRouterRegistrationsRequest.
- */
-public class GetRouterRegistrationsRequestPBImpl
- extends GetRouterRegistrationsRequest implements PBRecord {
-
- private FederationProtocolPBTranslator<GetRouterRegistrationsRequestProto,
- Builder, GetRouterRegistrationsRequestProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<
- GetRouterRegistrationsRequestProto, Builder,
- GetRouterRegistrationsRequestProtoOrBuilder>(
- GetRouterRegistrationsRequestProto.class);
-
- public GetRouterRegistrationsRequestPBImpl() {
- }
-
- @Override
- public GetRouterRegistrationsRequestProto getProto() {
- return this.translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationsResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationsResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationsResponsePBImpl.java
deleted file mode 100644
index 2d597fb..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetRouterRegistrationsResponsePBImpl.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationsResponseProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationsResponseProto.Builder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetRouterRegistrationsResponseProtoOrBuilder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterRecordProto;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationsResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.RouterStatePBImpl;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * GetRouterRegistrationsResponse.
- */
-public class GetRouterRegistrationsResponsePBImpl
- extends GetRouterRegistrationsResponse implements PBRecord {
-
- private FederationProtocolPBTranslator<GetRouterRegistrationsResponseProto,
- Builder, GetRouterRegistrationsResponseProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<
- GetRouterRegistrationsResponseProto, Builder,
- GetRouterRegistrationsResponseProtoOrBuilder>(
- GetRouterRegistrationsResponseProto.class);
-
- public GetRouterRegistrationsResponsePBImpl() {
-
- }
-
- @Override
- public GetRouterRegistrationsResponseProto getProto() {
- return this.translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public List<RouterState> getRouters() throws IOException {
-
- List<RouterState> ret = new ArrayList<RouterState>();
- List<RouterRecordProto> memberships =
- this.translator.getProtoOrBuilder().getRoutersList();
- for (RouterRecordProto memberProto : memberships) {
- RouterState membership = new RouterStatePBImpl(memberProto);
- ret.add(membership);
- }
- return ret;
- }
-
- @Override
- public void setRouters(List<RouterState> records) throws IOException {
-
- this.translator.getBuilder().clearRouters();
- for (RouterState router : records) {
- if (router instanceof RouterStatePBImpl) {
- RouterStatePBImpl routerPB = (RouterStatePBImpl) router;
- this.translator.getBuilder().addRouters(routerPB.getProto());
- }
- }
- }
-
- @Override
- public long getTimestamp() {
- return this.translator.getProtoOrBuilder().getTimestamp();
- }
-
- @Override
- public void setTimestamp(long time) {
- this.translator.getBuilder().setTimestamp(time);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetSafeModeRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetSafeModeRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetSafeModeRequestPBImpl.java
deleted file mode 100644
index 1a4f63b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetSafeModeRequestPBImpl.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetSafeModeRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetSafeModeRequestProto.Builder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetSafeModeRequestProtoOrBuilder;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeRequest;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * GetSafeModeRequest.
- */
-public class GetSafeModeRequestPBImpl extends GetSafeModeRequest
- implements PBRecord {
-
- private FederationProtocolPBTranslator<GetSafeModeRequestProto,
- Builder, GetSafeModeRequestProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<>(GetSafeModeRequestProto.class);
-
- public GetSafeModeRequestPBImpl() {
- }
-
- public GetSafeModeRequestPBImpl(GetSafeModeRequestProto proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public GetSafeModeRequestProto getProto() {
- return translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetSafeModeResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetSafeModeResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetSafeModeResponsePBImpl.java
deleted file mode 100644
index 77fbeca..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/GetSafeModeResponsePBImpl.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetSafeModeResponseProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetSafeModeResponseProto.Builder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetSafeModeResponseProtoOrBuilder;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * GetSafeModeResponse.
- */
-public class GetSafeModeResponsePBImpl extends GetSafeModeResponse
- implements PBRecord {
-
- private FederationProtocolPBTranslator<GetSafeModeResponseProto,
- Builder, GetSafeModeResponseProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<>(
- GetSafeModeResponseProto.class);
-
- public GetSafeModeResponsePBImpl() {
- }
-
- public GetSafeModeResponsePBImpl(GetSafeModeResponseProto proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public GetSafeModeResponseProto getProto() {
- return translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public boolean isInSafeMode() {
- return this.translator.getProtoOrBuilder().getIsInSafeMode();
- }
-
- @Override
- public void setSafeMode(boolean isInSafeMode) {
- this.translator.getBuilder().setIsInSafeMode(isInSafeMode);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/LeaveSafeModeRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/LeaveSafeModeRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/LeaveSafeModeRequestPBImpl.java
deleted file mode 100644
index 075099b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/LeaveSafeModeRequestPBImpl.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.LeaveSafeModeRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.LeaveSafeModeRequestProto.Builder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.LeaveSafeModeRequestProtoOrBuilder;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeRequest;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * LeaveSafeModeRequest.
- */
-public class LeaveSafeModeRequestPBImpl extends LeaveSafeModeRequest
- implements PBRecord {
-
- private FederationProtocolPBTranslator<LeaveSafeModeRequestProto,
- Builder, LeaveSafeModeRequestProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<>(LeaveSafeModeRequestProto.class);
-
- public LeaveSafeModeRequestPBImpl() {
- }
-
- public LeaveSafeModeRequestPBImpl(LeaveSafeModeRequestProto proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public LeaveSafeModeRequestProto getProto() {
- return translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/LeaveSafeModeResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/LeaveSafeModeResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/LeaveSafeModeResponsePBImpl.java
deleted file mode 100644
index 79f65fe..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/LeaveSafeModeResponsePBImpl.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.LeaveSafeModeResponseProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.LeaveSafeModeResponseProto.Builder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.LeaveSafeModeResponseProtoOrBuilder;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * LeaveSafeModeResponse.
- */
-public class LeaveSafeModeResponsePBImpl extends LeaveSafeModeResponse
- implements PBRecord {
-
- private FederationProtocolPBTranslator<LeaveSafeModeResponseProto,
- Builder, LeaveSafeModeResponseProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<>(
- LeaveSafeModeResponseProto.class);
-
- public LeaveSafeModeResponsePBImpl() {
- }
-
- public LeaveSafeModeResponsePBImpl(LeaveSafeModeResponseProto proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public LeaveSafeModeResponseProto getProto() {
- return translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public boolean getStatus() {
- return this.translator.getProtoOrBuilder().getStatus();
- }
-
- @Override
- public void setStatus(boolean result) {
- this.translator.getBuilder().setStatus(result);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/NamenodeHeartbeatRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/NamenodeHeartbeatRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/NamenodeHeartbeatRequestPBImpl.java
deleted file mode 100644
index d1fc73f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/NamenodeHeartbeatRequestPBImpl.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeHeartbeatRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeHeartbeatRequestProto.Builder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeHeartbeatRequestProtoOrBuilder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipRecordProto;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatRequest;
-import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.MembershipStatePBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * NamenodeHeartbeatRequest.
- */
-public class NamenodeHeartbeatRequestPBImpl
- extends NamenodeHeartbeatRequest implements PBRecord {
-
- private FederationProtocolPBTranslator<NamenodeHeartbeatRequestProto, Builder,
- NamenodeHeartbeatRequestProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<NamenodeHeartbeatRequestProto,
- Builder,
- NamenodeHeartbeatRequestProtoOrBuilder>(
- NamenodeHeartbeatRequestProto.class);
-
- public NamenodeHeartbeatRequestPBImpl() {
- }
-
- @Override
- public NamenodeHeartbeatRequestProto getProto() {
- return this.translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public MembershipState getNamenodeMembership() throws IOException {
- NamenodeMembershipRecordProto membershipProto =
- this.translator.getProtoOrBuilder().getNamenodeMembership();
- MembershipState membership =
- StateStoreSerializer.newRecord(MembershipState.class);
- if (membership instanceof MembershipStatePBImpl) {
- MembershipStatePBImpl membershipPB = (MembershipStatePBImpl)membership;
- membershipPB.setProto(membershipProto);
- return membershipPB;
- } else {
- throw new IOException("Cannot get membership from request");
- }
- }
-
- @Override
- public void setNamenodeMembership(MembershipState membership)
- throws IOException {
- if (membership instanceof MembershipStatePBImpl) {
- MembershipStatePBImpl membershipPB = (MembershipStatePBImpl)membership;
- NamenodeMembershipRecordProto membershipProto =
- (NamenodeMembershipRecordProto)membershipPB.getProto();
- this.translator.getBuilder().setNamenodeMembership(membershipProto);
- } else {
- throw new IOException("Cannot set mount table entry");
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/NamenodeHeartbeatResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/NamenodeHeartbeatResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/NamenodeHeartbeatResponsePBImpl.java
deleted file mode 100644
index c243a6f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/NamenodeHeartbeatResponsePBImpl.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeHeartbeatResponseProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeHeartbeatResponseProtoOrBuilder;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * NamenodeHeartbeatResponse.
- */
-public class NamenodeHeartbeatResponsePBImpl extends NamenodeHeartbeatResponse
- implements PBRecord {
-
- private FederationProtocolPBTranslator<NamenodeHeartbeatResponseProto,
- NamenodeHeartbeatResponseProto.Builder,
- NamenodeHeartbeatResponseProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<NamenodeHeartbeatResponseProto,
- NamenodeHeartbeatResponseProto.Builder,
- NamenodeHeartbeatResponseProtoOrBuilder>(
- NamenodeHeartbeatResponseProto.class);
-
- public NamenodeHeartbeatResponsePBImpl() {
- }
-
- @Override
- public NamenodeHeartbeatResponseProto getProto() {
- return this.translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public boolean getResult() {
- return this.translator.getProtoOrBuilder().getStatus();
- }
-
- @Override
- public void setResult(boolean result) {
- this.translator.getBuilder().setStatus(result);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntryRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntryRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntryRequestPBImpl.java
deleted file mode 100644
index 7f7c998..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntryRequestPBImpl.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryRequestProtoOrBuilder;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * RemoveMountTableEntryRequest.
- */
-public class RemoveMountTableEntryRequestPBImpl
- extends RemoveMountTableEntryRequest implements PBRecord {
-
- private FederationProtocolPBTranslator<RemoveMountTableEntryRequestProto,
- RemoveMountTableEntryRequestProto.Builder,
- RemoveMountTableEntryRequestProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<RemoveMountTableEntryRequestProto,
- RemoveMountTableEntryRequestProto.Builder,
- RemoveMountTableEntryRequestProtoOrBuilder>(
- RemoveMountTableEntryRequestProto.class);
-
- public RemoveMountTableEntryRequestPBImpl() {
- }
-
- public RemoveMountTableEntryRequestPBImpl(
- RemoveMountTableEntryRequestProto proto) {
- this.setProto(proto);
- }
-
- @Override
- public RemoveMountTableEntryRequestProto getProto() {
- return this.translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public String getSrcPath() {
- return this.translator.getProtoOrBuilder().getSrcPath();
- }
-
- @Override
- public void setSrcPath(String path) {
- this.translator.getBuilder().setSrcPath(path);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntryResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntryResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntryResponsePBImpl.java
deleted file mode 100644
index 0c943ac..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntryResponsePBImpl.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryResponseProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryResponseProto.Builder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryResponseProtoOrBuilder;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * RemoveMountTableEntryResponse.
- */
-public class RemoveMountTableEntryResponsePBImpl
- extends RemoveMountTableEntryResponse implements PBRecord {
-
- private FederationProtocolPBTranslator<RemoveMountTableEntryResponseProto,
- Builder, RemoveMountTableEntryResponseProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<RemoveMountTableEntryResponseProto,
- RemoveMountTableEntryResponseProto.Builder,
- RemoveMountTableEntryResponseProtoOrBuilder>(
- RemoveMountTableEntryResponseProto.class);
-
- public RemoveMountTableEntryResponsePBImpl() {
- }
-
- public RemoveMountTableEntryResponsePBImpl(
- RemoveMountTableEntryResponseProto proto) {
- this.setProto(proto);
- }
-
- @Override
- public RemoveMountTableEntryResponseProto getProto() {
- return this.translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public boolean getStatus() {
- return this.translator.getProtoOrBuilder().getStatus();
- }
-
- @Override
- public void setStatus(boolean result) {
- this.translator.getBuilder().setStatus(result);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RouterHeartbeatRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RouterHeartbeatRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RouterHeartbeatRequestPBImpl.java
deleted file mode 100644
index cc2be04..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RouterHeartbeatRequestPBImpl.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterHeartbeatRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterHeartbeatRequestProto.Builder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterHeartbeatRequestProtoOrBuilder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterRecordProto;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatRequest;
-import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.RouterStatePBImpl;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * RouterHeartbeatRequest.
- */
-public class RouterHeartbeatRequestPBImpl extends RouterHeartbeatRequest
- implements PBRecord {
-
- private FederationProtocolPBTranslator<RouterHeartbeatRequestProto, Builder,
- RouterHeartbeatRequestProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<RouterHeartbeatRequestProto,
- Builder, RouterHeartbeatRequestProtoOrBuilder>(
- RouterHeartbeatRequestProto.class);
-
- public RouterHeartbeatRequestPBImpl() {
- }
-
- @Override
- public RouterHeartbeatRequestProto getProto() {
- return this.translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public RouterState getRouter() throws IOException {
- RouterRecordProto routerProto =
- this.translator.getProtoOrBuilder().getRouter();
- return new RouterStatePBImpl(routerProto);
- }
-
- @Override
- public void setRouter(RouterState routerState) {
- if (routerState instanceof RouterStatePBImpl) {
- RouterStatePBImpl routerStatePB = (RouterStatePBImpl)routerState;
- this.translator.getBuilder().setRouter(routerStatePB.getProto());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RouterHeartbeatResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RouterHeartbeatResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RouterHeartbeatResponsePBImpl.java
deleted file mode 100644
index ac534fd..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RouterHeartbeatResponsePBImpl.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterHeartbeatResponseProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterHeartbeatResponseProto.Builder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterHeartbeatResponseProtoOrBuilder;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * RouterHeartbeatResponse.
- */
-public class RouterHeartbeatResponsePBImpl extends RouterHeartbeatResponse
- implements PBRecord {
-
- private FederationProtocolPBTranslator<RouterHeartbeatResponseProto, Builder,
- RouterHeartbeatResponseProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<RouterHeartbeatResponseProto,
- Builder, RouterHeartbeatResponseProtoOrBuilder>(
- RouterHeartbeatResponseProto.class);
-
- public RouterHeartbeatResponsePBImpl() {
- }
-
- @Override
- public RouterHeartbeatResponseProto getProto() {
- return this.translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public boolean getStatus() {
- return this.translator.getProtoOrBuilder().getStatus();
- }
-
- @Override
- public void setStatus(boolean result) {
- this.translator.getBuilder().setStatus(result);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateMountTableEntryRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateMountTableEntryRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateMountTableEntryRequestPBImpl.java
deleted file mode 100644
index 621bb3a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateMountTableEntryRequestPBImpl.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryRequestProtoOrBuilder;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.MountTablePBImpl;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * UpdateMountTableEntryRequest.
- */
-public class UpdateMountTableEntryRequestPBImpl
- extends UpdateMountTableEntryRequest implements PBRecord {
-
- private FederationProtocolPBTranslator<UpdateMountTableEntryRequestProto,
- UpdateMountTableEntryRequestProto.Builder,
- UpdateMountTableEntryRequestProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<UpdateMountTableEntryRequestProto,
- UpdateMountTableEntryRequestProto.Builder,
- UpdateMountTableEntryRequestProtoOrBuilder>(
- UpdateMountTableEntryRequestProto.class);
-
- public UpdateMountTableEntryRequestPBImpl() {
- }
-
- public UpdateMountTableEntryRequestPBImpl(
- UpdateMountTableEntryRequestProto proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public UpdateMountTableEntryRequestProto getProto() {
- return this.translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public MountTable getEntry() throws IOException {
- MountTableRecordProto statsProto =
- this.translator.getProtoOrBuilder().getEntry();
- MountTable stats = StateStoreSerializer.newRecord(MountTable.class);
- if (stats instanceof MountTablePBImpl) {
- MountTablePBImpl entryPB = (MountTablePBImpl)stats;
- entryPB.setProto(statsProto);
- return entryPB;
- } else {
- throw new IOException("Cannot get stats for the membership");
- }
- }
-
- @Override
- public void setEntry(MountTable mount) throws IOException {
- if (mount instanceof MountTablePBImpl) {
- MountTablePBImpl mountPB = (MountTablePBImpl)mount;
- MountTableRecordProto mountProto =
- (MountTableRecordProto)mountPB.getProto();
- this.translator.getBuilder().setEntry(mountProto);
- } else {
- throw new IOException("Cannot set mount table entry");
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateMountTableEntryResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateMountTableEntryResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateMountTableEntryResponsePBImpl.java
deleted file mode 100644
index 5d566d6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateMountTableEntryResponsePBImpl.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryResponseProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryResponseProtoOrBuilder;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * UpdateMountTableEntryResponse.
- */
-public class UpdateMountTableEntryResponsePBImpl
- extends UpdateMountTableEntryResponse implements PBRecord {
-
- private FederationProtocolPBTranslator<UpdateMountTableEntryResponseProto,
- UpdateMountTableEntryResponseProto.Builder,
- UpdateMountTableEntryResponseProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<UpdateMountTableEntryResponseProto,
- UpdateMountTableEntryResponseProto.Builder,
- UpdateMountTableEntryResponseProtoOrBuilder>(
- UpdateMountTableEntryResponseProto.class);
-
- public UpdateMountTableEntryResponsePBImpl() {
- }
-
- public UpdateMountTableEntryResponsePBImpl(
- UpdateMountTableEntryResponseProto proto) {
- this.setProto(proto);
- }
-
- @Override
- public UpdateMountTableEntryResponseProto getProto() {
- return this.translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public boolean getStatus() {
- return this.translator.getProtoOrBuilder().getStatus();
- }
-
- @Override
- public void setStatus(boolean result) {
- this.translator.getBuilder().setStatus(result);
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
new file mode 100644
index 0000000..2b12a25
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
@@ -0,0 +1,1072 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.addDirectory;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.countContents;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createFile;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.deleteFile;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.getFileStatus;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyFileExists;
+import static org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.TEST_STRING;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.CryptoProtocolVersion;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Options;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
+import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
+import org.apache.hadoop.io.EnumSetWritable;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.Service.STATE;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Supplier;
+import com.google.common.collect.Maps;
+
+/**
+ * The the RPC interface of the {@link Router} implemented by
+ * {@link RouterRpcServer}.
+ */
+public class TestRouterRpc {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestRouterRpc.class);
+
+ private static final Comparator<ErasureCodingPolicyInfo> EC_POLICY_CMP =
+ new Comparator<ErasureCodingPolicyInfo>() {
+ public int compare(
+ ErasureCodingPolicyInfo ec0,
+ ErasureCodingPolicyInfo ec1) {
+ String name0 = ec0.getPolicy().getName();
+ String name1 = ec1.getPolicy().getName();
+ return name0.compareTo(name1);
+ }
+ };
+
+ /** Federated HDFS cluster. */
+ private static RouterDFSCluster cluster;
+
+ /** Random Router for this federated cluster. */
+ private RouterContext router;
+
+ /** Random nameservice in the federated cluster. */
+ private String ns;
+ /** First namenode in the nameservice. */
+ private NamenodeContext namenode;
+
+ /** Client interface to the Router. */
+ private ClientProtocol routerProtocol;
+ /** Client interface to the Namenode. */
+ private ClientProtocol nnProtocol;
+
+ /** Filesystem interface to the Router. */
+ private FileSystem routerFS;
+ /** Filesystem interface to the Namenode. */
+ private FileSystem nnFS;
+
+ /** File in the Router. */
+ private String routerFile;
+ /** File in the Namenode. */
+ private String nnFile;
+
+
+ @BeforeClass
+ public static void globalSetUp() throws Exception {
+ cluster = new RouterDFSCluster(false, 2);
+ // We need 6 DNs to test Erasure Coding with RS-6-3-64k
+ cluster.setNumDatanodesPerNameservice(6);
+
+ // Start NNs and DNs and wait until ready
+ cluster.startCluster();
+
+ // Start routers with only an RPC service
+ cluster.addRouterOverrides((new RouterConfigBuilder()).rpc().build());
+ cluster.startRouters();
+
+ // Register and verify all NNs with all routers
+ cluster.registerNamenodes();
+ cluster.waitNamenodeRegistration();
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ cluster.shutdown();
+ }
+
+ @Before
+ public void testSetup() throws Exception {
+
+ // Create mock locations
+ cluster.installMockLocations();
+
+ // Delete all files via the NNs and verify
+ cluster.deleteAllFiles();
+
+ // Create test fixtures on NN
+ cluster.createTestDirectoriesNamenode();
+
+ // Wait to ensure NN has fully created its test directories
+ Thread.sleep(100);
+
+ // Default namenode and random router for this test
+ this.router = cluster.getRandomRouter();
+ this.ns = cluster.getNameservices().get(0);
+ this.namenode = cluster.getNamenode(ns, null);
+
+ // Handles to the ClientProtocol interface
+ this.routerProtocol = router.getClient().getNamenode();
+ this.nnProtocol = namenode.getClient().getNamenode();
+
+ // Handles to the filesystem client
+ this.nnFS = namenode.getFileSystem();
+ this.routerFS = router.getFileSystem();
+
+ // Create a test file on the NN
+ Random r = new Random();
+ String randomFile = "testfile-" + r.nextInt();
+ this.nnFile =
+ cluster.getNamenodeTestDirectoryForNS(ns) + "/" + randomFile;
+ this.routerFile =
+ cluster.getFederatedTestDirectoryForNS(ns) + "/" + randomFile;
+
+ createFile(nnFS, nnFile, 32);
+ verifyFileExists(nnFS, nnFile);
+ }
+
+ @Test
+ public void testRpcService() throws IOException {
+ Router testRouter = new Router();
+ List<String> nss = cluster.getNameservices();
+ String ns0 = nss.get(0);
+ Configuration routerConfig = cluster.generateRouterConfiguration(ns0, null);
+ RouterRpcServer server = new RouterRpcServer(routerConfig, testRouter,
+ testRouter.getNamenodeResolver(), testRouter.getSubclusterResolver());
+ server.init(routerConfig);
+ assertEquals(STATE.INITED, server.getServiceState());
+ server.start();
+ assertEquals(STATE.STARTED, server.getServiceState());
+ server.stop();
+ assertEquals(STATE.STOPPED, server.getServiceState());
+ server.close();
+ testRouter.close();
+ }
+
+ protected RouterDFSCluster getCluster() {
+ return TestRouterRpc.cluster;
+ }
+
+ protected RouterContext getRouterContext() {
+ return this.router;
+ }
+
+ protected void setRouter(RouterContext r)
+ throws IOException, URISyntaxException {
+ this.router = r;
+ this.routerProtocol = r.getClient().getNamenode();
+ this.routerFS = r.getFileSystem();
+ }
+
+ protected FileSystem getRouterFileSystem() {
+ return this.routerFS;
+ }
+
+ protected FileSystem getNamenodeFileSystem() {
+ return this.nnFS;
+ }
+
+ protected ClientProtocol getRouterProtocol() {
+ return this.routerProtocol;
+ }
+
+ protected ClientProtocol getNamenodeProtocol() {
+ return this.nnProtocol;
+ }
+
+ protected NamenodeContext getNamenode() {
+ return this.namenode;
+ }
+
+ protected void setNamenodeFile(String filename) {
+ this.nnFile = filename;
+ }
+
+ protected String getNamenodeFile() {
+ return this.nnFile;
+ }
+
+ protected void setRouterFile(String filename) {
+ this.routerFile = filename;
+ }
+
+ protected String getRouterFile() {
+ return this.routerFile;
+ }
+
+ protected void setNamenode(NamenodeContext nn)
+ throws IOException, URISyntaxException {
+ this.namenode = nn;
+ this.nnProtocol = nn.getClient().getNamenode();
+ this.nnFS = nn.getFileSystem();
+ }
+
+ protected String getNs() {
+ return this.ns;
+ }
+
+ protected void setNs(String nameservice) {
+ this.ns = nameservice;
+ }
+
+ protected static void compareResponses(
+ ClientProtocol protocol1, ClientProtocol protocol2,
+ Method m, Object[] paramList) {
+
+ Object return1 = null;
+ Exception exception1 = null;
+ try {
+ return1 = m.invoke(protocol1, paramList);
+ } catch (Exception ex) {
+ exception1 = ex;
+ }
+
+ Object return2 = null;
+ Exception exception2 = null;
+ try {
+ return2 = m.invoke(protocol2, paramList);
+ } catch (Exception ex) {
+ exception2 = ex;
+ }
+
+ assertEquals(return1, return2);
+ if (exception1 == null && exception2 == null) {
+ return;
+ }
+
+ assertEquals(
+ exception1.getCause().getClass(),
+ exception2.getCause().getClass());
+ }
+
+ @Test
+ public void testProxyListFiles() throws IOException, InterruptedException,
+ URISyntaxException, NoSuchMethodException, SecurityException {
+
+ // Verify that the root listing is a union of the mount table destinations
+ // and the files stored at all nameservices mounted at the root (ns0 + ns1)
+ //
+ // / -->
+ // /ns0 (from mount table)
+ // /ns1 (from mount table)
+ // all items in / of ns0 (default NS)
+
+ // Collect the mount table entries from the root mount point
+ Set<String> requiredPaths = new TreeSet<>();
+ FileSubclusterResolver fileResolver =
+ router.getRouter().getSubclusterResolver();
+ for (String mount : fileResolver.getMountPoints("/")) {
+ requiredPaths.add(mount);
+ }
+
+ // Collect all files/dirs on the root path of the default NS
+ String defaultNs = cluster.getNameservices().get(0);
+ NamenodeContext nn = cluster.getNamenode(defaultNs, null);
+ FileStatus[] iterator = nn.getFileSystem().listStatus(new Path("/"));
+ for (FileStatus file : iterator) {
+ requiredPaths.add(file.getPath().getName());
+ }
+
+ // Fetch listing
+ DirectoryListing listing =
+ routerProtocol.getListing("/", HdfsFileStatus.EMPTY_NAME, false);
+ Iterator<String> requiredPathsIterator = requiredPaths.iterator();
+ // Match each path returned and verify order returned
+ for(HdfsFileStatus f : listing.getPartialListing()) {
+ String fileName = requiredPathsIterator.next();
+ String currentFile = f.getFullPath(new Path("/")).getName();
+ assertEquals(currentFile, fileName);
+ }
+
+ // Verify the total number of results found/matched
+ assertEquals(requiredPaths.size(), listing.getPartialListing().length);
+
+ // List a path that doesn't exist and validate error response with NN
+ // behavior.
+ Method m = ClientProtocol.class.getMethod(
+ "getListing", String.class, byte[].class, boolean.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(routerProtocol, nnProtocol, m,
+ new Object[] {badPath, HdfsFileStatus.EMPTY_NAME, false});
+ }
+
+ @Test
+ public void testProxyListFilesWithConflict()
+ throws IOException, InterruptedException {
+
+ // Add a directory to the namespace that conflicts with a mount point
+ NamenodeContext nn = cluster.getNamenode(ns, null);
+ FileSystem nnFs = nn.getFileSystem();
+ addDirectory(nnFs, cluster.getFederatedTestDirectoryForNS(ns));
+
+ FileSystem routerFs = router.getFileSystem();
+ int initialCount = countContents(routerFs, "/");
+
+ // Root file system now for NS X:
+ // / ->
+ // /ns0 (mount table)
+ // /ns1 (mount table)
+ // /target-ns0 (the target folder for the NS0 mapped to /
+ // /nsX (local directory that duplicates mount table)
+ int newCount = countContents(routerFs, "/");
+ assertEquals(initialCount, newCount);
+
+ // Verify that each root path is readable and contains one test directory
+ assertEquals(1, countContents(routerFs, cluster.getFederatedPathForNS(ns)));
+
+ // Verify that real folder for the ns contains a single test directory
+ assertEquals(1, countContents(nnFs, cluster.getNamenodePathForNS(ns)));
+
+ }
+
+ protected void testRename(RouterContext testRouter, String filename,
+ String renamedFile, boolean exceptionExpected) throws IOException {
+
+ createFile(testRouter.getFileSystem(), filename, 32);
+ // verify
+ verifyFileExists(testRouter.getFileSystem(), filename);
+ // rename
+ boolean exceptionThrown = false;
+ try {
+ DFSClient client = testRouter.getClient();
+ ClientProtocol clientProtocol = client.getNamenode();
+ clientProtocol.rename(filename, renamedFile);
+ } catch (Exception ex) {
+ exceptionThrown = true;
+ }
+ if (exceptionExpected) {
+ // Error was expected
+ assertTrue(exceptionThrown);
+ FileContext fileContext = testRouter.getFileContext();
+ assertTrue(fileContext.delete(new Path(filename), true));
+ } else {
+ // No error was expected
+ assertFalse(exceptionThrown);
+ // verify
+ assertTrue(verifyFileExists(testRouter.getFileSystem(), renamedFile));
+ // delete
+ FileContext fileContext = testRouter.getFileContext();
+ assertTrue(fileContext.delete(new Path(renamedFile), true));
+ }
+ }
+
+ protected void testRename2(RouterContext testRouter, String filename,
+ String renamedFile, boolean exceptionExpected) throws IOException {
+ createFile(testRouter.getFileSystem(), filename, 32);
+ // verify
+ verifyFileExists(testRouter.getFileSystem(), filename);
+ // rename
+ boolean exceptionThrown = false;
+ try {
+ DFSClient client = testRouter.getClient();
+ ClientProtocol clientProtocol = client.getNamenode();
+ clientProtocol.rename2(filename, renamedFile, new Options.Rename[] {});
+ } catch (Exception ex) {
+ exceptionThrown = true;
+ }
+ assertEquals(exceptionExpected, exceptionThrown);
+ if (exceptionExpected) {
+ // Error was expected
+ FileContext fileContext = testRouter.getFileContext();
+ assertTrue(fileContext.delete(new Path(filename), true));
+ } else {
+ // verify
+ assertTrue(verifyFileExists(testRouter.getFileSystem(), renamedFile));
+ // delete
+ FileContext fileContext = testRouter.getFileContext();
+ assertTrue(fileContext.delete(new Path(renamedFile), true));
+ }
+ }
+
+ @Test
+ public void testProxyRenameFiles() throws IOException, InterruptedException {
+
+ Thread.sleep(5000);
+ List<String> nss = cluster.getNameservices();
+ String ns0 = nss.get(0);
+ String ns1 = nss.get(1);
+
+ // Rename within the same namespace
+ // /ns0/testdir/testrename -> /ns0/testdir/testrename-append
+ String filename =
+ cluster.getFederatedTestDirectoryForNS(ns0) + "/testrename";
+ String renamedFile = filename + "-append";
+ testRename(router, filename, renamedFile, false);
+ testRename2(router, filename, renamedFile, false);
+
+ // Rename a file to a destination that is in a different namespace (fails)
+ filename = cluster.getFederatedTestDirectoryForNS(ns0) + "/testrename";
+ renamedFile = cluster.getFederatedTestDirectoryForNS(ns1) + "/testrename";
+ testRename(router, filename, renamedFile, true);
+ testRename2(router, filename, renamedFile, true);
+ }
+
+ @Test
+ public void testProxyChownFiles() throws Exception {
+
+ String newUsername = "TestUser";
+ String newGroup = "TestGroup";
+
+ // change owner
+ routerProtocol.setOwner(routerFile, newUsername, newGroup);
+
+ // Verify with NN
+ FileStatus file = getFileStatus(namenode.getFileSystem(), nnFile);
+ assertEquals(file.getOwner(), newUsername);
+ assertEquals(file.getGroup(), newGroup);
+
+ // Bad request and validate router response matches NN response.
+ Method m = ClientProtocol.class.getMethod("setOwner", String.class,
+ String.class, String.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(routerProtocol, nnProtocol, m,
+ new Object[] {badPath, newUsername, newGroup});
+ }
+
+ @Test
+ public void testProxyGetStats() throws Exception {
+ // Some of the statistics are out of sync because of the mini cluster
+ Supplier<Boolean> check = new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ try {
+ long[] combinedData = routerProtocol.getStats();
+ long[] individualData = getAggregateStats();
+ int len = Math.min(combinedData.length, individualData.length);
+ for (int i = 0; i < len; i++) {
+ if (combinedData[i] != individualData[i]) {
+ LOG.error("Stats for {} don't match: {} != {}",
+ i, combinedData[i], individualData[i]);
+ return false;
+ }
+ }
+ return true;
+ } catch (Exception e) {
+ LOG.error("Cannot get stats: {}", e.getMessage());
+ return false;
+ }
+ }
+ };
+ GenericTestUtils.waitFor(check, 500, 5 * 1000);
+ }
+
+ /**
+ * Get the sum of each subcluster statistics.
+ * @return Aggregated statistics.
+ * @throws Exception If it cannot get the stats from the Router or Namenode.
+ */
+ private long[] getAggregateStats() throws Exception {
+ long[] individualData = new long[10];
+ for (String nameservice : cluster.getNameservices()) {
+ NamenodeContext n = cluster.getNamenode(nameservice, null);
+ DFSClient client = n.getClient();
+ ClientProtocol clientProtocol = client.getNamenode();
+ long[] data = clientProtocol.getStats();
+ for (int i = 0; i < data.length; i++) {
+ individualData[i] += data[i];
+ }
+ }
+ return individualData;
+ }
+
+ @Test
+ public void testProxyGetDatanodeReport() throws Exception {
+
+ DatanodeInfo[] combinedData =
+ routerProtocol.getDatanodeReport(DatanodeReportType.ALL);
+
+ Set<Integer> individualData = new HashSet<Integer>();
+ for (String nameservice : cluster.getNameservices()) {
+ NamenodeContext n = cluster.getNamenode(nameservice, null);
+ DFSClient client = n.getClient();
+ ClientProtocol clientProtocol = client.getNamenode();
+ DatanodeInfo[] data =
+ clientProtocol.getDatanodeReport(DatanodeReportType.ALL);
+ for (int i = 0; i < data.length; i++) {
+ // Collect unique DNs based on their xfer port
+ DatanodeInfo info = data[i];
+ individualData.add(info.getXferPort());
+ }
+ }
+ assertEquals(combinedData.length, individualData.size());
+ }
+
+ @Test
+ public void testProxyGetDatanodeStorageReport()
+ throws IOException, InterruptedException, URISyntaxException {
+
+ DatanodeStorageReport[] combinedData =
+ routerProtocol.getDatanodeStorageReport(DatanodeReportType.ALL);
+
+ Set<String> individualData = new HashSet<>();
+ for (String nameservice : cluster.getNameservices()) {
+ NamenodeContext n = cluster.getNamenode(nameservice, null);
+ DFSClient client = n.getClient();
+ ClientProtocol clientProtocol = client.getNamenode();
+ DatanodeStorageReport[] data =
+ clientProtocol.getDatanodeStorageReport(DatanodeReportType.ALL);
+ for (DatanodeStorageReport report : data) {
+ // Determine unique DN instances
+ DatanodeInfo dn = report.getDatanodeInfo();
+ individualData.add(dn.toString());
+ }
+ }
+ assertEquals(combinedData.length, individualData.size());
+ }
+
+ @Test
+ public void testProxyMkdir() throws Exception {
+
+ // Check the initial folders
+ FileStatus[] filesInitial = routerFS.listStatus(new Path("/"));
+
+ // Create a directory via the router at the root level
+ String dirPath = "/testdir";
+ FsPermission permission = new FsPermission("705");
+ routerProtocol.mkdirs(dirPath, permission, false);
+
+ // Verify the root listing has the item via the router
+ FileStatus[] files = routerFS.listStatus(new Path("/"));
+ assertEquals(Arrays.toString(files) + " should be " +
+ Arrays.toString(filesInitial) + " + " + dirPath,
+ filesInitial.length + 1, files.length);
+ assertTrue(verifyFileExists(routerFS, dirPath));
+
+ // Verify the directory is present in only 1 Namenode
+ int foundCount = 0;
+ for (NamenodeContext n : cluster.getNamenodes()) {
+ if (verifyFileExists(n.getFileSystem(), dirPath)) {
+ foundCount++;
+ }
+ }
+ assertEquals(1, foundCount);
+ assertTrue(deleteFile(routerFS, dirPath));
+
+ // Validate router failure response matches NN failure response.
+ Method m = ClientProtocol.class.getMethod("mkdirs", String.class,
+ FsPermission.class, boolean.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(routerProtocol, nnProtocol, m,
+ new Object[] {badPath, permission, false});
+ }
+
+ @Test
+ public void testProxyChmodFiles() throws Exception {
+
+ FsPermission permission = new FsPermission("444");
+
+ // change permissions
+ routerProtocol.setPermission(routerFile, permission);
+
+ // Validate permissions NN
+ FileStatus file = getFileStatus(namenode.getFileSystem(), nnFile);
+ assertEquals(permission, file.getPermission());
+
+ // Validate router failure response matches NN failure response.
+ Method m = ClientProtocol.class.getMethod(
+ "setPermission", String.class, FsPermission.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(routerProtocol, nnProtocol, m,
+ new Object[] {badPath, permission});
+ }
+
+ @Test
+ public void testProxySetReplication() throws Exception {
+
+ // Check current replication via NN
+ FileStatus file = getFileStatus(nnFS, nnFile);
+ assertEquals(1, file.getReplication());
+
+ // increment replication via router
+ routerProtocol.setReplication(routerFile, (short) 2);
+
+ // Verify via NN
+ file = getFileStatus(nnFS, nnFile);
+ assertEquals(2, file.getReplication());
+
+ // Validate router failure response matches NN failure response.
+ Method m = ClientProtocol.class.getMethod(
+ "setReplication", String.class, short.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(routerProtocol, nnProtocol, m,
+ new Object[] {badPath, (short) 2});
+ }
+
+ @Test
+ public void testProxyTruncateFile() throws Exception {
+
+ // Check file size via NN
+ FileStatus file = getFileStatus(nnFS, nnFile);
+ assertTrue(file.getLen() > 0);
+
+ // Truncate to 0 bytes via router
+ routerProtocol.truncate(routerFile, 0, "testclient");
+
+ // Verify via NN
+ file = getFileStatus(nnFS, nnFile);
+ assertEquals(0, file.getLen());
+
+ // Validate router failure response matches NN failure response.
+ Method m = ClientProtocol.class.getMethod(
+ "truncate", String.class, long.class, String.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(routerProtocol, nnProtocol, m,
+ new Object[] {badPath, (long) 0, "testclient"});
+ }
+
+ @Test
+ public void testProxyGetBlockLocations() throws Exception {
+
+ // Fetch block locations via router
+ LocatedBlocks locations =
+ routerProtocol.getBlockLocations(routerFile, 0, 1024);
+ assertEquals(1, locations.getLocatedBlocks().size());
+
+ // Validate router failure response matches NN failure response.
+ Method m = ClientProtocol.class.getMethod(
+ "getBlockLocations", String.class, long.class, long.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(routerProtocol, nnProtocol,
+ m, new Object[] {badPath, (long) 0, (long) 0});
+ }
+
+ @Test
+ public void testProxyStoragePolicy() throws Exception {
+
+ // Query initial policy via NN
+ HdfsFileStatus status = namenode.getClient().getFileInfo(nnFile);
+
+ // Set a random policy via router
+ BlockStoragePolicy[] policies = namenode.getClient().getStoragePolicies();
+ BlockStoragePolicy policy = policies[0];
+
+ while (policy.isCopyOnCreateFile()) {
+ // Pick a non copy on create policy
+ Random rand = new Random();
+ int randIndex = rand.nextInt(policies.length);
+ policy = policies[randIndex];
+ }
+ routerProtocol.setStoragePolicy(routerFile, policy.getName());
+
+ // Verify policy via NN
+ HdfsFileStatus newStatus = namenode.getClient().getFileInfo(nnFile);
+ assertTrue(newStatus.getStoragePolicy() == policy.getId());
+ assertTrue(newStatus.getStoragePolicy() != status.getStoragePolicy());
+
+ // Validate router failure response matches NN failure response.
+ Method m = ClientProtocol.class.getMethod("setStoragePolicy", String.class,
+ String.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(routerProtocol, nnProtocol,
+ m, new Object[] {badPath, "badpolicy"});
+ }
+
+ @Test
+ public void testProxyGetPreferedBlockSize() throws Exception {
+
+ // Query via NN and Router and verify
+ long namenodeSize = nnProtocol.getPreferredBlockSize(nnFile);
+ long routerSize = routerProtocol.getPreferredBlockSize(routerFile);
+ assertEquals(routerSize, namenodeSize);
+
+ // Validate router failure response matches NN failure response.
+ Method m = ClientProtocol.class.getMethod(
+ "getPreferredBlockSize", String.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(
+ routerProtocol, nnProtocol, m, new Object[] {badPath});
+ }
+
+ private void testConcat(
+ String source, String target, boolean failureExpected) {
+ boolean failure = false;
+ try {
+ // Concat test file with fill block length file via router
+ routerProtocol.concat(target, new String[] {source});
+ } catch (IOException ex) {
+ failure = true;
+ }
+ assertEquals(failureExpected, failure);
+ }
+
+ @Test
+ public void testProxyConcatFile() throws Exception {
+
+ // Create a stub file in the primary ns
+ String sameNameservice = ns;
+ String existingFile =
+ cluster.getFederatedTestDirectoryForNS(sameNameservice) +
+ "_concatfile";
+ int existingFileSize = 32;
+ createFile(routerFS, existingFile, existingFileSize);
+
+ // Identify an alternate nameservice that doesn't match the existing file
+ String alternateNameservice = null;
+ for (String n : cluster.getNameservices()) {
+ if (!n.equals(sameNameservice)) {
+ alternateNameservice = n;
+ break;
+ }
+ }
+
+ // Create new files, must be a full block to use concat. One file is in the
+ // same namespace as the target file, the other is in a different namespace.
+ String altRouterFile =
+ cluster.getFederatedTestDirectoryForNS(alternateNameservice) +
+ "_newfile";
+ String sameRouterFile =
+ cluster.getFederatedTestDirectoryForNS(sameNameservice) +
+ "_newfile";
+ createFile(routerFS, altRouterFile, DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
+ createFile(routerFS, sameRouterFile, DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
+
+ // Concat in different namespaces, fails
+ testConcat(existingFile, altRouterFile, true);
+
+ // Concat in same namespaces, succeeds
+ testConcat(existingFile, sameRouterFile, false);
+
+ // Check target file length
+ FileStatus status = getFileStatus(routerFS, sameRouterFile);
+ assertEquals(
+ existingFileSize + DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT,
+ status.getLen());
+
+ // Validate router failure response matches NN failure response.
+ Method m = ClientProtocol.class.getMethod(
+ "concat", String.class, String[].class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(routerProtocol, nnProtocol, m,
+ new Object[] {badPath, new String[] {routerFile}});
+ }
+
+ @Test
+ public void testProxyAppend() throws Exception {
+
+ // Append a test string via router
+ EnumSet<CreateFlag> createFlag = EnumSet.of(CreateFlag.APPEND);
+ DFSClient routerClient = getRouterContext().getClient();
+ HdfsDataOutputStream stream =
+ routerClient.append(routerFile, 1024, createFlag, null, null);
+ stream.writeBytes(TEST_STRING);
+ stream.close();
+
+ // Verify file size via NN
+ FileStatus status = getFileStatus(nnFS, nnFile);
+ assertTrue(status.getLen() > TEST_STRING.length());
+
+ // Validate router failure response matches NN failure response.
+ Method m = ClientProtocol.class.getMethod("append", String.class,
+ String.class, EnumSetWritable.class);
+ String badPath = "/unknownlocation/unknowndir";
+ EnumSetWritable<CreateFlag> createFlagWritable =
+ new EnumSetWritable<CreateFlag>(createFlag);
+ compareResponses(routerProtocol, nnProtocol, m,
+ new Object[] {badPath, "testClient", createFlagWritable});
+ }
+
+ @Test
+ public void testProxyGetAdditionalDatanode()
+ throws IOException, InterruptedException, URISyntaxException {
+
+ // Use primitive APIs to open a file, add a block, and get datanode location
+ EnumSet<CreateFlag> createFlag = EnumSet.of(CreateFlag.CREATE);
+ String clientName = getRouterContext().getClient().getClientName();
+ String newRouterFile = routerFile + "_additionalDatanode";
+ HdfsFileStatus status = routerProtocol.create(
+ newRouterFile, new FsPermission("777"), clientName,
+ new EnumSetWritable<CreateFlag>(createFlag), true, (short) 1,
+ (long) 1024, CryptoProtocolVersion.supported(), null);
+
+ // Add a block via router (requires client to have same lease)
+ LocatedBlock block = routerProtocol.addBlock(
+ newRouterFile, clientName, null, null,
+ status.getFileId(), null, null);
+
+ DatanodeInfo[] exclusions = new DatanodeInfo[0];
+ LocatedBlock newBlock = routerProtocol.getAdditionalDatanode(
+ newRouterFile, status.getFileId(), block.getBlock(),
+ block.getLocations(), block.getStorageIDs(), exclusions, 1, clientName);
+ assertNotNull(newBlock);
+ }
+
+ @Test
+ public void testProxyCreateFileAlternateUser()
+ throws IOException, URISyntaxException, InterruptedException {
+
+ // Create via Router
+ String routerDir = cluster.getFederatedTestDirectoryForNS(ns);
+ String namenodeDir = cluster.getNamenodeTestDirectoryForNS(ns);
+ String newRouterFile = routerDir + "/unknownuser";
+ String newNamenodeFile = namenodeDir + "/unknownuser";
+ String username = "unknownuser";
+
+ // Allow all user access to dir
+ namenode.getFileContext().setPermission(
+ new Path(namenodeDir), new FsPermission("777"));
+
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser(username);
+ DFSClient client = getRouterContext().getClient(ugi);
+ client.create(newRouterFile, true);
+
+ // Fetch via NN and check user
+ FileStatus status = getFileStatus(nnFS, newNamenodeFile);
+ assertEquals(status.getOwner(), username);
+ }
+
+ @Test
+ public void testProxyGetFileInfoAcessException() throws IOException {
+
+ UserGroupInformation ugi =
+ UserGroupInformation.createRemoteUser("unknownuser");
+
+ // List files from the NN and trap the exception
+ Exception nnFailure = null;
+ try {
+ String testFile = cluster.getNamenodeTestFileForNS(ns);
+ namenode.getClient(ugi).getLocatedBlocks(testFile, 0);
+ } catch (Exception e) {
+ nnFailure = e;
+ }
+ assertNotNull(nnFailure);
+
+ // List files from the router and trap the exception
+ Exception routerFailure = null;
+ try {
+ String testFile = cluster.getFederatedTestFileForNS(ns);
+ getRouterContext().getClient(ugi).getLocatedBlocks(testFile, 0);
+ } catch (Exception e) {
+ routerFailure = e;
+ }
+ assertNotNull(routerFailure);
+
+ assertEquals(routerFailure.getClass(), nnFailure.getClass());
+ }
+
+ @Test
+ public void testErasureCoding() throws IOException {
+
+ LOG.info("List the available erasurce coding policies");
+ ErasureCodingPolicyInfo[] policies = checkErasureCodingPolicies();
+ for (ErasureCodingPolicyInfo policy : policies) {
+ LOG.info(" {}", policy);
+ }
+
+ LOG.info("List the erasure coding codecs");
+ Map<String, String> codecsRouter = routerProtocol.getErasureCodingCodecs();
+ Map<String, String> codecsNamenode = nnProtocol.getErasureCodingCodecs();
+ assertTrue(Maps.difference(codecsRouter, codecsNamenode).areEqual());
+ for (Entry<String, String> entry : codecsRouter.entrySet()) {
+ LOG.info(" {}: {}", entry.getKey(), entry.getValue());
+ }
+
+ LOG.info("Create a testing directory via the router at the root level");
+ String dirPath = "/testec";
+ String filePath1 = dirPath + "/testfile1";
+ FsPermission permission = new FsPermission("755");
+ routerProtocol.mkdirs(dirPath, permission, false);
+ createFile(routerFS, filePath1, 32);
+ assertTrue(verifyFileExists(routerFS, filePath1));
+ DFSClient file1Protocol = getFileDFSClient(filePath1);
+
+ LOG.info("The policy for the new file should not be set");
+ assertNull(routerProtocol.getErasureCodingPolicy(filePath1));
+ assertNull(file1Protocol.getErasureCodingPolicy(filePath1));
+
+ String policyName = "RS-6-3-1024k";
+ LOG.info("Set policy \"{}\" for \"{}\"", policyName, dirPath);
+ routerProtocol.setErasureCodingPolicy(dirPath, policyName);
+
+ String filePath2 = dirPath + "/testfile2";
+ LOG.info("Create {} in the path with the new EC policy", filePath2);
+ createFile(routerFS, filePath2, 32);
+ assertTrue(verifyFileExists(routerFS, filePath2));
+ DFSClient file2Protocol = getFileDFSClient(filePath2);
+
+ LOG.info("Check that the policy is set for {}", filePath2);
+ ErasureCodingPolicy policyRouter1 =
+ routerProtocol.getErasureCodingPolicy(filePath2);
+ ErasureCodingPolicy policyNamenode1 =
+ file2Protocol.getErasureCodingPolicy(filePath2);
+ assertNotNull(policyRouter1);
+ assertEquals(policyName, policyRouter1.getName());
+ assertEquals(policyName, policyNamenode1.getName());
+
+ LOG.info("Create a new erasure coding policy");
+ String newPolicyName = "RS-6-3-128k";
+ ECSchema ecSchema = new ECSchema(ErasureCodeConstants.RS_CODEC_NAME, 6, 3);
+ ErasureCodingPolicy ecPolicy = new ErasureCodingPolicy(
+ newPolicyName,
+ ecSchema,
+ 128 * 1024,
+ (byte) -1);
+ ErasureCodingPolicy[] newPolicies = new ErasureCodingPolicy[] {
+ ecPolicy
+ };
+ AddErasureCodingPolicyResponse[] responses =
+ routerProtocol.addErasureCodingPolicies(newPolicies);
+ assertEquals(1, responses.length);
+ assertTrue(responses[0].isSucceed());
+ routerProtocol.disableErasureCodingPolicy(newPolicyName);
+
+ LOG.info("The new policy should be there and disabled");
+ policies = checkErasureCodingPolicies();
+ boolean found = false;
+ for (ErasureCodingPolicyInfo policy : policies) {
+ LOG.info(" {}" + policy);
+ if (policy.getPolicy().getName().equals(newPolicyName)) {
+ found = true;
+ assertEquals(ErasureCodingPolicyState.DISABLED, policy.getState());
+ break;
+ }
+ }
+ assertTrue(found);
+
+ LOG.info("Set the test folder to use the new policy");
+ routerProtocol.enableErasureCodingPolicy(newPolicyName);
+ routerProtocol.setErasureCodingPolicy(dirPath, newPolicyName);
+
+ LOG.info("Create a file in the path with the new EC policy");
+ String filePath3 = dirPath + "/testfile3";
+ createFile(routerFS, filePath3, 32);
+ assertTrue(verifyFileExists(routerFS, filePath3));
+ DFSClient file3Protocol = getFileDFSClient(filePath3);
+
+ ErasureCodingPolicy policyRouterFile3 =
+ routerProtocol.getErasureCodingPolicy(filePath3);
+ assertEquals(newPolicyName, policyRouterFile3.getName());
+ ErasureCodingPolicy policyNamenodeFile3 =
+ file3Protocol.getErasureCodingPolicy(filePath3);
+ assertEquals(newPolicyName, policyNamenodeFile3.getName());
+
+ LOG.info("Remove the policy and check the one for the test folder");
+ routerProtocol.removeErasureCodingPolicy(newPolicyName);
+ ErasureCodingPolicy policyRouter3 =
+ routerProtocol.getErasureCodingPolicy(filePath3);
+ assertEquals(newPolicyName, policyRouter3.getName());
+ ErasureCodingPolicy policyNamenode3 =
+ file3Protocol.getErasureCodingPolicy(filePath3);
+ assertEquals(newPolicyName, policyNamenode3.getName());
+
+ LOG.info("Check the stats");
+ ECBlockGroupStats statsRouter = routerProtocol.getECBlockGroupStats();
+ ECBlockGroupStats statsNamenode = nnProtocol.getECBlockGroupStats();
+ assertEquals(statsNamenode.toString(), statsRouter.toString());
+ }
+
+ /**
+ * Check the erasure coding policies in the Router and the Namenode.
+ * @return The erasure coding policies.
+ */
+ private ErasureCodingPolicyInfo[] checkErasureCodingPolicies()
+ throws IOException {
+ ErasureCodingPolicyInfo[] policiesRouter =
+ routerProtocol.getErasureCodingPolicies();
+ assertNotNull(policiesRouter);
+ ErasureCodingPolicyInfo[] policiesNamenode =
+ nnProtocol.getErasureCodingPolicies();
+ Arrays.sort(policiesRouter, EC_POLICY_CMP);
+ Arrays.sort(policiesNamenode, EC_POLICY_CMP);
+ assertArrayEquals(policiesRouter, policiesNamenode);
+ return policiesRouter;
+ }
+
+ /**
+ * Find the Namenode for a particular file and return the DFSClient.
+ * @param path Path of the file to check.
+ * @return The DFSClient to the Namenode holding the file.
+ */
+ private DFSClient getFileDFSClient(final String path) {
+ for (String nsId : cluster.getNameservices()) {
+ LOG.info("Checking {} for {}", nsId, path);
+ NamenodeContext nn = cluster.getNamenode(nsId, null);
+ try {
+ DFSClient nnClientProtocol = nn.getClient();
+ if (nnClientProtocol.getFileInfo(path) != null) {
+ return nnClientProtocol;
+ }
+ } catch (Exception ignore) {
+ // ignore
+ }
+ }
+ return null;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
new file mode 100644
index 0000000..5489691
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createFile;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyFileExists;
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.server.federation.MockResolver;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+
+/**
+ * The the RPC interface of the {@link getRouter()} implemented by
+ * {@link RouterRpcServer}.
+ */
+public class TestRouterRpcMultiDestination extends TestRouterRpc {
+
+ @Override
+ public void testSetup() throws Exception {
+
+ RouterDFSCluster cluster = getCluster();
+
+ // Create mock locations
+ getCluster().installMockLocations();
+ List<RouterContext> routers = cluster.getRouters();
+
+ // Add extra location to the root mount / such that the root mount points:
+ // /
+ // ns0 -> /
+ // ns1 -> /
+ for (RouterContext rc : routers) {
+ Router router = rc.getRouter();
+ MockResolver resolver = (MockResolver) router.getSubclusterResolver();
+ resolver.addLocation("/", cluster.getNameservices().get(1), "/");
+ }
+
+ // Create a mount that points to 2 dirs in the same ns:
+ // /same
+ // ns0 -> /
+ // ns0 -> /target-ns0
+ for (RouterContext rc : routers) {
+ Router router = rc.getRouter();
+ MockResolver resolver = (MockResolver) router.getSubclusterResolver();
+ List<String> nss = cluster.getNameservices();
+ String ns0 = nss.get(0);
+ resolver.addLocation("/same", ns0, "/");
+ resolver.addLocation("/same", ns0, cluster.getNamenodePathForNS(ns0));
+ }
+
+ // Delete all files via the NNs and verify
+ cluster.deleteAllFiles();
+
+ // Create test fixtures on NN
+ cluster.createTestDirectoriesNamenode();
+
+ // Wait to ensure NN has fully created its test directories
+ Thread.sleep(100);
+
+ // Pick a NS, namenode and getRouter() for this test
+ RouterContext router = cluster.getRandomRouter();
+ this.setRouter(router);
+
+ String ns = cluster.getRandomNameservice();
+ this.setNs(ns);
+ this.setNamenode(cluster.getNamenode(ns, null));
+
+ // Create a test file on a single NN that is accessed via a getRouter() path
+ // with 2 destinations. All tests should failover to the alternate
+ // destination if the wrong NN is attempted first.
+ Random r = new Random();
+ String randomString = "testfile-" + r.nextInt();
+ setNamenodeFile("/" + randomString);
+ setRouterFile("/" + randomString);
+
+ FileSystem nnFs = getNamenodeFileSystem();
+ FileSystem routerFs = getRouterFileSystem();
+ createFile(nnFs, getNamenodeFile(), 32);
+
+ verifyFileExists(nnFs, getNamenodeFile());
+ verifyFileExists(routerFs, getRouterFile());
+ }
+
+ private void testListing(String path) throws IOException {
+
+ // Collect the mount table entries for this path
+ Set<String> requiredPaths = new TreeSet<>();
+ RouterContext rc = getRouterContext();
+ Router router = rc.getRouter();
+ FileSubclusterResolver subclusterResolver = router.getSubclusterResolver();
+ for (String mount : subclusterResolver.getMountPoints(path)) {
+ requiredPaths.add(mount);
+ }
+
+ // Get files/dirs from the Namenodes
+ PathLocation location = subclusterResolver.getDestinationForPath(path);
+ for (RemoteLocation loc : location.getDestinations()) {
+ String nsId = loc.getNameserviceId();
+ String dest = loc.getDest();
+ NamenodeContext nn = getCluster().getNamenode(nsId, null);
+ FileSystem fs = nn.getFileSystem();
+ FileStatus[] files = fs.listStatus(new Path(dest));
+ for (FileStatus file : files) {
+ String pathName = file.getPath().getName();
+ requiredPaths.add(pathName);
+ }
+ }
+
+ // Get files/dirs from the Router
+ DirectoryListing listing =
+ getRouterProtocol().getListing(path, HdfsFileStatus.EMPTY_NAME, false);
+ Iterator<String> requiredPathsIterator = requiredPaths.iterator();
+
+ // Match each path returned and verify order returned
+ HdfsFileStatus[] partialListing = listing.getPartialListing();
+ for (HdfsFileStatus fileStatus : listing.getPartialListing()) {
+ String fileName = requiredPathsIterator.next();
+ String currentFile = fileStatus.getFullPath(new Path(path)).getName();
+ assertEquals(currentFile, fileName);
+ }
+
+ // Verify the total number of results found/matched
+ assertEquals(
+ requiredPaths + " doesn't match " + Arrays.toString(partialListing),
+ requiredPaths.size(), partialListing.length);
+ }
+
+ @Override
+ public void testProxyListFiles() throws IOException, InterruptedException,
+ URISyntaxException, NoSuchMethodException, SecurityException {
+
+ // Verify that the root listing is a union of the mount table destinations
+ // and the files stored at all nameservices mounted at the root (ns0 + ns1)
+ // / -->
+ // /ns0 (from mount table)
+ // /ns1 (from mount table)
+ // /same (from the mount table)
+ // all items in / of ns0 from mapping of / -> ns0:::/)
+ // all items in / of ns1 from mapping of / -> ns1:::/)
+ testListing("/");
+
+ // Verify that the "/same" mount point lists the contents of both dirs in
+ // the same ns
+ // /same -->
+ // /target-ns0 (from root of ns0)
+ // /testdir (from contents of /target-ns0)
+ testListing("/same");
+
+ // List a non-existing path and validate error response with NN behavior
+ ClientProtocol namenodeProtocol =
+ getCluster().getRandomNamenode().getClient().getNamenode();
+ Method m = ClientProtocol.class.getMethod(
+ "getListing", String.class, byte[].class, boolean.class);
+ String badPath = "/unknownlocation/unknowndir";
+ compareResponses(getRouterProtocol(), namenodeProtocol, m,
+ new Object[] {badPath, HdfsFileStatus.EMPTY_NAME, false});
+ }
+
+ @Override
+ public void testProxyRenameFiles() throws IOException, InterruptedException {
+
+ super.testProxyRenameFiles();
+
+ List<String> nss = getCluster().getNameservices();
+ String ns0 = nss.get(0);
+ String ns1 = nss.get(1);
+
+ // Rename a file from ns0 into the root (mapped to both ns0 and ns1)
+ String testDir0 = getCluster().getFederatedTestDirectoryForNS(ns0);
+ String filename0 = testDir0 + "/testrename";
+ String renamedFile = "/testrename";
+ testRename(getRouterContext(), filename0, renamedFile, false);
+ testRename2(getRouterContext(), filename0, renamedFile, false);
+
+ // Rename a file from ns1 into the root (mapped to both ns0 and ns1)
+ String testDir1 = getCluster().getFederatedTestDirectoryForNS(ns1);
+ String filename1 = testDir1 + "/testrename";
+ testRename(getRouterContext(), filename1, renamedFile, false);
+ testRename2(getRouterContext(), filename1, renamedFile, false);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterSafemode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterSafemode.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterSafemode.java
new file mode 100644
index 0000000..e5d8348
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterSafemode.java
@@ -0,0 +1,200 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_CACHE_TIME_TO_LIVE_MS;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_SAFEMODE_EXPIRATION;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_SAFEMODE_EXTENSION;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.deleteStateStore;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.getStateStoreConfiguration;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.service.Service.STATE;
+import org.apache.hadoop.util.Time;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test the safe mode for the {@link Router} controlled by
+ * {@link RouterSafemodeService}.
+ */
+public class TestRouterSafemode {
+
+ private Router router;
+ private static Configuration conf;
+
+ @BeforeClass
+ public static void create() throws IOException {
+ // Wipe state store
+ deleteStateStore();
+ // Configuration that supports the state store
+ conf = getStateStoreConfiguration();
+ // 2 sec startup standby
+ conf.setTimeDuration(DFS_ROUTER_SAFEMODE_EXTENSION,
+ TimeUnit.SECONDS.toMillis(2), TimeUnit.MILLISECONDS);
+ // 1 sec cache refresh
+ conf.setTimeDuration(DFS_ROUTER_CACHE_TIME_TO_LIVE_MS,
+ TimeUnit.SECONDS.toMillis(1), TimeUnit.MILLISECONDS);
+ // 2 sec post cache update before entering safemode (2 intervals)
+ conf.setTimeDuration(DFS_ROUTER_SAFEMODE_EXPIRATION,
+ TimeUnit.SECONDS.toMillis(2), TimeUnit.MILLISECONDS);
+
+ conf.set(RBFConfigKeys.DFS_ROUTER_RPC_BIND_HOST_KEY, "0.0.0.0");
+ conf.set(RBFConfigKeys.DFS_ROUTER_RPC_ADDRESS_KEY, "127.0.0.1:0");
+ conf.set(RBFConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_KEY, "127.0.0.1:0");
+ conf.set(RBFConfigKeys.DFS_ROUTER_ADMIN_BIND_HOST_KEY, "0.0.0.0");
+ conf.set(RBFConfigKeys.DFS_ROUTER_HTTP_ADDRESS_KEY, "127.0.0.1:0");
+ conf.set(RBFConfigKeys.DFS_ROUTER_HTTPS_ADDRESS_KEY, "127.0.0.1:0");
+
+ // RPC + State Store + Safe Mode only
+ conf = new RouterConfigBuilder(conf)
+ .rpc()
+ .safemode()
+ .stateStore()
+ .metrics()
+ .build();
+ }
+
+ @AfterClass
+ public static void destroy() {
+ }
+
+ @Before
+ public void setup() throws IOException, URISyntaxException {
+ router = new Router();
+ router.init(conf);
+ router.start();
+ }
+
+ @After
+ public void cleanup() throws IOException {
+ if (router != null) {
+ router.stop();
+ router = null;
+ }
+ }
+
+ @Test
+ public void testSafemodeService() throws IOException {
+ RouterSafemodeService server = new RouterSafemodeService(router);
+ server.init(conf);
+ assertEquals(STATE.INITED, server.getServiceState());
+ server.start();
+ assertEquals(STATE.STARTED, server.getServiceState());
+ server.stop();
+ assertEquals(STATE.STOPPED, server.getServiceState());
+ server.close();
+ }
+
+ @Test
+ public void testRouterExitSafemode()
+ throws InterruptedException, IllegalStateException, IOException {
+
+ assertTrue(router.getRpcServer().isInSafeMode());
+ verifyRouter(RouterServiceState.SAFEMODE);
+
+ // Wait for initial time in milliseconds
+ long interval =
+ conf.getTimeDuration(DFS_ROUTER_SAFEMODE_EXTENSION,
+ TimeUnit.SECONDS.toMillis(2), TimeUnit.MILLISECONDS) +
+ conf.getTimeDuration(DFS_ROUTER_CACHE_TIME_TO_LIVE_MS,
+ TimeUnit.SECONDS.toMillis(1), TimeUnit.MILLISECONDS);
+ Thread.sleep(interval);
+
+ assertFalse(router.getRpcServer().isInSafeMode());
+ verifyRouter(RouterServiceState.RUNNING);
+ }
+
+ @Test
+ public void testRouterEnterSafemode()
+ throws IllegalStateException, IOException, InterruptedException {
+
+ // Verify starting state
+ assertTrue(router.getRpcServer().isInSafeMode());
+ verifyRouter(RouterServiceState.SAFEMODE);
+
+ // We should be in safe mode for DFS_ROUTER_SAFEMODE_EXTENSION time
+ long interval0 = conf.getTimeDuration(DFS_ROUTER_SAFEMODE_EXTENSION,
+ TimeUnit.SECONDS.toMillis(2), TimeUnit.MILLISECONDS) - 1000;
+ long t0 = Time.now();
+ while (Time.now() - t0 < interval0) {
+ verifyRouter(RouterServiceState.SAFEMODE);
+ Thread.sleep(100);
+ }
+
+ // We wait some time for the state to propagate
+ long interval1 = 1000 + 2 * conf.getTimeDuration(
+ DFS_ROUTER_CACHE_TIME_TO_LIVE_MS, TimeUnit.SECONDS.toMillis(1),
+ TimeUnit.MILLISECONDS);
+ Thread.sleep(interval1);
+
+ // Running
+ assertFalse(router.getRpcServer().isInSafeMode());
+ verifyRouter(RouterServiceState.RUNNING);
+
+ // Disable cache
+ router.getStateStore().stopCacheUpdateService();
+
+ // Wait until the State Store cache is stale in milliseconds
+ long interval2 =
+ conf.getTimeDuration(DFS_ROUTER_SAFEMODE_EXPIRATION,
+ TimeUnit.SECONDS.toMillis(2), TimeUnit.MILLISECONDS) +
+ conf.getTimeDuration(DFS_ROUTER_CACHE_TIME_TO_LIVE_MS,
+ TimeUnit.SECONDS.toMillis(1), TimeUnit.MILLISECONDS);
+ Thread.sleep(interval2);
+
+ // Safemode
+ assertTrue(router.getRpcServer().isInSafeMode());
+ verifyRouter(RouterServiceState.SAFEMODE);
+ }
+
+ @Test
+ public void testRouterRpcSafeMode()
+ throws IllegalStateException, IOException {
+
+ assertTrue(router.getRpcServer().isInSafeMode());
+ verifyRouter(RouterServiceState.SAFEMODE);
+
+ // If the Router is in Safe Mode, we should get a SafeModeException
+ boolean exception = false;
+ try {
+ router.getRpcServer().delete("/testfile.txt", true);
+ fail("We should have thrown a safe mode exception");
+ } catch (RouterSafeModeException sme) {
+ exception = true;
+ }
+ assertTrue("We should have thrown a safe mode exception", exception);
+ }
+
+ private void verifyRouter(RouterServiceState status)
+ throws IllegalStateException, IOException {
+ assertEquals(status, router.getRouterState());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
new file mode 100644
index 0000000..2ec5d62
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
@@ -0,0 +1,274 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_STORE_DRIVER_CLASS;
+import static org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl.FEDERATION_STORE_FILE_DIRECTORY;
+import static org.junit.Assert.assertNotNull;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileBaseImpl;
+import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipStats;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.util.Time;
+
+/**
+ * Utilities to test the State Store.
+ */
+public final class FederationStateStoreTestUtils {
+
+ /** The State Store Driver implementation class for testing .*/
+ private static final Class<? extends StateStoreDriver>
+ FEDERATION_STORE_DRIVER_CLASS_FOR_TEST = StateStoreFileImpl.class;
+
+ private FederationStateStoreTestUtils() {
+ // Utility Class
+ }
+
+ /**
+ * Get the State Store driver implementation for testing.
+ *
+ * @return Class of the State Store driver implementation.
+ */
+ public static Class<? extends StateStoreDriver> getTestDriverClass() {
+ return FEDERATION_STORE_DRIVER_CLASS_FOR_TEST;
+ }
+
+ /**
+ * Create a default State Store configuration.
+ *
+ * @return State Store configuration.
+ */
+ public static Configuration getStateStoreConfiguration() {
+ Class<? extends StateStoreDriver> clazz = getTestDriverClass();
+ return getStateStoreConfiguration(clazz);
+ }
+
+ /**
+ * Create a new State Store configuration for a particular driver.
+ *
+ * @param clazz Class of the driver to create.
+ * @return State Store configuration.
+ */
+ public static Configuration getStateStoreConfiguration(
+ Class<? extends StateStoreDriver> clazz) {
+ Configuration conf = new HdfsConfiguration(false);
+
+ conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, true);
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, "hdfs://test");
+
+ conf.setClass(FEDERATION_STORE_DRIVER_CLASS, clazz, StateStoreDriver.class);
+
+ if (clazz.isAssignableFrom(StateStoreFileBaseImpl.class)) {
+ setFileConfiguration(conf);
+ }
+ return conf;
+ }
+
+ /**
+ * Create a new State Store based on a configuration.
+ *
+ * @param configuration Configuration for the State Store.
+ * @return New State Store service.
+ * @throws IOException If it cannot create the State Store.
+ * @throws InterruptedException If we cannot wait for the store to start.
+ */
+ public static StateStoreService newStateStore(
+ Configuration configuration) throws IOException, InterruptedException {
+
+ StateStoreService stateStore = new StateStoreService();
+ assertNotNull(stateStore);
+
+ // Set unique identifier, this is normally the router address
+ String identifier = UUID.randomUUID().toString();
+ stateStore.setIdentifier(identifier);
+
+ stateStore.init(configuration);
+ stateStore.start();
+
+ // Wait for state store to connect
+ waitStateStore(stateStore, TimeUnit.SECONDS.toMillis(10));
+
+ return stateStore;
+ }
+
+ /**
+ * Wait for the State Store to initialize its driver.
+ *
+ * @param stateStore State Store.
+ * @param timeoutMs Time out in milliseconds.
+ * @throws IOException If the State Store cannot be reached.
+ * @throws InterruptedException If the sleep is interrupted.
+ */
+ public static void waitStateStore(StateStoreService stateStore,
+ long timeoutMs) throws IOException, InterruptedException {
+ long startingTime = Time.monotonicNow();
+ while (!stateStore.isDriverReady()) {
+ Thread.sleep(100);
+ if (Time.monotonicNow() - startingTime > timeoutMs) {
+ throw new IOException("Timeout waiting for State Store to connect");
+ }
+ }
+ }
+
+ /**
+ * Delete the default State Store.
+ *
+ * @throws IOException
+ */
+ public static void deleteStateStore() throws IOException {
+ Class<? extends StateStoreDriver> driverClass = getTestDriverClass();
+ deleteStateStore(driverClass);
+ }
+
+ /**
+ * Delete the State Store.
+ * @param driverClass Class of the State Store driver implementation.
+ * @throws IOException If it cannot be deleted.
+ */
+ public static void deleteStateStore(
+ Class<? extends StateStoreDriver> driverClass) throws IOException {
+
+ if (StateStoreFileBaseImpl.class.isAssignableFrom(driverClass)) {
+ String workingDirectory = System.getProperty("user.dir");
+ File dir = new File(workingDirectory + "/statestore");
+ if (dir.exists()) {
+ FileUtils.cleanDirectory(dir);
+ }
+ }
+ }
+
+ /**
+ * Set the default configuration for drivers based on files.
+ *
+ * @param conf Configuration to extend.
+ */
+ public static void setFileConfiguration(Configuration conf) {
+ String workingPath = System.getProperty("user.dir");
+ String stateStorePath = workingPath + "/statestore";
+ conf.set(FEDERATION_STORE_FILE_DIRECTORY, stateStorePath);
+ }
+
+ /**
+ * Clear all the records from the State Store.
+ *
+ * @param store State Store to remove records from.
+ * @return If the State Store was cleared.
+ * @throws IOException If it cannot clear the State Store.
+ */
+ public static boolean clearAllRecords(StateStoreService store)
+ throws IOException {
+ Collection<Class<? extends BaseRecord>> allRecords =
+ store.getSupportedRecords();
+ for (Class<? extends BaseRecord> recordType : allRecords) {
+ if (!clearRecords(store, recordType)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Clear records from a certain type from the State Store.
+ *
+ * @param store State Store to remove records from.
+ * @param recordClass Class of the records to remove.
+ * @return If the State Store was cleared.
+ * @throws IOException If it cannot clear the State Store.
+ */
+ public static <T extends BaseRecord> boolean clearRecords(
+ StateStoreService store, Class<T> recordClass) throws IOException {
+ List<T> emptyList = new ArrayList<>();
+ if (!synchronizeRecords(store, emptyList, recordClass)) {
+ return false;
+ }
+ store.refreshCaches(true);
+ return true;
+ }
+
+ /**
+ * Synchronize a set of records. Remove all and keep the ones specified.
+ *
+ * @param stateStore State Store service managing the driver.
+ * @param records Records to add.
+ * @param clazz Class of the record to synchronize.
+ * @return If the synchronization succeeded.
+ * @throws IOException If it cannot connect to the State Store.
+ */
+ public static <T extends BaseRecord> boolean synchronizeRecords(
+ StateStoreService stateStore, List<T> records, Class<T> clazz)
+ throws IOException {
+ StateStoreDriver driver = stateStore.getDriver();
+ driver.verifyDriverReady();
+ if (driver.removeAll(clazz)) {
+ if (driver.putAll(records, true, false)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public static List<MountTable> createMockMountTable(
+ List<String> nameservices) throws IOException {
+ // create table entries
+ List<MountTable> entries = new ArrayList<>();
+ for (String ns : nameservices) {
+ Map<String, String> destMap = new HashMap<>();
+ destMap.put(ns, "/target-" + ns);
+ MountTable entry = MountTable.newInstance("/" + ns, destMap);
+ entries.add(entry);
+ }
+ return entries;
+ }
+
+ public static MembershipState createMockRegistrationForNamenode(
+ String nameserviceId, String namenodeId,
+ FederationNamenodeServiceState state) throws IOException {
+ MembershipState entry = MembershipState.newInstance(
+ "routerId", nameserviceId, namenodeId, "clusterId", "test",
+ "0.0.0.0:0", "0.0.0.0:0", "0.0.0.0:0", "0.0.0.0:0", state, false);
+ MembershipStats stats = MembershipStats.newInstance();
+ stats.setNumOfActiveDatanodes(100);
+ stats.setNumOfDeadDatanodes(10);
+ stats.setNumOfDecommissioningDatanodes(20);
+ stats.setNumOfDecomActiveDatanodes(15);
+ stats.setNumOfDecomDeadDatanodes(5);
+ stats.setNumOfBlocks(10);
+ entry.setStats(stats);
+ return entry;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreBase.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreBase.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreBase.java
new file mode 100644
index 0000000..247d866
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreBase.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.newStateStore;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.getStateStoreConfiguration;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.waitStateStore;
+import static org.junit.Assert.assertNotNull;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+/**
+ * Test the basic {@link StateStoreService} {@link MountTableStore}
+ * functionality.
+ */
+public class TestStateStoreBase {
+
+ private static StateStoreService stateStore;
+ private static Configuration conf;
+
+ protected static StateStoreService getStateStore() {
+ return stateStore;
+ }
+
+ protected static Configuration getConf() {
+ return conf;
+ }
+
+ @BeforeClass
+ public static void createBase() throws IOException, InterruptedException {
+
+ conf = getStateStoreConfiguration();
+
+ // Disable auto-reconnect to data store
+ conf.setLong(RBFConfigKeys.FEDERATION_STORE_CONNECTION_TEST_MS,
+ TimeUnit.HOURS.toMillis(1));
+ }
+
+ @AfterClass
+ public static void destroyBase() throws Exception {
+ if (stateStore != null) {
+ stateStore.stop();
+ stateStore.close();
+ stateStore = null;
+ }
+ }
+
+ @Before
+ public void setupBase() throws IOException, InterruptedException,
+ InstantiationException, IllegalAccessException {
+ if (stateStore == null) {
+ stateStore = newStateStore(conf);
+ assertNotNull(stateStore);
+ }
+ // Wait for state store to connect
+ stateStore.loadDriver();
+ waitStateStore(stateStore, TimeUnit.SECONDS.toMillis(10));
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateNamenodeRegistrationRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateNamenodeRegistrationRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateNamenodeRegistrationRequestPBImpl.java
deleted file mode 100644
index 5091360..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateNamenodeRegistrationRequestPBImpl.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateNamenodeRegistrationRequestProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateNamenodeRegistrationRequestProtoOrBuilder;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateNamenodeRegistrationRequest;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * OverrideNamenodeRegistrationRequest.
- */
-public class UpdateNamenodeRegistrationRequestPBImpl
- extends UpdateNamenodeRegistrationRequest implements PBRecord {
-
- private FederationProtocolPBTranslator<
- UpdateNamenodeRegistrationRequestProto,
- UpdateNamenodeRegistrationRequestProto.Builder,
- UpdateNamenodeRegistrationRequestProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<
- UpdateNamenodeRegistrationRequestProto,
- UpdateNamenodeRegistrationRequestProto.Builder,
- UpdateNamenodeRegistrationRequestProtoOrBuilder>(
- UpdateNamenodeRegistrationRequestProto.class);
-
- public UpdateNamenodeRegistrationRequestPBImpl() {
- }
-
- @Override
- public UpdateNamenodeRegistrationRequestProto getProto() {
- return this.translator.build();
- }
-
- @Override
- public void setProto(Message protocol) {
- this.translator.setProto(protocol);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public String getNameserviceId() {
- return this.translator.getProtoOrBuilder().getNameserviceId();
- }
-
- @Override
- public String getNamenodeId() {
- return this.translator.getProtoOrBuilder().getNamenodeId();
- }
-
- @Override
- public FederationNamenodeServiceState getState() {
- return FederationNamenodeServiceState
- .valueOf(this.translator.getProtoOrBuilder().getState());
- }
-
- @Override
- public void setNameserviceId(String nsId) {
- this.translator.getBuilder().setNameserviceId(nsId);
- }
-
- @Override
- public void setNamenodeId(String nnId) {
- this.translator.getBuilder().setNamenodeId(nnId);
- }
-
- @Override
- public void setState(FederationNamenodeServiceState state) {
- this.translator.getBuilder().setState(state.toString());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateNamenodeRegistrationResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateNamenodeRegistrationResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateNamenodeRegistrationResponsePBImpl.java
deleted file mode 100644
index 4558f06..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/UpdateNamenodeRegistrationResponsePBImpl.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateNamenodeRegistrationResponseProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateNamenodeRegistrationResponseProtoOrBuilder;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateNamenodeRegistrationResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the state store API object
- * OverrideNamenodeRegistrationResponse.
- */
-public class UpdateNamenodeRegistrationResponsePBImpl
- extends UpdateNamenodeRegistrationResponse implements PBRecord {
-
- private FederationProtocolPBTranslator<
- UpdateNamenodeRegistrationResponseProto,
- UpdateNamenodeRegistrationResponseProto.Builder,
- UpdateNamenodeRegistrationResponseProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<
- UpdateNamenodeRegistrationResponseProto,
- UpdateNamenodeRegistrationResponseProto.Builder,
- UpdateNamenodeRegistrationResponseProtoOrBuilder>(
- UpdateNamenodeRegistrationResponseProto.class);
-
- public UpdateNamenodeRegistrationResponsePBImpl() {
- }
-
- @Override
- public UpdateNamenodeRegistrationResponseProto getProto() {
- return this.translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public boolean getResult() {
- return this.translator.getProtoOrBuilder().getStatus();
- }
-
- @Override
- public void setResult(boolean result) {
- this.translator.getBuilder().setStatus(result);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/package-info.java
deleted file mode 100644
index 43c94be..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/package-info.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Protobuf implementations of FederationProtocolBase request/response objects
- * used by state store APIs. Each state store API is defined in the
- * org.apache.hadoop.hdfs.server.federation.store.protocol package.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/package-info.java
deleted file mode 100644
index 0249d2c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/package-info.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Contains the abstract definitions of the API request and response objects for
- * the various state store APIs. The state store supports multiple interface
- * APIs and multiple data records. Each protocol object requires a serialization
- * implementation, the default is protobuf.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java
deleted file mode 100644
index d5e60ce..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/BaseRecord.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.records;
-
-import java.util.Map;
-
-import org.apache.hadoop.util.Time;
-
-/**
- * Abstract base of a data record in the StateStore. All StateStore records are
- * derived from this class. Data records are persisted in the data store and
- * are identified by their primary key. Each data record contains:
- * <ul>
- * <li>A primary key consisting of a combination of record data fields.
- * <li>A modification date.
- * <li>A creation date.
- * </ul>
- */
-public abstract class BaseRecord implements Comparable<BaseRecord> {
- public static final String ERROR_MSG_CREATION_TIME_NEGATIVE =
- "The creation time for the record cannot be negative.";
- public static final String ERROR_MSG_MODIFICATION_TIME_NEGATIVE =
- "The modification time for the record cannot be negative.";
-
- /**
- * Set the modification time for the record.
- *
- * @param time Modification time of the record.
- */
- public abstract void setDateModified(long time);
-
- /**
- * Get the modification time for the record.
- *
- * @return Modification time of the record.
- */
- public abstract long getDateModified();
-
- /**
- * Set the creation time for the record.
- *
- * @param time Creation time of the record.
- */
- public abstract void setDateCreated(long time);
-
- /**
- * Get the creation time for the record.
- *
- * @return Creation time of the record
- */
- public abstract long getDateCreated();
-
- /**
- * Get the expiration time for the record.
- *
- * @return Expiration time for the record.
- */
- public abstract long getExpirationMs();
-
- /**
- * Map of primary key names->values for the record. The primary key can be a
- * combination of 1-n different State Store serialized values.
- *
- * @return Map of key/value pairs that constitute this object's primary key.
- */
- public abstract Map<String, String> getPrimaryKeys();
-
- /**
- * Initialize the object.
- */
- public void init() {
- // Call this after the object has been constructed
- initDefaultTimes();
- }
-
- /**
- * Initialize default times. The driver may update these timestamps on insert
- * and/or update. This should only be called when initializing an object that
- * is not backed by a data store.
- */
- private void initDefaultTimes() {
- long now = Time.now();
- this.setDateCreated(now);
- this.setDateModified(now);
- }
-
- /**
- * Join the primary keys into one single primary key.
- *
- * @return A string that is guaranteed to be unique amongst all records of
- * this type.
- */
- public String getPrimaryKey() {
- return generateMashupKey(getPrimaryKeys());
- }
-
- /**
- * Generates a cache key from a map of values.
- *
- * @param keys Map of values.
- * @return String mashup of key values.
- */
- protected static String generateMashupKey(final Map<String, String> keys) {
- StringBuilder builder = new StringBuilder();
- for (Object value : keys.values()) {
- if (builder.length() > 0) {
- builder.append("-");
- }
- builder.append(value);
- }
- return builder.toString();
- }
-
- /**
- * Check if this record matches a partial record.
- *
- * @param other Partial record.
- * @return If this record matches.
- */
- public boolean like(BaseRecord other) {
- if (other == null) {
- return false;
- }
- Map<String, String> thisKeys = this.getPrimaryKeys();
- Map<String, String> otherKeys = other.getPrimaryKeys();
- if (thisKeys == null) {
- return otherKeys == null;
- }
- return thisKeys.equals(otherKeys);
- }
-
- /**
- * Override equals check to use primary key(s) for comparison.
- */
- @Override
- public boolean equals(Object obj) {
- if (!(obj instanceof BaseRecord)) {
- return false;
- }
-
- BaseRecord baseObject = (BaseRecord) obj;
- Map<String, String> keyset1 = this.getPrimaryKeys();
- Map<String, String> keyset2 = baseObject.getPrimaryKeys();
- return keyset1.equals(keyset2);
- }
-
- /**
- * Override hash code to use primary key(s) for comparison.
- */
- @Override
- public int hashCode() {
- Map<String, String> keyset = this.getPrimaryKeys();
- return keyset.hashCode();
- }
-
- @Override
- public int compareTo(BaseRecord record) {
- if (record == null) {
- return -1;
- }
- // Descending date order
- return (int) (record.getDateModified() - this.getDateModified());
- }
-
- /**
- * Called when the modification time and current time is available, checks for
- * expirations.
- *
- * @param currentTime The current timestamp in ms from the data store, to be
- * compared against the modification and creation dates of the
- * object.
- * @return boolean True if the record has been updated and should be
- * committed to the data store. Override for customized behavior.
- */
- public boolean checkExpired(long currentTime) {
- long expiration = getExpirationMs();
- if (getDateModified() > 0 && expiration > 0) {
- return (getDateModified() + expiration) < currentTime;
- }
- return false;
- }
-
- /**
- * Validates the record. Called when the record is created, populated from the
- * state store, and before committing to the state store. If validate failed,
- * there throws an exception.
- */
- public void validate() {
- if (getDateCreated() <= 0) {
- throw new IllegalArgumentException(ERROR_MSG_CREATION_TIME_NEGATIVE);
- } else if (getDateModified() <= 0) {
- throw new IllegalArgumentException(ERROR_MSG_MODIFICATION_TIME_NEGATIVE);
- }
- }
-
- @Override
- public String toString() {
- return getPrimaryKey();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipState.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipState.java
deleted file mode 100644
index e33dedf..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipState.java
+++ /dev/null
@@ -1,336 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.records;
-
-import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.ACTIVE;
-import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.EXPIRED;
-import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.UNAVAILABLE;
-
-import java.io.IOException;
-import java.util.Comparator;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-
-/**
- * Data schema for storing NN registration information in the
- * {@link org.apache.hadoop.hdfs.server.federation.store.StateStoreService
- * FederationStateStoreService}.
- */
-public abstract class MembershipState extends BaseRecord
- implements FederationNamenodeContext {
- public static final String ERROR_MSG_NO_NS_SPECIFIED =
- "Invalid registration, no nameservice specified ";
- public static final String ERROR_MSG_NO_WEB_ADDR_SPECIFIED =
- "Invalid registration, no web address specified ";
- public static final String ERROR_MSG_NO_RPC_ADDR_SPECIFIED =
- "Invalid registration, no rpc address specified ";
- public static final String ERROR_MSG_NO_BP_SPECIFIED =
- "Invalid registration, no block pool specified ";
-
- /** Expiration time in ms for this entry. */
- private static long expirationMs;
-
-
- /** Comparator based on the name.*/
- public static final Comparator<MembershipState> NAME_COMPARATOR =
- new Comparator<MembershipState>() {
- public int compare(MembershipState m1, MembershipState m2) {
- return m1.compareNameTo(m2);
- }
- };
-
-
- /**
- * Constructors.
- */
- public MembershipState() {
- super();
- }
-
- /**
- * Create a new membership instance.
- * @return Membership instance.
- * @throws IOException
- */
- public static MembershipState newInstance() {
- MembershipState record =
- StateStoreSerializer.newRecord(MembershipState.class);
- record.init();
- return record;
- }
-
- /**
- * Create a new membership instance.
- *
- * @param router Identifier of the router.
- * @param nameservice Identifier of the nameservice.
- * @param namenode Identifier of the namenode.
- * @param clusterId Identifier of the cluster.
- * @param blockPoolId Identifier of the blockpool.
- * @param rpcAddress RPC address.
- * @param serviceAddress Service RPC address.
- * @param lifelineAddress Lifeline RPC address.
- * @param webAddress HTTP address.
- * @param state State of the federation.
- * @param safemode If the safe mode is enabled.
- * @return Membership instance.
- * @throws IOException If we cannot create the instance.
- */
- public static MembershipState newInstance(String router, String nameservice,
- String namenode, String clusterId, String blockPoolId, String rpcAddress,
- String serviceAddress, String lifelineAddress, String webAddress,
- FederationNamenodeServiceState state, boolean safemode) {
-
- MembershipState record = MembershipState.newInstance();
- record.setRouterId(router);
- record.setNameserviceId(nameservice);
- record.setNamenodeId(namenode);
- record.setRpcAddress(rpcAddress);
- record.setServiceAddress(serviceAddress);
- record.setLifelineAddress(lifelineAddress);
- record.setWebAddress(webAddress);
- record.setIsSafeMode(safemode);
- record.setState(state);
- record.setClusterId(clusterId);
- record.setBlockPoolId(blockPoolId);
- record.validate();
- return record;
- }
-
- public abstract void setRouterId(String routerId);
-
- public abstract String getRouterId();
-
- public abstract void setNameserviceId(String nameserviceId);
-
- public abstract void setNamenodeId(String namenodeId);
-
- public abstract void setWebAddress(String webAddress);
-
- public abstract void setRpcAddress(String rpcAddress);
-
- public abstract void setServiceAddress(String serviceAddress);
-
- public abstract void setLifelineAddress(String lifelineAddress);
-
- public abstract void setIsSafeMode(boolean isSafeMode);
-
- public abstract void setClusterId(String clusterId);
-
- public abstract void setBlockPoolId(String blockPoolId);
-
- public abstract void setState(FederationNamenodeServiceState state);
-
- public abstract String getNameserviceId();
-
- public abstract String getNamenodeId();
-
- public abstract String getClusterId();
-
- public abstract String getBlockPoolId();
-
- public abstract String getRpcAddress();
-
- public abstract String getServiceAddress();
-
- public abstract String getLifelineAddress();
-
- public abstract String getWebAddress();
-
- public abstract boolean getIsSafeMode();
-
- public abstract FederationNamenodeServiceState getState();
-
- public abstract void setStats(MembershipStats stats);
-
- public abstract MembershipStats getStats();
-
- public abstract void setLastContact(long contact);
-
- public abstract long getLastContact();
-
- @Override
- public boolean like(BaseRecord o) {
- if (o instanceof MembershipState) {
- MembershipState other = (MembershipState)o;
- if (getRouterId() != null &&
- !getRouterId().equals(other.getRouterId())) {
- return false;
- }
- if (getNameserviceId() != null &&
- !getNameserviceId().equals(other.getNameserviceId())) {
- return false;
- }
- if (getNamenodeId() != null &&
- !getNamenodeId().equals(other.getNamenodeId())) {
- return false;
- }
- if (getRpcAddress() != null &&
- !getRpcAddress().equals(other.getRpcAddress())) {
- return false;
- }
- if (getClusterId() != null &&
- !getClusterId().equals(other.getClusterId())) {
- return false;
- }
- if (getBlockPoolId() != null &&
- !getBlockPoolId().equals(other.getBlockPoolId())) {
- return false;
- }
- if (getState() != null &&
- !getState().equals(other.getState())) {
- return false;
- }
- return true;
- }
- return false;
- }
-
- @Override
- public String toString() {
- return getRouterId() + "->" + getNameserviceId() + ":" + getNamenodeId()
- + ":" + getRpcAddress() + "-" + getState();
- }
-
- @Override
- public SortedMap<String, String> getPrimaryKeys() {
- SortedMap<String, String> map = new TreeMap<String, String>();
- map.put("routerId", getRouterId());
- map.put("nameserviceId", getNameserviceId());
- map.put("namenodeId", getNamenodeId());
- return map;
- }
-
- /**
- * Check if the namenode is available.
- *
- * @return If the namenode is available.
- */
- public boolean isAvailable() {
- return getState() == ACTIVE;
- }
-
- /**
- * Validates the entry. Throws an IllegalArgementException if the data record
- * is missing required information.
- */
- @Override
- public void validate() {
- super.validate();
- if (getNameserviceId() == null || getNameserviceId().length() == 0) {
- throw new IllegalArgumentException(
- ERROR_MSG_NO_NS_SPECIFIED + this);
- }
- if (getWebAddress() == null || getWebAddress().length() == 0) {
- throw new IllegalArgumentException(
- ERROR_MSG_NO_WEB_ADDR_SPECIFIED + this);
- }
- if (getRpcAddress() == null || getRpcAddress().length() == 0) {
- throw new IllegalArgumentException(
- ERROR_MSG_NO_RPC_ADDR_SPECIFIED + this);
- }
- if (!isBadState() &&
- (getBlockPoolId().isEmpty() || getBlockPoolId().length() == 0)) {
- throw new IllegalArgumentException(
- ERROR_MSG_NO_BP_SPECIFIED + this);
- }
- }
-
-
- /**
- * Overrides the cached getBlockPoolId() with an update. The state will be
- * reset when the cache is flushed
- *
- * @param newState Service state of the namenode.
- */
- public void overrideState(FederationNamenodeServiceState newState) {
- this.setState(newState);
- }
-
- /**
- * Sort by nameservice, namenode, and router.
- *
- * @param other Another membership to compare to.
- * @return If this object goes before the parameter.
- */
- public int compareNameTo(MembershipState other) {
- int ret = this.getNameserviceId().compareTo(other.getNameserviceId());
- if (ret == 0) {
- ret = this.getNamenodeId().compareTo(other.getNamenodeId());
- }
- if (ret == 0) {
- ret = this.getRouterId().compareTo(other.getRouterId());
- }
- return ret;
- }
-
- /**
- * Get the identifier of this namenode registration.
- * @return Identifier of the namenode.
- */
- public String getNamenodeKey() {
- return getNamenodeKey(this.getNameserviceId(), this.getNamenodeId());
- }
-
- /**
- * Generate the identifier for a Namenode in the HDFS federation.
- *
- * @param nsId Nameservice of the Namenode.
- * @param nnId Namenode within the Nameservice (HA).
- * @return Namenode identifier within the federation.
- */
- public static String getNamenodeKey(String nsId, String nnId) {
- return nsId + "-" + nnId;
- }
-
- /**
- * Check if the membership is in a bad state (expired or unavailable).
- * @return If the membership is in a bad state (expired or unavailable).
- */
- private boolean isBadState() {
- return this.getState() == EXPIRED || this.getState() == UNAVAILABLE;
- }
-
- @Override
- public boolean checkExpired(long currentTime) {
- if (super.checkExpired(currentTime)) {
- this.setState(EXPIRED);
- // Commit it
- return true;
- }
- return false;
- }
-
- @Override
- public long getExpirationMs() {
- return MembershipState.expirationMs;
- }
-
- /**
- * Set the expiration time for this class.
- *
- * @param time Expiration time in milliseconds.
- */
- public static void setExpirationMs(long time) {
- MembershipState.expirationMs = time;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipStats.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipStats.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipStats.java
deleted file mode 100644
index 654140c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipStats.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.records;
-
-import java.io.IOException;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-
-/**
- * Data schema for storing NN stats in the
- * {@link org.apache.hadoop.hdfs.server.federation.store.StateStoreService
- * StateStoreService}.
- */
-public abstract class MembershipStats extends BaseRecord {
-
- public static MembershipStats newInstance() throws IOException {
- MembershipStats record =
- StateStoreSerializer.newRecord(MembershipStats.class);
- record.init();
- return record;
- }
-
- public abstract void setTotalSpace(long space);
-
- public abstract long getTotalSpace();
-
- public abstract void setAvailableSpace(long space);
-
- public abstract long getAvailableSpace();
-
- public abstract void setProvidedSpace(long capacity);
-
- public abstract long getProvidedSpace();
-
- public abstract void setNumOfFiles(long files);
-
- public abstract long getNumOfFiles();
-
- public abstract void setNumOfBlocks(long blocks);
-
- public abstract long getNumOfBlocks();
-
- public abstract void setNumOfBlocksMissing(long blocks);
-
- public abstract long getNumOfBlocksMissing();
-
- public abstract void setNumOfBlocksPendingReplication(long blocks);
-
- public abstract long getNumOfBlocksPendingReplication();
-
- public abstract void setNumOfBlocksUnderReplicated(long blocks);
-
- public abstract long getNumOfBlocksUnderReplicated();
-
- public abstract void setNumOfBlocksPendingDeletion(long blocks);
-
- public abstract long getNumOfBlocksPendingDeletion();
-
- public abstract void setNumOfActiveDatanodes(int nodes);
-
- public abstract int getNumOfActiveDatanodes();
-
- public abstract void setNumOfDeadDatanodes(int nodes);
-
- public abstract int getNumOfDeadDatanodes();
-
- public abstract void setNumOfDecommissioningDatanodes(int nodes);
-
- public abstract int getNumOfDecommissioningDatanodes();
-
- public abstract void setNumOfDecomActiveDatanodes(int nodes);
-
- public abstract int getNumOfDecomActiveDatanodes();
-
- public abstract void setNumOfDecomDeadDatanodes(int nodes);
-
- public abstract int getNumOfDecomDeadDatanodes();
-
- @Override
- public SortedMap<String, String> getPrimaryKeys() {
- // This record is not stored directly, no key needed
- SortedMap<String, String> map = new TreeMap<String, String>();
- return map;
- }
-
- @Override
- public long getExpirationMs() {
- // This record is not stored directly, no expiration needed
- return -1;
- }
-
- @Override
- public void setDateModified(long time) {
- // We don't store this record directly
- }
-
- @Override
- public long getDateModified() {
- // We don't store this record directly
- return 0;
- }
-
- @Override
- public void setDateCreated(long time) {
- // We don't store this record directly
- }
-
- @Override
- public long getDateCreated() {
- // We don't store this record directly
- return 0;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
deleted file mode 100644
index 0eab043..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
+++ /dev/null
@@ -1,430 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.records;
-
-import java.io.IOException;
-import java.util.Comparator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
-import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
-import org.apache.hadoop.hdfs.server.federation.router.RouterPermissionChecker;
-import org.apache.hadoop.hdfs.server.federation.router.RouterQuotaUsage;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Data schema for
- * {@link org.apache.hadoop.hdfs.server.federation.store.
- * MountTableStore FederationMountTableStore} data stored in the
- * {@link org.apache.hadoop.hdfs.server.federation.store.
- * StateStoreService FederationStateStoreService}. Supports string
- * serialization.
- */
-public abstract class MountTable extends BaseRecord {
-
- private static final Logger LOG = LoggerFactory.getLogger(MountTable.class);
- public static final String ERROR_MSG_NO_SOURCE_PATH =
- "Invalid entry, no source path specified ";
- public static final String ERROR_MSG_MUST_START_WITH_BACK_SLASH =
- "Invalid entry, all mount points must start with / ";
- public static final String ERROR_MSG_NO_DEST_PATH_SPECIFIED =
- "Invalid entry, no destination paths specified ";
- public static final String ERROR_MSG_INVAILD_DEST_NS =
- "Invalid entry, invalid destination nameservice ";
- public static final String ERROR_MSG_INVAILD_DEST_PATH =
- "Invalid entry, invalid destination path ";
- public static final String ERROR_MSG_ALL_DEST_MUST_START_WITH_BACK_SLASH =
- "Invalid entry, all destination must start with / ";
-
- /** Comparator for paths which considers the /. */
- public static final Comparator<String> PATH_COMPARATOR =
- new Comparator<String>() {
- @Override
- public int compare(String o1, String o2) {
- String s1 = o1.replace('/', ' ');
- String s2 = o2.replace('/', ' ');
- return s1.compareTo(s2);
- }
- };
-
- /** Comparator based on the mount table source. */
- public static final Comparator<MountTable> SOURCE_COMPARATOR =
- new Comparator<MountTable>() {
- public int compare(MountTable m1, MountTable m2) {
- String src1 = m1.getSourcePath();
- String src2 = m2.getSourcePath();
- return PATH_COMPARATOR.compare(src1, src2);
- }
- };
-
-
- /**
- * Default constructor for a mount table entry.
- */
- public MountTable() {
- super();
- }
-
- public static MountTable newInstance() {
- MountTable record = StateStoreSerializer.newRecord(MountTable.class);
- record.init();
- return record;
- }
-
- /**
- * Constructor for a mount table entry with a single destinations.
- *
- * @param src Source path in the mount entry.
- * @param destinations Nameservice destination of the mount point.
- * @param dateCreated Created date.
- * @param dateModified Modified date.
- * @throws IOException
- */
- public static MountTable newInstance(final String src,
- final Map<String, String> destinations,
- long dateCreated, long dateModified) throws IOException {
-
- MountTable record = newInstance(src, destinations);
- record.setDateCreated(dateCreated);
- record.setDateModified(dateModified);
- return record;
- }
-
- /**
- * Constructor for a mount table entry with multiple destinations.
- *
- * @param src Source path in the mount entry.
- * @param destinations Nameservice destinations of the mount point.
- * @throws IOException
- */
- public static MountTable newInstance(final String src,
- final Map<String, String> destinations) throws IOException {
- MountTable record = newInstance();
-
- // Normalize the mount path
- record.setSourcePath(normalizeFileSystemPath(src));
-
- // Build a list of remote locations
- final List<RemoteLocation> locations = new LinkedList<>();
- for (Entry<String, String> entry : destinations.entrySet()) {
- String nsId = entry.getKey();
- String path = normalizeFileSystemPath(entry.getValue());
- RemoteLocation location = new RemoteLocation(nsId, path);
- locations.add(location);
- }
-
- // Set the serialized dest string
- record.setDestinations(locations);
-
- // Set permission fields
- UserGroupInformation ugi = NameNode.getRemoteUser();
- record.setOwnerName(ugi.getShortUserName());
- String group = ugi.getGroups().isEmpty() ? ugi.getShortUserName()
- : ugi.getPrimaryGroupName();
- record.setGroupName(group);
- record.setMode(new FsPermission(
- RouterPermissionChecker.MOUNT_TABLE_PERMISSION_DEFAULT));
-
- // Set quota for mount table
- RouterQuotaUsage quota = new RouterQuotaUsage.Builder()
- .fileAndDirectoryCount(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT)
- .quota(HdfsConstants.QUOTA_DONT_SET)
- .spaceConsumed(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT)
- .spaceQuota(HdfsConstants.QUOTA_DONT_SET).build();
- record.setQuota(quota);
-
- // Validate
- record.validate();
- return record;
- }
-
- /**
- * Get source path in the federated namespace.
- *
- * @return Source path in the federated namespace.
- */
- public abstract String getSourcePath();
-
- /**
- * Set source path in the federated namespace.
- *
- * @param path Source path in the federated namespace.
- */
- public abstract void setSourcePath(String path);
-
- /**
- * Get a list of destinations (namespace + path) present for this entry.
- *
- * @return List of RemoteLocation destinations. Null if no destinations.
- */
- public abstract List<RemoteLocation> getDestinations();
-
- /**
- * Set the destination paths.
- *
- * @param paths Destination paths.
- */
- public abstract void setDestinations(List<RemoteLocation> dests);
-
- /**
- * Add a new destination to this mount table entry.
- */
- public abstract boolean addDestination(String nsId, String path);
-
- /**
- * Check if the entry is read only.
- *
- * @return If the entry is read only.
- */
- public abstract boolean isReadOnly();
-
- /**
- * Set an entry to be read only.
- *
- * @param ro If the entry is read only.
- */
- public abstract void setReadOnly(boolean ro);
-
- /**
- * Get the order of the destinations for this mount table entry.
- *
- * @return Order of the destinations.
- */
- public abstract DestinationOrder getDestOrder();
-
- /**
- * Set the order of the destinations for this mount table entry.
- *
- * @param order Order of the destinations.
- */
- public abstract void setDestOrder(DestinationOrder order);
-
- /**
- * Get owner name of this mount table entry.
- *
- * @return Owner name
- */
- public abstract String getOwnerName();
-
- /**
- * Set owner name of this mount table entry.
- *
- * @param owner Owner name for mount table entry
- */
- public abstract void setOwnerName(String owner);
-
- /**
- * Get group name of this mount table entry.
- *
- * @return Group name
- */
- public abstract String getGroupName();
-
- /**
- * Set group name of this mount table entry.
- *
- * @param group Group name for mount table entry
- */
- public abstract void setGroupName(String group);
-
- /**
- * Get permission of this mount table entry.
- *
- * @return FsPermission permission mode
- */
- public abstract FsPermission getMode();
-
- /**
- * Set permission for this mount table entry.
- *
- * @param mode Permission for mount table entry
- */
- public abstract void setMode(FsPermission mode);
-
- /**
- * Get quota of this mount table entry.
- *
- * @return RouterQuotaUsage quota usage
- */
- public abstract RouterQuotaUsage getQuota();
-
- /**
- * Set quota for this mount table entry.
- *
- * @param quota QuotaUsage for mount table entry
- */
- public abstract void setQuota(RouterQuotaUsage quota);
-
- /**
- * Get the default location.
- * @return The default location.
- */
- public RemoteLocation getDefaultLocation() {
- List<RemoteLocation> dests = this.getDestinations();
- if (dests == null || dests.isEmpty()) {
- return null;
- }
- return dests.get(0);
- }
-
- @Override
- public boolean like(final BaseRecord o) {
- if (o instanceof MountTable) {
- MountTable other = (MountTable)o;
- if (getSourcePath() != null &&
- !getSourcePath().equals(other.getSourcePath())) {
- return false;
- }
- if (getDestinations() != null &&
- !getDestinations().equals(other.getDestinations())) {
- return false;
- }
- return true;
- }
- return false;
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append(this.getSourcePath());
- sb.append("->");
- List<RemoteLocation> destinations = this.getDestinations();
- sb.append(destinations);
- if (destinations != null && destinations.size() > 1) {
- sb.append("[" + this.getDestOrder() + "]");
- }
- if (this.isReadOnly()) {
- sb.append("[RO]");
- }
-
- if (this.getOwnerName() != null) {
- sb.append("[owner:").append(this.getOwnerName()).append("]");
- }
-
- if (this.getGroupName() != null) {
- sb.append("[group:").append(this.getGroupName()).append("]");
- }
-
- if (this.getMode() != null) {
- sb.append("[mode:").append(this.getMode()).append("]");
- }
-
- if (this.getQuota() != null) {
- sb.append("[quota:").append(this.getQuota()).append("]");
- }
-
- return sb.toString();
- }
-
- @Override
- public SortedMap<String, String> getPrimaryKeys() {
- SortedMap<String, String> map = new TreeMap<>();
- map.put("sourcePath", this.getSourcePath());
- return map;
- }
-
- @Override
- public void validate() {
- super.validate();
- if (this.getSourcePath() == null || this.getSourcePath().length() == 0) {
- throw new IllegalArgumentException(
- ERROR_MSG_NO_SOURCE_PATH + this);
- }
- if (!this.getSourcePath().startsWith("/")) {
- throw new IllegalArgumentException(
- ERROR_MSG_MUST_START_WITH_BACK_SLASH + this);
- }
- if (this.getDestinations() == null || this.getDestinations().size() == 0) {
- throw new IllegalArgumentException(
- ERROR_MSG_NO_DEST_PATH_SPECIFIED + this);
- }
- for (RemoteLocation loc : getDestinations()) {
- String nsId = loc.getNameserviceId();
- if (nsId == null || nsId.length() == 0) {
- throw new IllegalArgumentException(
- ERROR_MSG_INVAILD_DEST_NS + this);
- }
- if (loc.getDest() == null || loc.getDest().length() == 0) {
- throw new IllegalArgumentException(
- ERROR_MSG_INVAILD_DEST_PATH + this);
- }
- if (!loc.getDest().startsWith("/")) {
- throw new IllegalArgumentException(
- ERROR_MSG_ALL_DEST_MUST_START_WITH_BACK_SLASH + this);
- }
- }
- }
-
- @Override
- public long getExpirationMs() {
- return 0;
- }
-
- @Override
- public int hashCode() {
- return new HashCodeBuilder(17, 31)
- .append(this.getSourcePath())
- .append(this.getDestinations())
- .append(this.isReadOnly())
- .append(this.getDestOrder())
- .toHashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj instanceof MountTable) {
- MountTable other = (MountTable)obj;
- if (!this.getSourcePath().equals(other.getSourcePath())) {
- return false;
- } else if (!this.getDestinations().equals(other.getDestinations())) {
- return false;
- } else if (this.isReadOnly() != other.isReadOnly()) {
- return false;
- } else if (!this.getDestOrder().equals(other.getDestOrder())) {
- return false;
- }
- return true;
- }
- return false;
- }
-
- /**
- * Normalize a path for that filesystem.
- *
- * @param path Path to normalize.
- * @return Normalized path.
- */
- private static String normalizeFileSystemPath(final String path) {
- Path normalizedPath = new Path(path);
- return normalizedPath.toString();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/Query.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/Query.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/Query.java
deleted file mode 100644
index 3c59abf..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/Query.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.records;
-
-/**
- * Check if a record matches a query. The query is usually a partial record.
- *
- * @param <T> Type of the record to query.
- */
-public class Query<T extends BaseRecord> {
-
- /** Partial object to compare against. */
- private final T partial;
-
-
- /**
- * Create a query to search for a partial record.
- *
- * @param partial It defines the attributes to search.
- */
- public Query(final T part) {
- this.partial = part;
- }
-
- /**
- * Get the partial record used to query.
- *
- * @return The partial record used for the query.
- */
- public T getPartial() {
- return this.partial;
- }
-
- /**
- * Check if a record matches the primary keys or the partial record.
- *
- * @param other Record to check.
- * @return If the record matches. Don't match if there is no partial.
- */
- public boolean matches(T other) {
- if (this.partial == null) {
- return false;
- }
- return this.partial.like(other);
- }
-
- @Override
- public String toString() {
- return "Checking: " + this.partial;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/QueryResult.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/QueryResult.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/QueryResult.java
deleted file mode 100644
index 64c2c71..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/QueryResult.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.records;
-
-import java.util.Collections;
-import java.util.List;
-
-/**
- * Encapsulates a state store query result that includes a set of records and a
- * time stamp for the result.
- */
-public class QueryResult<T extends BaseRecord> {
-
- /** Data result. */
- private final List<T> records;
- /** Time stamp of the data results. */
- private final long timestamp;
-
- public QueryResult(final List<T> recs, final long time) {
- this.records = recs;
- this.timestamp = time;
- }
-
- /**
- * Get the result of the query.
- *
- * @return List of records.
- */
- public List<T> getRecords() {
- return Collections.unmodifiableList(this.records);
- }
-
- /**
- * The timetamp in driver time of this query.
- *
- * @return Timestamp in driver time.
- */
- public long getTimestamp() {
- return this.timestamp;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/RouterState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/RouterState.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/RouterState.java
deleted file mode 100644
index c90abcc..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/RouterState.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.records;
-
-import java.io.IOException;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import org.apache.hadoop.hdfs.server.federation.router.FederationUtil;
-import org.apache.hadoop.hdfs.server.federation.router.RouterServiceState;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Entry to log the state of a
- * {@link org.apache.hadoop.hdfs.server.federation.router.Router Router} in the
- * {@link org.apache.hadoop.hdfs.server.federation.store.StateStoreService
- * FederationStateStoreService}.
- */
-public abstract class RouterState extends BaseRecord {
-
- private static final Logger LOG = LoggerFactory.getLogger(RouterState.class);
-
- /** Expiration time in ms for this entry. */
- private static long expirationMs;
-
- /**
- * Constructors.
- */
- public RouterState() {
- super();
- }
-
- public static RouterState newInstance() {
- RouterState record = StateStoreSerializer.newRecord(RouterState.class);
- record.init();
- return record;
- }
-
- public static RouterState newInstance(String addr, long startTime,
- RouterServiceState status) {
- RouterState record = newInstance();
- record.setDateStarted(startTime);
- record.setAddress(addr);
- record.setStatus(status);
- record.setCompileInfo(FederationUtil.getCompileInfo());
- record.setVersion(FederationUtil.getVersion());
- return record;
- }
-
- public abstract void setAddress(String address);
-
- public abstract void setDateStarted(long dateStarted);
-
- public abstract String getAddress();
-
- public abstract StateStoreVersion getStateStoreVersion() throws IOException;
-
- public abstract void setStateStoreVersion(StateStoreVersion version);
-
- public abstract RouterServiceState getStatus();
-
- public abstract void setStatus(RouterServiceState newStatus);
-
- public abstract String getVersion();
-
- public abstract void setVersion(String version);
-
- public abstract String getCompileInfo();
-
- public abstract void setCompileInfo(String info);
-
- public abstract long getDateStarted();
-
- /**
- * Get the identifier for the Router. It uses the address.
- *
- * @return Identifier for the Router.
- */
- public String getRouterId() {
- return getAddress();
- }
-
- @Override
- public boolean like(BaseRecord o) {
- if (o instanceof RouterState) {
- RouterState other = (RouterState)o;
- if (getAddress() != null &&
- !getAddress().equals(other.getAddress())) {
- return false;
- }
- if (getStatus() != null &&
- !getStatus().equals(other.getStatus())) {
- return false;
- }
- return true;
- }
- return false;
- }
-
- @Override
- public String toString() {
- return getAddress() + " -> " + getStatus() + "," + getVersion();
- }
-
- @Override
- public SortedMap<String, String> getPrimaryKeys() {
- SortedMap<String, String> map = new TreeMap<>();
- map.put("address", getAddress());
- return map;
- }
-
- @Override
- public void validate() {
- super.validate();
- if ((getAddress() == null || getAddress().length() == 0) &&
- getStatus() != RouterServiceState.INITIALIZING) {
- throw new IllegalArgumentException(
- "Invalid router entry, no address specified " + this);
- }
- }
-
- @Override
- public int compareTo(BaseRecord other) {
- if (other == null) {
- return -1;
- } else if (other instanceof RouterState) {
- RouterState router = (RouterState) other;
- return this.getAddress().compareTo(router.getAddress());
- } else {
- return super.compareTo(other);
- }
- }
-
- @Override
- public boolean checkExpired(long currentTime) {
- if (super.checkExpired(currentTime)) {
- setStatus(RouterServiceState.EXPIRED);
- return true;
- }
- return false;
- }
-
- @Override
- public long getExpirationMs() {
- return RouterState.expirationMs;
- }
-
- public static void setExpirationMs(long time) {
- RouterState.expirationMs = time;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/StateStoreVersion.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/StateStoreVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/StateStoreVersion.java
deleted file mode 100644
index ce86eb0..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/StateStoreVersion.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.records;
-
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-
-/**
- * Entry to track the version of the State Store data stored in the State Store
- * by a Router.
- */
-public abstract class StateStoreVersion extends BaseRecord {
-
- public static StateStoreVersion newInstance() {
- return StateStoreSerializer.newRecord(StateStoreVersion.class);
- }
-
- public static StateStoreVersion newInstance(long membershipVersion,
- long mountTableVersion) {
- StateStoreVersion record = newInstance();
- record.setMembershipVersion(membershipVersion);
- record.setMountTableVersion(mountTableVersion);
- return record;
- }
-
- public abstract long getMembershipVersion();
-
- public abstract void setMembershipVersion(long version);
-
- public abstract long getMountTableVersion();
-
- public abstract void setMountTableVersion(long version);
-
- @Override
- public SortedMap<String, String> getPrimaryKeys() {
- // This record is not stored directly, no key needed
- SortedMap<String, String> map = new TreeMap<String, String>();
- return map;
- }
-
- @Override
- public long getExpirationMs() {
- // This record is not stored directly, no expiration needed
- return -1;
- }
-
- @Override
- public void setDateModified(long time) {
- // We don't store this record directly
- }
-
- @Override
- public long getDateModified() {
- // We don't store this record directly
- return 0;
- }
-
- @Override
- public void setDateCreated(long time) {
- // We don't store this record directly
- }
-
- @Override
- public long getDateCreated() {
- // We don't store this record directly
- return 0;
- }
-
- @Override
- public String toString() {
- return "Membership: " + getMembershipVersion() +
- " Mount Table: " + getMountTableVersion();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatePBImpl.java
deleted file mode 100644
index 614957b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatePBImpl.java
+++ /dev/null
@@ -1,335 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.records.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipRecordProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipRecordProto.Builder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipRecordProtoOrBuilder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipStatsRecordProto;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.FederationProtocolPBTranslator;
-import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
-import org.apache.hadoop.hdfs.server.federation.store.records.MembershipStats;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the MembershipState record.
- */
-public class MembershipStatePBImpl extends MembershipState implements PBRecord {
-
- private FederationProtocolPBTranslator<NamenodeMembershipRecordProto, Builder,
- NamenodeMembershipRecordProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<NamenodeMembershipRecordProto,
- Builder, NamenodeMembershipRecordProtoOrBuilder>(
- NamenodeMembershipRecordProto.class);
-
- public MembershipStatePBImpl() {
- }
-
- public MembershipStatePBImpl(NamenodeMembershipRecordProto proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public NamenodeMembershipRecordProto getProto() {
- return this.translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public void setRouterId(String routerId) {
- Builder builder = this.translator.getBuilder();
- if (routerId == null) {
- builder.clearRouterId();
- } else {
- builder.setRouterId(routerId);
- }
- }
-
- @Override
- public void setNameserviceId(String nameserviceId) {
- Builder builder = this.translator.getBuilder();
- if (nameserviceId == null) {
- builder.clearNameserviceId();
- } else {
- builder.setNameserviceId(nameserviceId);
- }
- }
-
- @Override
- public void setNamenodeId(String namenodeId) {
- Builder builder = this.translator.getBuilder();
- if (namenodeId == null) {
- builder.clearNamenodeId();
- } else {
- builder.setNamenodeId(namenodeId);
- }
- }
-
- @Override
- public void setWebAddress(String webAddress) {
- Builder builder = this.translator.getBuilder();
- if (webAddress == null) {
- builder.clearWebAddress();
- } else {
- builder.setWebAddress(webAddress);
- }
- }
-
- @Override
- public void setRpcAddress(String rpcAddress) {
- Builder builder = this.translator.getBuilder();
- if (rpcAddress == null) {
- builder.clearRpcAddress();
- } else {
- builder.setRpcAddress(rpcAddress);
- }
- }
-
- @Override
- public void setServiceAddress(String serviceAddress) {
- this.translator.getBuilder().setServiceAddress(serviceAddress);
- }
-
- @Override
- public void setLifelineAddress(String lifelineAddress) {
- Builder builder = this.translator.getBuilder();
- if (lifelineAddress == null) {
- builder.clearLifelineAddress();
- } else {
- builder.setLifelineAddress(lifelineAddress);
- }
- }
-
- @Override
- public void setIsSafeMode(boolean isSafeMode) {
- Builder builder = this.translator.getBuilder();
- builder.setIsSafeMode(isSafeMode);
- }
-
- @Override
- public void setClusterId(String clusterId) {
- Builder builder = this.translator.getBuilder();
- if (clusterId == null) {
- builder.clearClusterId();
- } else {
- builder.setClusterId(clusterId);
- }
- }
-
- @Override
- public void setBlockPoolId(String blockPoolId) {
- Builder builder = this.translator.getBuilder();
- if (blockPoolId == null) {
- builder.clearBlockPoolId();
- } else {
- builder.setBlockPoolId(blockPoolId);
- }
- }
-
- @Override
- public void setState(FederationNamenodeServiceState state) {
- Builder builder = this.translator.getBuilder();
- if (state == null) {
- builder.clearState();
- } else {
- builder.setState(state.toString());
- }
- }
-
- @Override
- public String getRouterId() {
- NamenodeMembershipRecordProtoOrBuilder proto =
- this.translator.getProtoOrBuilder();
- if (!proto.hasRouterId()) {
- return null;
- }
- return proto.getRouterId();
- }
-
- @Override
- public String getNameserviceId() {
- NamenodeMembershipRecordProtoOrBuilder proto =
- this.translator.getProtoOrBuilder();
- if (!proto.hasNameserviceId()) {
- return null;
- }
- return this.translator.getProtoOrBuilder().getNameserviceId();
- }
-
- @Override
- public String getNamenodeId() {
- NamenodeMembershipRecordProtoOrBuilder proto =
- this.translator.getProtoOrBuilder();
- if (!proto.hasNamenodeId()) {
- return null;
- }
- return this.translator.getProtoOrBuilder().getNamenodeId();
- }
-
- @Override
- public String getClusterId() {
- NamenodeMembershipRecordProtoOrBuilder proto =
- this.translator.getProtoOrBuilder();
- if (!proto.hasClusterId()) {
- return null;
- }
- return this.translator.getProtoOrBuilder().getClusterId();
- }
-
- @Override
- public String getBlockPoolId() {
- NamenodeMembershipRecordProtoOrBuilder proto =
- this.translator.getProtoOrBuilder();
- if (!proto.hasBlockPoolId()) {
- return null;
- }
- return this.translator.getProtoOrBuilder().getBlockPoolId();
- }
-
- @Override
- public String getRpcAddress() {
- NamenodeMembershipRecordProtoOrBuilder proto =
- this.translator.getProtoOrBuilder();
- if (!proto.hasRpcAddress()) {
- return null;
- }
- return this.translator.getProtoOrBuilder().getRpcAddress();
- }
-
- @Override
- public String getServiceAddress() {
- NamenodeMembershipRecordProtoOrBuilder proto =
- this.translator.getProtoOrBuilder();
- if (!proto.hasServiceAddress()) {
- return null;
- }
- return this.translator.getProtoOrBuilder().getServiceAddress();
- }
-
- @Override
- public String getWebAddress() {
- NamenodeMembershipRecordProtoOrBuilder proto =
- this.translator.getProtoOrBuilder();
- if (!proto.hasWebAddress()) {
- return null;
- }
- return this.translator.getProtoOrBuilder().getWebAddress();
- }
-
- @Override
- public String getLifelineAddress() {
- NamenodeMembershipRecordProtoOrBuilder proto =
- this.translator.getProtoOrBuilder();
- if (!proto.hasLifelineAddress()) {
- return null;
- }
- return this.translator.getProtoOrBuilder().getLifelineAddress();
- }
-
- @Override
- public boolean getIsSafeMode() {
- return this.translator.getProtoOrBuilder().getIsSafeMode();
- }
-
- @Override
- public FederationNamenodeServiceState getState() {
- FederationNamenodeServiceState ret =
- FederationNamenodeServiceState.UNAVAILABLE;
- NamenodeMembershipRecordProtoOrBuilder proto =
- this.translator.getProtoOrBuilder();
- if (!proto.hasState()) {
- return null;
- }
- try {
- ret = FederationNamenodeServiceState.valueOf(proto.getState());
- } catch (IllegalArgumentException e) {
- // Ignore this error
- }
- return ret;
- }
-
- @Override
- public void setStats(MembershipStats stats) {
- if (stats instanceof MembershipStatsPBImpl) {
- MembershipStatsPBImpl statsPB = (MembershipStatsPBImpl)stats;
- NamenodeMembershipStatsRecordProto statsProto =
- (NamenodeMembershipStatsRecordProto)statsPB.getProto();
- this.translator.getBuilder().setStats(statsProto);
- }
- }
-
- @Override
- public MembershipStats getStats() {
- NamenodeMembershipStatsRecordProto statsProto =
- this.translator.getProtoOrBuilder().getStats();
- MembershipStats stats =
- StateStoreSerializer.newRecord(MembershipStats.class);
- if (stats instanceof MembershipStatsPBImpl) {
- MembershipStatsPBImpl statsPB = (MembershipStatsPBImpl)stats;
- statsPB.setProto(statsProto);
- return statsPB;
- } else {
- throw new IllegalArgumentException(
- "Cannot get stats for the membership");
- }
- }
-
- @Override
- public void setLastContact(long contact) {
- this.translator.getBuilder().setLastContact(contact);
- }
-
- @Override
- public long getLastContact() {
- return this.translator.getProtoOrBuilder().getLastContact();
- }
-
- @Override
- public void setDateModified(long time) {
- this.translator.getBuilder().setDateModified(time);
- }
-
- @Override
- public long getDateModified() {
- return this.translator.getProtoOrBuilder().getDateModified();
- }
-
- @Override
- public void setDateCreated(long time) {
- this.translator.getBuilder().setDateCreated(time);
- }
-
- @Override
- public long getDateCreated() {
- return this.translator.getProtoOrBuilder().getDateCreated();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatsPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatsPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatsPBImpl.java
deleted file mode 100644
index 3347bc6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatsPBImpl.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.records.impl.pb;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipStatsRecordProto;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipStatsRecordProto.Builder;
-import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.NamenodeMembershipStatsRecordProtoOrBuilder;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.FederationProtocolPBTranslator;
-import org.apache.hadoop.hdfs.server.federation.store.records.MembershipStats;
-
-import com.google.protobuf.Message;
-
-/**
- * Protobuf implementation of the MembershipStats record.
- */
-public class MembershipStatsPBImpl extends MembershipStats
- implements PBRecord {
-
- private FederationProtocolPBTranslator<NamenodeMembershipStatsRecordProto,
- Builder, NamenodeMembershipStatsRecordProtoOrBuilder> translator =
- new FederationProtocolPBTranslator<NamenodeMembershipStatsRecordProto,
- Builder, NamenodeMembershipStatsRecordProtoOrBuilder>(
- NamenodeMembershipStatsRecordProto.class);
-
- public MembershipStatsPBImpl() {
- }
-
- @Override
- public NamenodeMembershipStatsRecordProto getProto() {
- return this.translator.build();
- }
-
- @Override
- public void setProto(Message proto) {
- this.translator.setProto(proto);
- }
-
- @Override
- public void readInstance(String base64String) throws IOException {
- this.translator.readInstance(base64String);
- }
-
- @Override
- public void setTotalSpace(long space) {
- this.translator.getBuilder().setTotalSpace(space);
- }
-
- @Override
- public long getTotalSpace() {
- return this.translator.getProtoOrBuilder().getTotalSpace();
- }
-
- @Override
- public void setAvailableSpace(long space) {
- this.translator.getBuilder().setAvailableSpace(space);
- }
-
- @Override
- public long getAvailableSpace() {
- return this.translator.getProtoOrBuilder().getAvailableSpace();
- }
-
- @Override
- public void setProvidedSpace(long capacity) {
- this.translator.getBuilder().setProvidedSpace(capacity);
- }
-
- @Override
- public long getProvidedSpace() {
- return this.translator.getProtoOrBuilder().getProvidedSpace();
- }
-
- @Override
- public void setNumOfFiles(long files) {
- this.translator.getBuilder().setNumOfFiles(files);
- }
-
- @Override
- public long getNumOfFiles() {
- return this.translator.getProtoOrBuilder().getNumOfFiles();
- }
-
- @Override
- public void setNumOfBlocks(long blocks) {
- this.translator.getBuilder().setNumOfBlocks(blocks);
- }
-
- @Override
- public long getNumOfBlocks() {
- return this.translator.getProtoOrBuilder().getNumOfBlocks();
- }
-
- @Override
- public void setNumOfBlocksMissing(long blocks) {
- this.translator.getBuilder().setNumOfBlocksMissing(blocks);
- }
-
- @Override
- public long getNumOfBlocksMissing() {
- return this.translator.getProtoOrBuilder().getNumOfBlocksMissing();
- }
-
- @Override
- public void setNumOfBlocksPendingReplication(long blocks) {
- this.translator.getBuilder().setNumOfBlocksPendingReplication(blocks);
- }
-
- @Override
- public long getNumOfBlocksPendingReplication() {
- return this.translator.getProtoOrBuilder()
- .getNumOfBlocksPendingReplication();
- }
-
- @Override
- public void setNumOfBlocksUnderReplicated(long blocks) {
- this.translator.getBuilder().setNumOfBlocksUnderReplicated(blocks);
- }
-
- @Override
- public long getNumOfBlocksUnderReplicated() {
- return this.translator.getProtoOrBuilder().getNumOfBlocksUnderReplicated();
- }
-
- @Override
- public void setNumOfBlocksPendingDeletion(long blocks) {
- this.translator.getBuilder().setNumOfBlocksPendingDeletion(blocks);
- }
-
- @Override
- public long getNumOfBlocksPendingDeletion() {
- return this.translator.getProtoOrBuilder().getNumOfBlocksPendingDeletion();
- }
-
- @Override
- public void setNumOfActiveDatanodes(int nodes) {
- this.translator.getBuilder().setNumOfActiveDatanodes(nodes);
- }
-
- @Override
- public int getNumOfActiveDatanodes() {
- return this.translator.getProtoOrBuilder().getNumOfActiveDatanodes();
- }
-
- @Override
- public void setNumOfDeadDatanodes(int nodes) {
- this.translator.getBuilder().setNumOfDeadDatanodes(nodes);
- }
-
- @Override
- public int getNumOfDeadDatanodes() {
- return this.translator.getProtoOrBuilder().getNumOfDeadDatanodes();
- }
-
- @Override
- public void setNumOfDecommissioningDatanodes(int nodes) {
- this.translator.getBuilder().setNumOfDecommissioningDatanodes(nodes);
- }
-
- @Override
- public int getNumOfDecommissioningDatanodes() {
- return this.translator.getProtoOrBuilder()
- .getNumOfDecommissioningDatanodes();
- }
-
- @Override
- public void setNumOfDecomActiveDatanodes(int nodes) {
- this.translator.getBuilder().setNumOfDecomActiveDatanodes(nodes);
- }
-
- @Override
- public int getNumOfDecomActiveDatanodes() {
- return this.translator.getProtoOrBuilder().getNumOfDecomActiveDatanodes();
- }
-
- @Override
- public void setNumOfDecomDeadDatanodes(int nodes) {
- this.translator.getBuilder().setNumOfDecomDeadDatanodes(nodes);
- }
-
- @Override
- public int getNumOfDecomDeadDatanodes() {
- return this.translator.getProtoOrBuilder().getNumOfDecomDeadDatanodes();
- }
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMembershipState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMembershipState.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMembershipState.java
new file mode 100644
index 0000000..dd349da
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMembershipState.java
@@ -0,0 +1,463 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMENODES;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMESERVICES;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.ROUTERS;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyException;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.clearRecords;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.createMockRegistrationForNamenode;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.synchronizeRecords;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateNamenodeRegistrationRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.util.Time;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test the basic {@link MembershipStore} membership functionality.
+ */
+public class TestStateStoreMembershipState extends TestStateStoreBase {
+
+ private static MembershipStore membershipStore;
+
+ @BeforeClass
+ public static void create() {
+ // Reduce expirations to 5 seconds
+ getConf().setLong(
+ RBFConfigKeys.FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS,
+ TimeUnit.SECONDS.toMillis(5));
+ }
+
+ @Before
+ public void setup() throws IOException, InterruptedException {
+
+ membershipStore =
+ getStateStore().getRegisteredRecordStore(MembershipStore.class);
+
+ // Clear NN registrations
+ assertTrue(clearRecords(getStateStore(), MembershipState.class));
+ }
+
+ @Test
+ public void testNamenodeStateOverride() throws Exception {
+ // Populate the state store
+ // 1) ns0:nn0 - Standby
+ String ns = "ns0";
+ String nn = "nn0";
+ MembershipState report = createRegistration(
+ ns, nn, ROUTERS[1], FederationNamenodeServiceState.STANDBY);
+ assertTrue(namenodeHeartbeat(report));
+
+ // Load data into cache and calculate quorum
+ assertTrue(getStateStore().loadCache(MembershipStore.class, true));
+
+ MembershipState existingState = getNamenodeRegistration(ns, nn);
+ assertEquals(
+ FederationNamenodeServiceState.STANDBY, existingState.getState());
+
+ // Override cache
+ UpdateNamenodeRegistrationRequest request =
+ UpdateNamenodeRegistrationRequest.newInstance(
+ ns, nn, FederationNamenodeServiceState.ACTIVE);
+ assertTrue(membershipStore.updateNamenodeRegistration(request).getResult());
+
+ MembershipState newState = getNamenodeRegistration(ns, nn);
+ assertEquals(FederationNamenodeServiceState.ACTIVE, newState.getState());
+ }
+
+ @Test
+ public void testStateStoreDisconnected() throws Exception {
+
+ // Close the data store driver
+ getStateStore().closeDriver();
+ assertFalse(getStateStore().isDriverReady());
+
+ NamenodeHeartbeatRequest hbRequest = NamenodeHeartbeatRequest.newInstance();
+ hbRequest.setNamenodeMembership(
+ createMockRegistrationForNamenode(
+ "test", "test", FederationNamenodeServiceState.UNAVAILABLE));
+ verifyException(membershipStore, "namenodeHeartbeat",
+ StateStoreUnavailableException.class,
+ new Class[] {NamenodeHeartbeatRequest.class},
+ new Object[] {hbRequest });
+
+ // Information from cache, no exception should be triggered for these
+ // TODO - should cached info expire at some point?
+ GetNamenodeRegistrationsRequest getRequest =
+ GetNamenodeRegistrationsRequest.newInstance();
+ verifyException(membershipStore,
+ "getNamenodeRegistrations", null,
+ new Class[] {GetNamenodeRegistrationsRequest.class},
+ new Object[] {getRequest});
+
+ verifyException(membershipStore,
+ "getExpiredNamenodeRegistrations", null,
+ new Class[] {GetNamenodeRegistrationsRequest.class},
+ new Object[] {getRequest});
+
+ UpdateNamenodeRegistrationRequest overrideRequest =
+ UpdateNamenodeRegistrationRequest.newInstance();
+ verifyException(membershipStore,
+ "updateNamenodeRegistration", null,
+ new Class[] {UpdateNamenodeRegistrationRequest.class},
+ new Object[] {overrideRequest});
+ }
+
+ private void registerAndLoadRegistrations(
+ List<MembershipState> registrationList) throws IOException {
+ // Populate
+ assertTrue(synchronizeRecords(
+ getStateStore(), registrationList, MembershipState.class));
+
+ // Load into cache
+ assertTrue(getStateStore().loadCache(MembershipStore.class, true));
+ }
+
+ private MembershipState createRegistration(String ns, String nn,
+ String router, FederationNamenodeServiceState state) throws IOException {
+ MembershipState record = MembershipState.newInstance(
+ router, ns,
+ nn, "testcluster", "testblock-" + ns, "testrpc-"+ ns + nn,
+ "testservice-"+ ns + nn, "testlifeline-"+ ns + nn,
+ "testweb-" + ns + nn, state, false);
+ return record;
+ }
+
+ @Test
+ public void testRegistrationMajorityQuorum()
+ throws InterruptedException, IOException {
+
+ // Populate the state store with a set of non-matching elements
+ // 1) ns0:nn0 - Standby (newest)
+ // 2) ns0:nn0 - Active (oldest)
+ // 3) ns0:nn0 - Active (2nd oldest)
+ // 4) ns0:nn0 - Active (3nd oldest element, newest active element)
+ // Verify the selected entry is the newest majority opinion (4)
+ String ns = "ns0";
+ String nn = "nn0";
+
+ // Active - oldest
+ MembershipState report = createRegistration(
+ ns, nn, ROUTERS[1], FederationNamenodeServiceState.ACTIVE);
+ assertTrue(namenodeHeartbeat(report));
+ Thread.sleep(1000);
+
+ // Active - 2nd oldest
+ report = createRegistration(
+ ns, nn, ROUTERS[2], FederationNamenodeServiceState.ACTIVE);
+ assertTrue(namenodeHeartbeat(report));
+ Thread.sleep(1000);
+
+ // Active - 3rd oldest, newest active element
+ report = createRegistration(
+ ns, nn, ROUTERS[3], FederationNamenodeServiceState.ACTIVE);
+ assertTrue(namenodeHeartbeat(report));
+
+ // standby - newest overall
+ report = createRegistration(
+ ns, nn, ROUTERS[0], FederationNamenodeServiceState.STANDBY);
+ assertTrue(namenodeHeartbeat(report));
+
+ // Load and calculate quorum
+ assertTrue(getStateStore().loadCache(MembershipStore.class, true));
+
+ // Verify quorum entry
+ MembershipState quorumEntry = getNamenodeRegistration(
+ report.getNameserviceId(), report.getNamenodeId());
+ assertNotNull(quorumEntry);
+ assertEquals(quorumEntry.getRouterId(), ROUTERS[3]);
+ }
+
+ @Test
+ public void testRegistrationQuorumExcludesExpired()
+ throws InterruptedException, IOException {
+
+ // Populate the state store with some expired entries and verify the expired
+ // entries are ignored.
+ // 1) ns0:nn0 - Active
+ // 2) ns0:nn0 - Expired
+ // 3) ns0:nn0 - Expired
+ // 4) ns0:nn0 - Expired
+ // Verify the selected entry is the active entry
+ List<MembershipState> registrationList = new ArrayList<>();
+ String ns = "ns0";
+ String nn = "nn0";
+ String rpcAddress = "testrpcaddress";
+ String serviceAddress = "testserviceaddress";
+ String lifelineAddress = "testlifelineaddress";
+ String blockPoolId = "testblockpool";
+ String clusterId = "testcluster";
+ String webAddress = "testwebaddress";
+ boolean safemode = false;
+
+ // Active
+ MembershipState record = MembershipState.newInstance(
+ ROUTERS[0], ns, nn, clusterId, blockPoolId,
+ rpcAddress, serviceAddress, lifelineAddress, webAddress,
+ FederationNamenodeServiceState.ACTIVE, safemode);
+ registrationList.add(record);
+
+ // Expired
+ record = MembershipState.newInstance(
+ ROUTERS[1], ns, nn, clusterId, blockPoolId,
+ rpcAddress, serviceAddress, lifelineAddress, webAddress,
+ FederationNamenodeServiceState.EXPIRED, safemode);
+ registrationList.add(record);
+
+ // Expired
+ record = MembershipState.newInstance(
+ ROUTERS[2], ns, nn, clusterId, blockPoolId,
+ rpcAddress, serviceAddress, lifelineAddress, webAddress,
+ FederationNamenodeServiceState.EXPIRED, safemode);
+ registrationList.add(record);
+
+ // Expired
+ record = MembershipState.newInstance(
+ ROUTERS[3], ns, nn, clusterId, blockPoolId,
+ rpcAddress, serviceAddress, lifelineAddress, webAddress,
+ FederationNamenodeServiceState.EXPIRED, safemode);
+ registrationList.add(record);
+ registerAndLoadRegistrations(registrationList);
+
+ // Verify quorum entry chooses active membership
+ MembershipState quorumEntry = getNamenodeRegistration(
+ record.getNameserviceId(), record.getNamenodeId());
+ assertNotNull(quorumEntry);
+ assertEquals(ROUTERS[0], quorumEntry.getRouterId());
+ }
+
+ @Test
+ public void testRegistrationQuorumAllExpired() throws IOException {
+
+ // 1) ns0:nn0 - Expired (oldest)
+ // 2) ns0:nn0 - Expired
+ // 3) ns0:nn0 - Expired
+ // 4) ns0:nn0 - Expired
+ // Verify no entry is either selected or cached
+ List<MembershipState> registrationList = new ArrayList<>();
+ String ns = NAMESERVICES[0];
+ String nn = NAMENODES[0];
+ String rpcAddress = "testrpcaddress";
+ String serviceAddress = "testserviceaddress";
+ String lifelineAddress = "testlifelineaddress";
+ String blockPoolId = "testblockpool";
+ String clusterId = "testcluster";
+ String webAddress = "testwebaddress";
+ boolean safemode = false;
+ long startingTime = Time.now();
+
+ // Expired
+ MembershipState record = MembershipState.newInstance(
+ ROUTERS[0], ns, nn, clusterId, blockPoolId,
+ rpcAddress, webAddress, lifelineAddress, webAddress,
+ FederationNamenodeServiceState.EXPIRED, safemode);
+ record.setDateModified(startingTime - 10000);
+ registrationList.add(record);
+
+ // Expired
+ record = MembershipState.newInstance(
+ ROUTERS[1], ns, nn, clusterId, blockPoolId,
+ rpcAddress, serviceAddress, lifelineAddress, webAddress,
+ FederationNamenodeServiceState.EXPIRED, safemode);
+ record.setDateModified(startingTime);
+ registrationList.add(record);
+
+ // Expired
+ record = MembershipState.newInstance(
+ ROUTERS[2], ns, nn, clusterId, blockPoolId,
+ rpcAddress, serviceAddress, lifelineAddress, webAddress,
+ FederationNamenodeServiceState.EXPIRED, safemode);
+ record.setDateModified(startingTime);
+ registrationList.add(record);
+
+ // Expired
+ record = MembershipState.newInstance(
+ ROUTERS[3], ns, nn, clusterId, blockPoolId,
+ rpcAddress, serviceAddress, lifelineAddress, webAddress,
+ FederationNamenodeServiceState.EXPIRED, safemode);
+ record.setDateModified(startingTime);
+ registrationList.add(record);
+
+ registerAndLoadRegistrations(registrationList);
+
+ // Verify no entry is found for this nameservice
+ assertNull(getNamenodeRegistration(
+ record.getNameserviceId(), record.getNamenodeId()));
+ }
+
+ @Test
+ public void testRegistrationNoQuorum()
+ throws InterruptedException, IOException {
+
+ // Populate the state store with a set of non-matching elements
+ // 1) ns0:nn0 - Standby (newest)
+ // 2) ns0:nn0 - Standby (oldest)
+ // 3) ns0:nn0 - Active (2nd oldest)
+ // 4) ns0:nn0 - Active (3nd oldest element, newest active element)
+ // Verify the selected entry is the newest entry (1)
+ MembershipState report1 = createRegistration(
+ NAMESERVICES[0], NAMENODES[0], ROUTERS[1],
+ FederationNamenodeServiceState.STANDBY);
+ assertTrue(namenodeHeartbeat(report1));
+ Thread.sleep(100);
+ MembershipState report2 = createRegistration(
+ NAMESERVICES[0], NAMENODES[0], ROUTERS[2],
+ FederationNamenodeServiceState.ACTIVE);
+ assertTrue(namenodeHeartbeat(report2));
+ Thread.sleep(100);
+ MembershipState report3 = createRegistration(
+ NAMESERVICES[0], NAMENODES[0], ROUTERS[3],
+ FederationNamenodeServiceState.ACTIVE);
+ assertTrue(namenodeHeartbeat(report3));
+ Thread.sleep(100);
+ MembershipState report4 = createRegistration(
+ NAMESERVICES[0], NAMENODES[0], ROUTERS[0],
+ FederationNamenodeServiceState.STANDBY);
+ assertTrue(namenodeHeartbeat(report4));
+
+ // Load and calculate quorum
+ assertTrue(getStateStore().loadCache(MembershipStore.class, true));
+
+ // Verify quorum entry uses the newest data, even though it is standby
+ MembershipState quorumEntry = getNamenodeRegistration(
+ report1.getNameserviceId(), report1.getNamenodeId());
+ assertNotNull(quorumEntry);
+ assertEquals(ROUTERS[0], quorumEntry.getRouterId());
+ assertEquals(
+ FederationNamenodeServiceState.STANDBY, quorumEntry.getState());
+ }
+
+ @Test
+ public void testRegistrationExpired()
+ throws InterruptedException, IOException {
+
+ // Populate the state store with a single NN element
+ // 1) ns0:nn0 - Active
+ // Wait for the entry to expire without heartbeating
+ // Verify the NN entry is populated as EXPIRED internally in the state store
+
+ MembershipState report = createRegistration(
+ NAMESERVICES[0], NAMENODES[0], ROUTERS[0],
+ FederationNamenodeServiceState.ACTIVE);
+ assertTrue(namenodeHeartbeat(report));
+
+ // Load cache
+ assertTrue(getStateStore().loadCache(MembershipStore.class, true));
+
+ // Verify quorum and entry
+ MembershipState quorumEntry = getNamenodeRegistration(
+ report.getNameserviceId(), report.getNamenodeId());
+ assertNotNull(quorumEntry);
+ assertEquals(ROUTERS[0], quorumEntry.getRouterId());
+ assertEquals(FederationNamenodeServiceState.ACTIVE, quorumEntry.getState());
+
+ // Wait past expiration (set in conf to 5 seconds)
+ Thread.sleep(6000);
+ // Reload cache
+ assertTrue(getStateStore().loadCache(MembershipStore.class, true));
+
+ // Verify entry is now expired and is no longer in the cache
+ quorumEntry = getNamenodeRegistration(NAMESERVICES[0], NAMENODES[0]);
+ assertNull(quorumEntry);
+
+ // Verify entry is now expired and can't be used by RPC service
+ quorumEntry = getNamenodeRegistration(
+ report.getNameserviceId(), report.getNamenodeId());
+ assertNull(quorumEntry);
+
+ // Heartbeat again, updates dateModified
+ assertTrue(namenodeHeartbeat(report));
+ // Reload cache
+ assertTrue(getStateStore().loadCache(MembershipStore.class, true));
+
+ // Verify updated entry marked as active and is accessible to RPC server
+ quorumEntry = getNamenodeRegistration(
+ report.getNameserviceId(), report.getNamenodeId());
+ assertNotNull(quorumEntry);
+ assertEquals(ROUTERS[0], quorumEntry.getRouterId());
+ assertEquals(FederationNamenodeServiceState.ACTIVE, quorumEntry.getState());
+ }
+
+ /**
+ * Get a single namenode membership record from the store.
+ *
+ * @param nsId The HDFS nameservice ID to search for
+ * @param nnId The HDFS namenode ID to search for
+ * @return The single NamenodeMembershipRecord that matches the query or null
+ * if not found.
+ * @throws IOException if the query could not be executed.
+ */
+ private MembershipState getNamenodeRegistration(
+ final String nsId, final String nnId) throws IOException {
+
+ MembershipState partial = MembershipState.newInstance();
+ partial.setNameserviceId(nsId);
+ partial.setNamenodeId(nnId);
+ GetNamenodeRegistrationsRequest request =
+ GetNamenodeRegistrationsRequest.newInstance(partial);
+ GetNamenodeRegistrationsResponse response =
+ membershipStore.getNamenodeRegistrations(request);
+
+ List<MembershipState> results = response.getNamenodeMemberships();
+ if (results != null && results.size() == 1) {
+ MembershipState record = results.get(0);
+ return record;
+ }
+ return null;
+ }
+
+ /**
+ * Register a namenode heartbeat with the state store.
+ *
+ * @param store FederationMembershipStateStore instance to retrieve the
+ * membership data records.
+ * @param namenode A fully populated namenode membership record to be
+ * committed to the data store.
+ * @return True if successful, false otherwise.
+ * @throws IOException if the state store query could not be performed.
+ */
+ private boolean namenodeHeartbeat(MembershipState namenode)
+ throws IOException {
+
+ NamenodeHeartbeatRequest request =
+ NamenodeHeartbeatRequest.newInstance(namenode);
+ NamenodeHeartbeatResponse response =
+ membershipStore.namenodeHeartbeat(request);
+ return response.getResult();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMountTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMountTable.java
new file mode 100644
index 0000000..d30d6ba
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMountTable.java
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMESERVICES;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyException;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.clearRecords;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.createMockMountTable;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.synchronizeRecords;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test the basic {@link StateStoreService}
+ * {@link MountTableStore} functionality.
+ */
+public class TestStateStoreMountTable extends TestStateStoreBase {
+
+ private static List<String> nameservices;
+ private static MountTableStore mountStore;
+
+ @BeforeClass
+ public static void create() throws IOException {
+ nameservices = new ArrayList<>();
+ nameservices.add(NAMESERVICES[0]);
+ nameservices.add(NAMESERVICES[1]);
+ }
+
+ @Before
+ public void setup() throws IOException, InterruptedException {
+ mountStore =
+ getStateStore().getRegisteredRecordStore(MountTableStore.class);
+ // Clear Mount table registrations
+ assertTrue(clearRecords(getStateStore(), MountTable.class));
+ }
+
+ @Test
+ public void testStateStoreDisconnected() throws Exception {
+
+ // Close the data store driver
+ getStateStore().closeDriver();
+ assertFalse(getStateStore().isDriverReady());
+
+ // Test APIs that access the store to check they throw the correct exception
+ AddMountTableEntryRequest addRequest =
+ AddMountTableEntryRequest.newInstance();
+ verifyException(mountStore, "addMountTableEntry",
+ StateStoreUnavailableException.class,
+ new Class[] {AddMountTableEntryRequest.class},
+ new Object[] {addRequest});
+
+ UpdateMountTableEntryRequest updateRequest =
+ UpdateMountTableEntryRequest.newInstance();
+ verifyException(mountStore, "updateMountTableEntry",
+ StateStoreUnavailableException.class,
+ new Class[] {UpdateMountTableEntryRequest.class},
+ new Object[] {updateRequest});
+
+ RemoveMountTableEntryRequest removeRequest =
+ RemoveMountTableEntryRequest.newInstance();
+ verifyException(mountStore, "removeMountTableEntry",
+ StateStoreUnavailableException.class,
+ new Class[] {RemoveMountTableEntryRequest.class},
+ new Object[] {removeRequest});
+
+ GetMountTableEntriesRequest getRequest =
+ GetMountTableEntriesRequest.newInstance();
+ mountStore.loadCache(true);
+ verifyException(mountStore, "getMountTableEntries",
+ StateStoreUnavailableException.class,
+ new Class[] {GetMountTableEntriesRequest.class},
+ new Object[] {getRequest});
+ }
+
+ @Test
+ public void testSynchronizeMountTable() throws IOException {
+ // Synchronize and get mount table entries
+ List<MountTable> entries = createMockMountTable(nameservices);
+ assertTrue(synchronizeRecords(getStateStore(), entries, MountTable.class));
+ for (MountTable e : entries) {
+ mountStore.loadCache(true);
+ MountTable entry = getMountTableEntry(e.getSourcePath());
+ assertNotNull(entry);
+ assertEquals(e.getDefaultLocation().getDest(),
+ entry.getDefaultLocation().getDest());
+ }
+ }
+
+ @Test
+ public void testAddMountTableEntry() throws IOException {
+
+ // Add 1
+ List<MountTable> entries = createMockMountTable(nameservices);
+ List<MountTable> entries1 = getMountTableEntries("/").getRecords();
+ assertEquals(0, entries1.size());
+ MountTable entry0 = entries.get(0);
+ AddMountTableEntryRequest request =
+ AddMountTableEntryRequest.newInstance(entry0);
+ AddMountTableEntryResponse response =
+ mountStore.addMountTableEntry(request);
+ assertTrue(response.getStatus());
+
+ mountStore.loadCache(true);
+ List<MountTable> entries2 = getMountTableEntries("/").getRecords();
+ assertEquals(1, entries2.size());
+ }
+
+ @Test
+ public void testRemoveMountTableEntry() throws IOException {
+
+ // Add many
+ List<MountTable> entries = createMockMountTable(nameservices);
+ synchronizeRecords(getStateStore(), entries, MountTable.class);
+ mountStore.loadCache(true);
+ List<MountTable> entries1 = getMountTableEntries("/").getRecords();
+ assertEquals(entries.size(), entries1.size());
+
+ // Remove 1
+ RemoveMountTableEntryRequest request =
+ RemoveMountTableEntryRequest.newInstance();
+ request.setSrcPath(entries.get(0).getSourcePath());
+ assertTrue(mountStore.removeMountTableEntry(request).getStatus());
+
+ // Verify remove
+ mountStore.loadCache(true);
+ List<MountTable> entries2 = getMountTableEntries("/").getRecords();
+ assertEquals(entries.size() - 1, entries2.size());
+ }
+
+ @Test
+ public void testUpdateMountTableEntry() throws IOException {
+
+ // Add 1
+ List<MountTable> entries = createMockMountTable(nameservices);
+ MountTable entry0 = entries.get(0);
+ String srcPath = entry0.getSourcePath();
+ String nsId = entry0.getDefaultLocation().getNameserviceId();
+ AddMountTableEntryRequest request =
+ AddMountTableEntryRequest.newInstance(entry0);
+ AddMountTableEntryResponse response =
+ mountStore.addMountTableEntry(request);
+ assertTrue(response.getStatus());
+
+ // Verify
+ mountStore.loadCache(true);
+ MountTable matchingEntry0 = getMountTableEntry(srcPath);
+ assertNotNull(matchingEntry0);
+ assertEquals(nsId, matchingEntry0.getDefaultLocation().getNameserviceId());
+
+ // Edit destination nameservice for source path
+ Map<String, String> destMap =
+ Collections.singletonMap("testnameservice", "/");
+ MountTable replacement =
+ MountTable.newInstance(srcPath, destMap);
+ UpdateMountTableEntryRequest updateRequest =
+ UpdateMountTableEntryRequest.newInstance(replacement);
+ UpdateMountTableEntryResponse updateResponse =
+ mountStore.updateMountTableEntry(updateRequest);
+ assertTrue(updateResponse.getStatus());
+
+ // Verify
+ mountStore.loadCache(true);
+ MountTable matchingEntry1 = getMountTableEntry(srcPath);
+ assertNotNull(matchingEntry1);
+ assertEquals("testnameservice",
+ matchingEntry1.getDefaultLocation().getNameserviceId());
+ }
+
+ /**
+ * Gets an existing mount table record in the state store.
+ *
+ * @param mount The mount point of the record to remove.
+ * @return The matching record if found, null if it is not found.
+ * @throws IOException If the state store could not be accessed.
+ */
+ private MountTable getMountTableEntry(String mount) throws IOException {
+ GetMountTableEntriesRequest request =
+ GetMountTableEntriesRequest.newInstance(mount);
+ GetMountTableEntriesResponse response =
+ mountStore.getMountTableEntries(request);
+ List<MountTable> results = response.getEntries();
+ if (results.size() > 0) {
+ // First result is sorted to have the shortest mount string length
+ return results.get(0);
+ }
+ return null;
+ }
+
+ /**
+ * Fetch all mount table records beneath a root path.
+ *
+ * @param store FederationMountTableStore instance to commit the data.
+ * @param mount The root search path, enter "/" to return all mount table
+ * records.
+ *
+ * @return A list of all mount table records found below the root mount.
+ *
+ * @throws IOException If the state store could not be accessed.
+ */
+ private QueryResult<MountTable> getMountTableEntries(String mount)
+ throws IOException {
+ if (mount == null) {
+ throw new IOException("Please specify a root search path");
+ }
+ GetMountTableEntriesRequest request =
+ GetMountTableEntriesRequest.newInstance();
+ request.setSrcPath(mount);
+ GetMountTableEntriesResponse response =
+ mountStore.getMountTableEntries(request);
+ List<MountTable> records = response.getEntries();
+ long timestamp = response.getTimestamp();
+ return new QueryResult<MountTable>(records, timestamp);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreRouterState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreRouterState.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreRouterState.java
new file mode 100644
index 0000000..db1df19
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreRouterState.java
@@ -0,0 +1,195 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyException;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.clearRecords;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.hdfs.server.federation.router.FederationUtil;
+import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.router.RouterServiceState;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationsRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+import org.apache.hadoop.util.Time;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test the basic {@link StateStoreService} {@link RouterStore} functionality.
+ */
+public class TestStateStoreRouterState extends TestStateStoreBase {
+
+ private static RouterStore routerStore;
+
+ @BeforeClass
+ public static void create() {
+ // Reduce expirations to 5 seconds
+ getConf().setTimeDuration(
+ RBFConfigKeys.FEDERATION_STORE_ROUTER_EXPIRATION_MS,
+ 5, TimeUnit.SECONDS);
+ }
+
+ @Before
+ public void setup() throws IOException, InterruptedException {
+
+ if (routerStore == null) {
+ routerStore =
+ getStateStore().getRegisteredRecordStore(RouterStore.class);
+ }
+
+ // Clear router status registrations
+ assertTrue(clearRecords(getStateStore(), RouterState.class));
+ }
+
+ @Test
+ public void testStateStoreDisconnected() throws Exception {
+
+ // Close the data store driver
+ getStateStore().closeDriver();
+ assertEquals(false, getStateStore().isDriverReady());
+
+ // Test all APIs that access the data store to ensure they throw the correct
+ // exception.
+ GetRouterRegistrationRequest getSingleRequest =
+ GetRouterRegistrationRequest.newInstance();
+ verifyException(routerStore, "getRouterRegistration",
+ StateStoreUnavailableException.class,
+ new Class[] {GetRouterRegistrationRequest.class},
+ new Object[] {getSingleRequest});
+
+ GetRouterRegistrationsRequest getRequest =
+ GetRouterRegistrationsRequest.newInstance();
+ routerStore.loadCache(true);
+ verifyException(routerStore, "getRouterRegistrations",
+ StateStoreUnavailableException.class,
+ new Class[] {GetRouterRegistrationsRequest.class},
+ new Object[] {getRequest});
+
+ RouterHeartbeatRequest hbRequest = RouterHeartbeatRequest.newInstance(
+ RouterState.newInstance("test", 0, RouterServiceState.UNINITIALIZED));
+ verifyException(routerStore, "routerHeartbeat",
+ StateStoreUnavailableException.class,
+ new Class[] {RouterHeartbeatRequest.class},
+ new Object[] {hbRequest});
+ }
+
+ //
+ // Router
+ //
+ @Test
+ public void testUpdateRouterStatus()
+ throws IllegalStateException, IOException {
+
+ long dateStarted = Time.now();
+ String address = "testaddress";
+
+ // Set
+ RouterHeartbeatRequest request = RouterHeartbeatRequest.newInstance(
+ RouterState.newInstance(
+ address, dateStarted, RouterServiceState.RUNNING));
+ assertTrue(routerStore.routerHeartbeat(request).getStatus());
+
+ // Verify
+ GetRouterRegistrationRequest getRequest =
+ GetRouterRegistrationRequest.newInstance(address);
+ RouterState record =
+ routerStore.getRouterRegistration(getRequest).getRouter();
+ assertNotNull(record);
+ assertEquals(RouterServiceState.RUNNING, record.getStatus());
+ assertEquals(address, record.getAddress());
+ assertEquals(FederationUtil.getCompileInfo(), record.getCompileInfo());
+ // Build version may vary a bit
+ assertFalse(record.getVersion().isEmpty());
+ }
+
+ @Test
+ public void testRouterStateExpired()
+ throws IOException, InterruptedException {
+
+ long dateStarted = Time.now();
+ String address = "testaddress";
+
+ RouterHeartbeatRequest request = RouterHeartbeatRequest.newInstance(
+ RouterState.newInstance(
+ address, dateStarted, RouterServiceState.RUNNING));
+ // Set
+ assertTrue(routerStore.routerHeartbeat(request).getStatus());
+
+ // Verify
+ GetRouterRegistrationRequest getRequest =
+ GetRouterRegistrationRequest.newInstance(address);
+ RouterState record =
+ routerStore.getRouterRegistration(getRequest).getRouter();
+ assertNotNull(record);
+
+ // Wait past expiration (set to 5 sec in config)
+ Thread.sleep(6000);
+
+ // Verify expired
+ RouterState r = routerStore.getRouterRegistration(getRequest).getRouter();
+ assertEquals(RouterServiceState.EXPIRED, r.getStatus());
+
+ // Heartbeat again and this shouldn't be EXPIRED anymore
+ assertTrue(routerStore.routerHeartbeat(request).getStatus());
+ r = routerStore.getRouterRegistration(getRequest).getRouter();
+ assertEquals(RouterServiceState.RUNNING, r.getStatus());
+ }
+
+ @Test
+ public void testGetAllRouterStates()
+ throws StateStoreUnavailableException, IOException {
+
+ // Set 2 entries
+ RouterHeartbeatRequest heartbeatRequest1 =
+ RouterHeartbeatRequest.newInstance(
+ RouterState.newInstance(
+ "testaddress1", Time.now(), RouterServiceState.RUNNING));
+ assertTrue(routerStore.routerHeartbeat(heartbeatRequest1).getStatus());
+
+ RouterHeartbeatRequest heartbeatRequest2 =
+ RouterHeartbeatRequest.newInstance(
+ RouterState.newInstance(
+ "testaddress2", Time.now(), RouterServiceState.RUNNING));
+ assertTrue(routerStore.routerHeartbeat(heartbeatRequest2).getStatus());
+
+ // Verify
+ routerStore.loadCache(true);
+ GetRouterRegistrationsRequest request =
+ GetRouterRegistrationsRequest.newInstance();
+ List<RouterState> entries =
+ routerStore.getRouterRegistrations(request).getRouters();
+ assertEquals(2, entries.size());
+ Collections.sort(entries);
+ assertEquals("testaddress1", entries.get(0).getAddress());
+ assertEquals("testaddress2", entries.get(1).getAddress());
+ assertEquals(RouterServiceState.RUNNING, entries.get(0).getStatus());
+ assertEquals(RouterServiceState.RUNNING, entries.get(1).getStatus());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
new file mode 100644
index 0000000..fd29e37
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
@@ -0,0 +1,613 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.metrics.StateStoreMetrics;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.router.RouterServiceState;
+import org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.hdfs.server.federation.store.records.Query;
+import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+import org.apache.hadoop.hdfs.server.federation.store.records.StateStoreVersion;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Base tests for the driver. The particular implementations will use this to
+ * test their functionality.
+ */
+public class TestStateStoreDriverBase {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestStateStoreDriverBase.class);
+
+ private static StateStoreService stateStore;
+ private static Configuration conf;
+
+ private static final Random RANDOM = new Random();
+
+
+ /**
+ * Get the State Store driver.
+ * @return State Store driver.
+ */
+ protected StateStoreDriver getStateStoreDriver() {
+ return stateStore.getDriver();
+ }
+
+ @After
+ public void cleanMetrics() {
+ if (stateStore != null) {
+ StateStoreMetrics metrics = stateStore.getMetrics();
+ metrics.reset();
+ }
+ }
+
+ @AfterClass
+ public static void tearDownCluster() {
+ if (stateStore != null) {
+ stateStore.stop();
+ }
+ }
+
+ /**
+ * Get a new State Store using this configuration.
+ *
+ * @param config Configuration for the State Store.
+ * @throws Exception If we cannot get the State Store.
+ */
+ public static void getStateStore(Configuration config) throws Exception {
+ conf = config;
+ stateStore = FederationStateStoreTestUtils.newStateStore(conf);
+ }
+
+ private String generateRandomString() {
+ String randomString = "randomString-" + RANDOM.nextInt();
+ return randomString;
+ }
+
+ private long generateRandomLong() {
+ return RANDOM.nextLong();
+ }
+
+ @SuppressWarnings("rawtypes")
+ private <T extends Enum> T generateRandomEnum(Class<T> enumClass) {
+ int x = RANDOM.nextInt(enumClass.getEnumConstants().length);
+ T data = enumClass.getEnumConstants()[x];
+ return data;
+ }
+
+ @SuppressWarnings("unchecked")
+ private <T extends BaseRecord> T generateFakeRecord(Class<T> recordClass)
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+
+ if (recordClass == MembershipState.class) {
+ return (T) MembershipState.newInstance(generateRandomString(),
+ generateRandomString(), generateRandomString(),
+ generateRandomString(), generateRandomString(),
+ generateRandomString(), generateRandomString(),
+ generateRandomString(), generateRandomString(),
+ generateRandomEnum(FederationNamenodeServiceState.class), false);
+ } else if (recordClass == MountTable.class) {
+ String src = "/" + generateRandomString();
+ Map<String, String> destMap = Collections.singletonMap(
+ generateRandomString(), "/" + generateRandomString());
+ return (T) MountTable.newInstance(src, destMap);
+ } else if (recordClass == RouterState.class) {
+ RouterState routerState = RouterState.newInstance(generateRandomString(),
+ generateRandomLong(), generateRandomEnum(RouterServiceState.class));
+ StateStoreVersion version = generateFakeRecord(StateStoreVersion.class);
+ routerState.setStateStoreVersion(version);
+ return (T) routerState;
+ }
+
+ return null;
+ }
+
+ /**
+ * Validate if a record is the same.
+ *
+ * @param original Original record.
+ * @param committed Committed record.
+ * @param assertEquals Assert if the records are equal or just return.
+ * @return If the record is successfully validated.
+ */
+ private boolean validateRecord(
+ BaseRecord original, BaseRecord committed, boolean assertEquals) {
+
+ boolean ret = true;
+
+ Map<String, Class<?>> fields = getFields(original);
+ for (String key : fields.keySet()) {
+ if (key.equals("dateModified") ||
+ key.equals("dateCreated") ||
+ key.equals("proto")) {
+ // Fields are updated/set on commit and fetch and may not match
+ // the fields that are initialized in a non-committed object.
+ continue;
+ }
+ Object data1 = getField(original, key);
+ Object data2 = getField(committed, key);
+ if (assertEquals) {
+ assertEquals("Field " + key + " does not match", data1, data2);
+ } else if (!data1.equals(data2)) {
+ ret = false;
+ }
+ }
+
+ long now = stateStore.getDriver().getTime();
+ assertTrue(
+ committed.getDateCreated() <= now && committed.getDateCreated() > 0);
+ assertTrue(committed.getDateModified() >= committed.getDateCreated());
+
+ return ret;
+ }
+
+ public static void removeAll(StateStoreDriver driver) throws IOException {
+ driver.removeAll(MembershipState.class);
+ driver.removeAll(MountTable.class);
+ }
+
+ public <T extends BaseRecord> void testInsert(
+ StateStoreDriver driver, Class<T> recordClass)
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+
+ assertTrue(driver.removeAll(recordClass));
+ QueryResult<T> queryResult0 = driver.get(recordClass);
+ List<T> records0 = queryResult0.getRecords();
+ assertTrue(records0.isEmpty());
+
+ // Insert single
+ BaseRecord record = generateFakeRecord(recordClass);
+ driver.put(record, true, false);
+
+ // Verify
+ QueryResult<T> queryResult1 = driver.get(recordClass);
+ List<T> records1 = queryResult1.getRecords();
+ assertEquals(1, records1.size());
+ T record0 = records1.get(0);
+ validateRecord(record, record0, true);
+
+ // Insert multiple
+ List<T> insertList = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ T newRecord = generateFakeRecord(recordClass);
+ insertList.add(newRecord);
+ }
+ driver.putAll(insertList, true, false);
+
+ // Verify
+ QueryResult<T> queryResult2 = driver.get(recordClass);
+ List<T> records2 = queryResult2.getRecords();
+ assertEquals(11, records2.size());
+ }
+
+ public <T extends BaseRecord> void testFetchErrors(StateStoreDriver driver,
+ Class<T> clazz) throws IllegalAccessException, IOException {
+
+ // Fetch empty list
+ driver.removeAll(clazz);
+ QueryResult<T> result0 = driver.get(clazz);
+ assertNotNull(result0);
+ List<T> records0 = result0.getRecords();
+ assertEquals(records0.size(), 0);
+
+ // Insert single
+ BaseRecord record = generateFakeRecord(clazz);
+ assertTrue(driver.put(record, true, false));
+
+ // Verify
+ QueryResult<T> result1 = driver.get(clazz);
+ List<T> records1 = result1.getRecords();
+ assertEquals(1, records1.size());
+ validateRecord(record, records1.get(0), true);
+
+ // Test fetch single object with a bad query
+ final T fakeRecord = generateFakeRecord(clazz);
+ final Query<T> query = new Query<T>(fakeRecord);
+ T getRecord = driver.get(clazz, query);
+ assertNull(getRecord);
+
+ // Test fetch multiple objects does not exist returns empty list
+ assertEquals(driver.getMultiple(clazz, query).size(), 0);
+ }
+
+ public <T extends BaseRecord> void testPut(
+ StateStoreDriver driver, Class<T> clazz)
+ throws IllegalArgumentException, ReflectiveOperationException,
+ IOException, SecurityException {
+
+ driver.removeAll(clazz);
+ QueryResult<T> records = driver.get(clazz);
+ assertTrue(records.getRecords().isEmpty());
+
+ // Insert multiple
+ List<T> insertList = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ T newRecord = generateFakeRecord(clazz);
+ insertList.add(newRecord);
+ }
+
+ // Verify
+ assertTrue(driver.putAll(insertList, false, true));
+ records = driver.get(clazz);
+ assertEquals(records.getRecords().size(), 10);
+
+ // Generate a new record with the same PK fields as an existing record
+ BaseRecord updatedRecord = generateFakeRecord(clazz);
+ BaseRecord existingRecord = records.getRecords().get(0);
+ Map<String, String> primaryKeys = existingRecord.getPrimaryKeys();
+ for (Entry<String, String> entry : primaryKeys.entrySet()) {
+ String key = entry.getKey();
+ String value = entry.getValue();
+ Class<?> fieldType = getFieldType(existingRecord, key);
+ Object field = fromString(value, fieldType);
+ assertTrue(setField(updatedRecord, key, field));
+ }
+
+ // Attempt an update of an existing entry, but it is not allowed.
+ assertFalse(driver.put(updatedRecord, false, true));
+
+ // Verify no update occurred, all original records are unchanged
+ QueryResult<T> newRecords = driver.get(clazz);
+ assertTrue(newRecords.getRecords().size() == 10);
+ assertEquals("A single entry was improperly updated in the store", 10,
+ countMatchingEntries(records.getRecords(), newRecords.getRecords()));
+
+ // Update the entry (allowing updates)
+ assertTrue(driver.put(updatedRecord, true, false));
+
+ // Verify that one entry no longer matches the original set
+ newRecords = driver.get(clazz);
+ assertEquals(10, newRecords.getRecords().size());
+ assertEquals(
+ "Record of type " + clazz + " not updated in the store", 9,
+ countMatchingEntries(records.getRecords(), newRecords.getRecords()));
+ }
+
+ private int countMatchingEntries(
+ Collection<? extends BaseRecord> committedList,
+ Collection<? extends BaseRecord> matchList) {
+
+ int matchingCount = 0;
+ for (BaseRecord committed : committedList) {
+ for (BaseRecord match : matchList) {
+ try {
+ if (match.getPrimaryKey().equals(committed.getPrimaryKey())) {
+ if (validateRecord(match, committed, false)) {
+ matchingCount++;
+ }
+ break;
+ }
+ } catch (Exception ex) {
+ }
+ }
+ }
+ return matchingCount;
+ }
+
+ public <T extends BaseRecord> void testRemove(
+ StateStoreDriver driver, Class<T> clazz)
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+
+ // Remove all
+ assertTrue(driver.removeAll(clazz));
+ QueryResult<T> records = driver.get(clazz);
+ assertTrue(records.getRecords().isEmpty());
+
+ // Insert multiple
+ List<T> insertList = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ T newRecord = generateFakeRecord(clazz);
+ insertList.add(newRecord);
+ }
+
+ // Verify
+ assertTrue(driver.putAll(insertList, false, true));
+ records = driver.get(clazz);
+ assertEquals(records.getRecords().size(), 10);
+
+ // Remove Single
+ assertTrue(driver.remove(records.getRecords().get(0)));
+
+ // Verify
+ records = driver.get(clazz);
+ assertEquals(records.getRecords().size(), 9);
+
+ // Remove with filter
+ final T firstRecord = records.getRecords().get(0);
+ final Query<T> query0 = new Query<T>(firstRecord);
+ assertTrue(driver.remove(clazz, query0) > 0);
+
+ final T secondRecord = records.getRecords().get(1);
+ final Query<T> query1 = new Query<T>(secondRecord);
+ assertTrue(driver.remove(clazz, query1) > 0);
+
+ // Verify
+ records = driver.get(clazz);
+ assertEquals(records.getRecords().size(), 7);
+
+ // Remove all
+ assertTrue(driver.removeAll(clazz));
+
+ // Verify
+ records = driver.get(clazz);
+ assertTrue(records.getRecords().isEmpty());
+ }
+
+ public void testInsert(StateStoreDriver driver)
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testInsert(driver, MembershipState.class);
+ testInsert(driver, MountTable.class);
+ }
+
+ public void testPut(StateStoreDriver driver)
+ throws IllegalArgumentException, ReflectiveOperationException,
+ IOException, SecurityException {
+ testPut(driver, MembershipState.class);
+ testPut(driver, MountTable.class);
+ }
+
+ public void testRemove(StateStoreDriver driver)
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testRemove(driver, MembershipState.class);
+ testRemove(driver, MountTable.class);
+ }
+
+ public void testFetchErrors(StateStoreDriver driver)
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testFetchErrors(driver, MembershipState.class);
+ testFetchErrors(driver, MountTable.class);
+ }
+
+ public void testMetrics(StateStoreDriver driver)
+ throws IOException, IllegalArgumentException, IllegalAccessException {
+
+ MountTable insertRecord =
+ this.generateFakeRecord(MountTable.class);
+
+ // Put single
+ StateStoreMetrics metrics = stateStore.getMetrics();
+ assertEquals(0, metrics.getWriteOps());
+ driver.put(insertRecord, true, false);
+ assertEquals(1, metrics.getWriteOps());
+
+ // Put multiple
+ metrics.reset();
+ assertEquals(0, metrics.getWriteOps());
+ driver.put(insertRecord, true, false);
+ assertEquals(1, metrics.getWriteOps());
+
+ // Get Single
+ metrics.reset();
+ assertEquals(0, metrics.getReadOps());
+
+ final String querySourcePath = insertRecord.getSourcePath();
+ MountTable partial = MountTable.newInstance();
+ partial.setSourcePath(querySourcePath);
+ final Query<MountTable> query = new Query<>(partial);
+ driver.get(MountTable.class, query);
+ assertEquals(1, metrics.getReadOps());
+
+ // GetAll
+ metrics.reset();
+ assertEquals(0, metrics.getReadOps());
+ driver.get(MountTable.class);
+ assertEquals(1, metrics.getReadOps());
+
+ // GetMultiple
+ metrics.reset();
+ assertEquals(0, metrics.getReadOps());
+ driver.getMultiple(MountTable.class, query);
+ assertEquals(1, metrics.getReadOps());
+
+ // Insert fails
+ metrics.reset();
+ assertEquals(0, metrics.getFailureOps());
+ driver.put(insertRecord, false, true);
+ assertEquals(1, metrics.getFailureOps());
+
+ // Remove single
+ metrics.reset();
+ assertEquals(0, metrics.getRemoveOps());
+ driver.remove(insertRecord);
+ assertEquals(1, metrics.getRemoveOps());
+
+ // Remove multiple
+ metrics.reset();
+ driver.put(insertRecord, true, false);
+ assertEquals(0, metrics.getRemoveOps());
+ driver.remove(MountTable.class, query);
+ assertEquals(1, metrics.getRemoveOps());
+
+ // Remove all
+ metrics.reset();
+ driver.put(insertRecord, true, false);
+ assertEquals(0, metrics.getRemoveOps());
+ driver.removeAll(MountTable.class);
+ assertEquals(1, metrics.getRemoveOps());
+ }
+
+ /**
+ * Sets the value of a field on the object.
+ *
+ * @param fieldName The string name of the field.
+ * @param data The data to pass to the field's setter.
+ *
+ * @return True if successful, fails if failed.
+ */
+ private static boolean setField(
+ BaseRecord record, String fieldName, Object data) {
+
+ Method m = locateSetter(record, fieldName);
+ if (m != null) {
+ try {
+ m.invoke(record, data);
+ } catch (Exception e) {
+ LOG.error("Cannot set field " + fieldName + " on object "
+ + record.getClass().getName() + " to data " + data + " of type "
+ + data.getClass(), e);
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Finds the appropriate setter for a field name.
+ *
+ * @param fieldName The legacy name of the field.
+ * @return The matching setter or null if not found.
+ */
+ private static Method locateSetter(BaseRecord record, String fieldName) {
+ for (Method m : record.getClass().getMethods()) {
+ if (m.getName().equalsIgnoreCase("set" + fieldName)) {
+ return m;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Returns all serializable fields in the object.
+ *
+ * @return Map with the fields.
+ */
+ private static Map<String, Class<?>> getFields(BaseRecord record) {
+ Map<String, Class<?>> getters = new HashMap<>();
+ for (Method m : record.getClass().getDeclaredMethods()) {
+ if (m.getName().startsWith("get")) {
+ try {
+ Class<?> type = m.getReturnType();
+ char[] c = m.getName().substring(3).toCharArray();
+ c[0] = Character.toLowerCase(c[0]);
+ String key = new String(c);
+ getters.put(key, type);
+ } catch (Exception e) {
+ LOG.error("Cannot execute getter " + m.getName()
+ + " on object " + record);
+ }
+ }
+ }
+ return getters;
+ }
+
+ /**
+ * Get the type of a field.
+ *
+ * @param fieldName
+ * @return Field type
+ */
+ private static Class<?> getFieldType(BaseRecord record, String fieldName) {
+ Method m = locateGetter(record, fieldName);
+ return m.getReturnType();
+ }
+
+ /**
+ * Fetches the value for a field name.
+ *
+ * @param fieldName the legacy name of the field.
+ * @return The field data or null if not found.
+ */
+ private static Object getField(BaseRecord record, String fieldName) {
+ Object result = null;
+ Method m = locateGetter(record, fieldName);
+ if (m != null) {
+ try {
+ result = m.invoke(record);
+ } catch (Exception e) {
+ LOG.error("Cannot get field " + fieldName + " on object " + record);
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Finds the appropriate getter for a field name.
+ *
+ * @param fieldName The legacy name of the field.
+ * @return The matching getter or null if not found.
+ */
+ private static Method locateGetter(BaseRecord record, String fieldName) {
+ for (Method m : record.getClass().getMethods()) {
+ if (m.getName().equalsIgnoreCase("get" + fieldName)) {
+ return m;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Expands a data object from the store into an record object. Default store
+ * data type is a String. Override if additional serialization is required.
+ *
+ * @param data Object containing the serialized data. Only string is
+ * supported.
+ * @param clazz Target object class to hold the deserialized data.
+ * @return An instance of the target data object initialized with the
+ * deserialized data.
+ */
+ @Deprecated
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ private static <T> T fromString(String data, Class<T> clazz) {
+
+ if (data.equals("null")) {
+ return null;
+ } else if (clazz == String.class) {
+ return (T) data;
+ } else if (clazz == Long.class || clazz == long.class) {
+ return (T) Long.valueOf(data);
+ } else if (clazz == Integer.class || clazz == int.class) {
+ return (T) Integer.valueOf(data);
+ } else if (clazz == Double.class || clazz == double.class) {
+ return (T) Double.valueOf(data);
+ } else if (clazz == Float.class || clazz == float.class) {
+ return (T) Float.valueOf(data);
+ } else if (clazz == Boolean.class || clazz == boolean.class) {
+ return (T) Boolean.valueOf(data);
+ } else if (clazz.isEnum()) {
+ return (T) Enum.valueOf((Class<Enum>) clazz, data);
+ }
+ return null;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFile.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFile.java
new file mode 100644
index 0000000..a8a9020
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFile.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver;
+
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.getStateStoreConfiguration;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test the FileSystem (e.g., HDFS) implementation of the State Store driver.
+ */
+public class TestStateStoreFile extends TestStateStoreDriverBase {
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ Configuration conf = getStateStoreConfiguration(StateStoreFileImpl.class);
+ getStateStore(conf);
+ }
+
+ @Before
+ public void startup() throws IOException {
+ removeAll(getStateStoreDriver());
+ }
+
+ @Test
+ public void testInsert()
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testInsert(getStateStoreDriver());
+ }
+
+ @Test
+ public void testUpdate()
+ throws IllegalArgumentException, ReflectiveOperationException,
+ IOException, SecurityException {
+ testPut(getStateStoreDriver());
+ }
+
+ @Test
+ public void testDelete()
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testRemove(getStateStoreDriver());
+ }
+
+ @Test
+ public void testFetchErrors()
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testFetchErrors(getStateStoreDriver());
+ }
+
+ @Test
+ public void testMetrics()
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testMetrics(getStateStoreDriver());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileBase.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileBase.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileBase.java
new file mode 100644
index 0000000..9adfe33
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileBase.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver;
+
+import static org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileBaseImpl.isOldTempRecord;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.util.Time;
+import org.junit.Test;
+
+/**
+ * Tests for the State Store file based implementation.
+ */
+public class TestStateStoreFileBase {
+
+ @Test
+ public void testTempOld() {
+ assertFalse(isOldTempRecord("test.txt"));
+ assertFalse(isOldTempRecord("testfolder/test.txt"));
+
+ long tnow = Time.now();
+ String tmpFile1 = "test." + tnow + ".tmp";
+ assertFalse(isOldTempRecord(tmpFile1));
+
+ long told = Time.now() - TimeUnit.MINUTES.toMillis(1);
+ String tmpFile2 = "test." + told + ".tmp";
+ assertTrue(isOldTempRecord(tmpFile2));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileSystem.java
new file mode 100644
index 0000000..8c4b188
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileSystem.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils;
+import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileSystemImpl;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test the FileSystem (e.g., HDFS) implementation of the State Store driver.
+ */
+public class TestStateStoreFileSystem extends TestStateStoreDriverBase {
+
+ private static MiniDFSCluster dfsCluster;
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ Configuration conf = FederationStateStoreTestUtils
+ .getStateStoreConfiguration(StateStoreFileSystemImpl.class);
+ conf.set(StateStoreFileSystemImpl.FEDERATION_STORE_FS_PATH,
+ "/hdfs-federation/");
+
+ // Create HDFS cluster to back the state tore
+ MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
+ builder.numDataNodes(1);
+ dfsCluster = builder.build();
+ dfsCluster.waitClusterUp();
+ getStateStore(conf);
+ }
+
+ @AfterClass
+ public static void tearDownCluster() {
+ if (dfsCluster != null) {
+ dfsCluster.shutdown();
+ }
+ }
+
+ @Before
+ public void startup() throws IOException {
+ removeAll(getStateStoreDriver());
+ }
+
+ @Test
+ public void testInsert()
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testInsert(getStateStoreDriver());
+ }
+
+ @Test
+ public void testUpdate() throws IllegalArgumentException, IOException,
+ SecurityException, ReflectiveOperationException {
+ testPut(getStateStoreDriver());
+ }
+
+ @Test
+ public void testDelete()
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testRemove(getStateStoreDriver());
+ }
+
+ @Test
+ public void testFetchErrors()
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testFetchErrors(getStateStoreDriver());
+ }
+
+ @Test
+ public void testMetrics()
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testMetrics(getStateStoreDriver());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreZK.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreZK.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreZK.java
new file mode 100644
index 0000000..3cf7c91
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreZK.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver;
+
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.getStateStoreConfiguration;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.RetryNTimes;
+import org.apache.curator.test.TestingServer;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test the ZooKeeper implementation of the State Store driver.
+ */
+public class TestStateStoreZK extends TestStateStoreDriverBase {
+
+ private static TestingServer curatorTestingServer;
+ private static CuratorFramework curatorFramework;
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ curatorTestingServer = new TestingServer();
+ curatorTestingServer.start();
+ String connectString = curatorTestingServer.getConnectString();
+ curatorFramework = CuratorFrameworkFactory.builder()
+ .connectString(connectString)
+ .retryPolicy(new RetryNTimes(100, 100))
+ .build();
+ curatorFramework.start();
+
+ // Create the ZK State Store
+ Configuration conf =
+ getStateStoreConfiguration(StateStoreZooKeeperImpl.class);
+ conf.set(CommonConfigurationKeys.ZK_ADDRESS, connectString);
+ // Disable auto-repair of connection
+ conf.setLong(RBFConfigKeys.FEDERATION_STORE_CONNECTION_TEST_MS,
+ TimeUnit.HOURS.toMillis(1));
+ getStateStore(conf);
+ }
+
+ @AfterClass
+ public static void tearDownCluster() {
+ curatorFramework.close();
+ try {
+ curatorTestingServer.stop();
+ } catch (IOException e) {
+ }
+ }
+
+ @Before
+ public void startup() throws IOException {
+ removeAll(getStateStoreDriver());
+ }
+
+ @Test
+ public void testInsert()
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testInsert(getStateStoreDriver());
+ }
+
+ @Test
+ public void testUpdate()
+ throws IllegalArgumentException, ReflectiveOperationException,
+ IOException, SecurityException {
+ testPut(getStateStoreDriver());
+ }
+
+ @Test
+ public void testDelete()
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testRemove(getStateStoreDriver());
+ }
+
+ @Test
+ public void testFetchErrors()
+ throws IllegalArgumentException, IllegalAccessException, IOException {
+ testFetchErrors(getStateStoreDriver());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMembershipState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMembershipState.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMembershipState.java
new file mode 100644
index 0000000..d922414
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMembershipState.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.junit.Test;
+
+/**
+ * Test the Membership State records.
+ */
+public class TestMembershipState {
+
+ private static final String ROUTER = "router";
+ private static final String NAMESERVICE = "nameservice";
+ private static final String NAMENODE = "namenode";
+ private static final String CLUSTER_ID = "cluster";
+ private static final String BLOCKPOOL_ID = "blockpool";
+ private static final String RPC_ADDRESS = "rpcaddress";
+ private static final String SERVICE_ADDRESS = "serviceaddress";
+ private static final String LIFELINE_ADDRESS = "lifelineaddress";
+ private static final String WEB_ADDRESS = "webaddress";
+ private static final boolean SAFE_MODE = false;
+
+ private static final long DATE_CREATED = 100;
+ private static final long DATE_MODIFIED = 200;
+
+ private static final long NUM_BLOCKS = 300;
+ private static final long NUM_FILES = 400;
+ private static final int NUM_DEAD = 500;
+ private static final int NUM_ACTIVE = 600;
+ private static final int NUM_DECOM = 700;
+ private static final int NUM_DECOM_ACTIVE = 800;
+ private static final int NUM_DECOM_DEAD = 900;
+ private static final long NUM_BLOCK_MISSING = 1000;
+
+ private static final long TOTAL_SPACE = 1100;
+ private static final long AVAILABLE_SPACE = 1200;
+
+ private static final FederationNamenodeServiceState STATE =
+ FederationNamenodeServiceState.ACTIVE;
+
+ private MembershipState createRecord() throws IOException {
+
+ MembershipState record = MembershipState.newInstance(
+ ROUTER, NAMESERVICE, NAMENODE, CLUSTER_ID,
+ BLOCKPOOL_ID, RPC_ADDRESS, SERVICE_ADDRESS, LIFELINE_ADDRESS,
+ WEB_ADDRESS, STATE, SAFE_MODE);
+ record.setDateCreated(DATE_CREATED);
+ record.setDateModified(DATE_MODIFIED);
+
+ MembershipStats stats = MembershipStats.newInstance();
+ stats.setNumOfBlocks(NUM_BLOCKS);
+ stats.setNumOfFiles(NUM_FILES);
+ stats.setNumOfActiveDatanodes(NUM_ACTIVE);
+ stats.setNumOfDeadDatanodes(NUM_DEAD);
+ stats.setNumOfDecommissioningDatanodes(NUM_DECOM);
+ stats.setNumOfDecomActiveDatanodes(NUM_DECOM_ACTIVE);
+ stats.setNumOfDecomDeadDatanodes(NUM_DECOM_DEAD);
+ stats.setNumOfBlocksMissing(NUM_BLOCK_MISSING);
+ stats.setTotalSpace(TOTAL_SPACE);
+ stats.setAvailableSpace(AVAILABLE_SPACE);
+ record.setStats(stats);
+ return record;
+ }
+
+ private void validateRecord(MembershipState record) throws IOException {
+
+ assertEquals(ROUTER, record.getRouterId());
+ assertEquals(NAMESERVICE, record.getNameserviceId());
+ assertEquals(CLUSTER_ID, record.getClusterId());
+ assertEquals(BLOCKPOOL_ID, record.getBlockPoolId());
+ assertEquals(RPC_ADDRESS, record.getRpcAddress());
+ assertEquals(WEB_ADDRESS, record.getWebAddress());
+ assertEquals(STATE, record.getState());
+ assertEquals(SAFE_MODE, record.getIsSafeMode());
+ assertEquals(DATE_CREATED, record.getDateCreated());
+ assertEquals(DATE_MODIFIED, record.getDateModified());
+
+ MembershipStats stats = record.getStats();
+ assertEquals(NUM_BLOCKS, stats.getNumOfBlocks());
+ assertEquals(NUM_FILES, stats.getNumOfFiles());
+ assertEquals(NUM_ACTIVE, stats.getNumOfActiveDatanodes());
+ assertEquals(NUM_DEAD, stats.getNumOfDeadDatanodes());
+ assertEquals(NUM_DECOM, stats.getNumOfDecommissioningDatanodes());
+ assertEquals(NUM_DECOM_ACTIVE, stats.getNumOfDecomActiveDatanodes());
+ assertEquals(NUM_DECOM_DEAD, stats.getNumOfDecomDeadDatanodes());
+ assertEquals(TOTAL_SPACE, stats.getTotalSpace());
+ assertEquals(AVAILABLE_SPACE, stats.getAvailableSpace());
+ }
+
+ @Test
+ public void testGetterSetter() throws IOException {
+ MembershipState record = createRecord();
+ validateRecord(record);
+ }
+
+ @Test
+ public void testSerialization() throws IOException {
+
+ MembershipState record = createRecord();
+
+ StateStoreSerializer serializer = StateStoreSerializer.getSerializer();
+ String serializedString = serializer.serializeString(record);
+ MembershipState newRecord =
+ serializer.deserialize(serializedString, MembershipState.class);
+
+ validateRecord(newRecord);
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSafemodeService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSafemodeService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSafemodeService.java
new file mode 100644
index 0000000..5dfb356
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSafemodeService.java
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.apache.hadoop.util.Time.now;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Service to periodically check if the {@link org.apache.hadoop.hdfs.server.
+ * federation.store.StateStoreService StateStoreService} cached information in
+ * the {@link Router} is up to date. This is for performance and removes the
+ * {@link org.apache.hadoop.hdfs.server.federation.store.StateStoreService
+ * StateStoreService} from the critical path in common operations.
+ */
+public class RouterSafemodeService extends PeriodicService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RouterSafemodeService.class);
+
+ /** Router to manage safe mode. */
+ private final Router router;
+
+ /** Interval in ms to wait post startup before allowing RPC requests. */
+ private long startupInterval;
+ /** Interval in ms after which the State Store cache is too stale. */
+ private long staleInterval;
+ /** Start time in ms of this service. */
+ private long startupTime;
+
+ /** The time the Router enters safe mode in milliseconds. */
+ private long enterSafeModeTime = now();
+
+
+ /**
+ * Create a new Cache update service.
+ *
+ * @param router Router containing the cache.
+ */
+ public RouterSafemodeService(Router router) {
+ super(RouterSafemodeService.class.getSimpleName());
+ this.router = router;
+ }
+
+ /**
+ * Enter safe mode.
+ */
+ private void enter() {
+ LOG.info("Entering safe mode");
+ enterSafeModeTime = now();
+ RouterRpcServer rpcServer = router.getRpcServer();
+ rpcServer.setSafeMode(true);
+ router.updateRouterState(RouterServiceState.SAFEMODE);
+ }
+
+ /**
+ * Leave safe mode.
+ */
+ private void leave() {
+ // Cache recently updated, leave safemode
+ long timeInSafemode = now() - enterSafeModeTime;
+ LOG.info("Leaving safe mode after {} milliseconds", timeInSafemode);
+ RouterMetrics routerMetrics = router.getRouterMetrics();
+ if (routerMetrics == null) {
+ LOG.error("The Router metrics are not enabled");
+ } else {
+ routerMetrics.setSafeModeTime(timeInSafemode);
+ }
+ RouterRpcServer rpcServer = router.getRpcServer();
+ rpcServer.setSafeMode(false);
+ router.updateRouterState(RouterServiceState.RUNNING);
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+
+ // Use same interval as cache update service
+ this.setIntervalMs(conf.getTimeDuration(
+ RBFConfigKeys.DFS_ROUTER_CACHE_TIME_TO_LIVE_MS,
+ RBFConfigKeys.DFS_ROUTER_CACHE_TIME_TO_LIVE_MS_DEFAULT,
+ TimeUnit.MILLISECONDS));
+
+ this.startupInterval = conf.getTimeDuration(
+ RBFConfigKeys.DFS_ROUTER_SAFEMODE_EXTENSION,
+ RBFConfigKeys.DFS_ROUTER_SAFEMODE_EXTENSION_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ LOG.info("Leave startup safe mode after {} ms", this.startupInterval);
+
+ this.staleInterval = conf.getTimeDuration(
+ RBFConfigKeys.DFS_ROUTER_SAFEMODE_EXPIRATION,
+ RBFConfigKeys.DFS_ROUTER_SAFEMODE_EXPIRATION_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ LOG.info("Enter safe mode after {} ms without reaching the State Store",
+ this.staleInterval);
+
+ this.startupTime = Time.now();
+
+ // Initializing the RPC server in safe mode, it will disable it later
+ enter();
+
+ super.serviceInit(conf);
+ }
+
+ @Override
+ public void periodicInvoke() {
+ long now = Time.now();
+ long delta = now - startupTime;
+ if (delta < startupInterval) {
+ LOG.info("Delaying safemode exit for {} milliseconds...",
+ this.startupInterval - delta);
+ return;
+ }
+ RouterRpcServer rpcServer = router.getRpcServer();
+ StateStoreService stateStore = router.getStateStore();
+ long cacheUpdateTime = stateStore.getCacheUpdateTime();
+ boolean isCacheStale = (now - cacheUpdateTime) > this.staleInterval;
+
+ // Always update to indicate our cache was updated
+ if (isCacheStale) {
+ if (!rpcServer.isInSafeMode()) {
+ enter();
+ }
+ } else if (rpcServer.isInSafeMode()) {
+ // Cache recently updated, leave safe mode
+ leave();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterServiceState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterServiceState.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterServiceState.java
new file mode 100644
index 0000000..3accbe9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterServiceState.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+/**
+ * States of the Router.
+ */
+public enum RouterServiceState {
+ UNINITIALIZED,
+ INITIALIZING,
+ SAFEMODE,
+ RUNNING,
+ STOPPING,
+ SHUTDOWN,
+ EXPIRED;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterStateManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterStateManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterStateManager.java
new file mode 100644
index 0000000..527600c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterStateManager.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeResponse;
+
+/**
+ * Interface of managing the Router state.
+ */
+public interface RouterStateManager {
+ /**
+ * Enter safe mode and change Router state to RouterServiceState#SAFEMODE.
+ */
+ EnterSafeModeResponse enterSafeMode(EnterSafeModeRequest request)
+ throws IOException;
+
+ /**
+ * Leave safe mode and change Router state to RouterServiceState#RUNNING.
+ */
+ LeaveSafeModeResponse leaveSafeMode(LeaveSafeModeRequest request)
+ throws IOException;
+
+ /**
+ * Verify if current Router state is safe mode.
+ */
+ GetSafeModeResponse getSafeMode(GetSafeModeRequest request)
+ throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/package-info.java
new file mode 100644
index 0000000..327f39b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/package-info.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The router package includes the core services for a HDFS federation router.
+ * The {@link Router} acts as a transparent proxy in front of a cluster of
+ * multiple NameNodes and nameservices. The {@link RouterRpcServer} exposes the
+ * NameNode clientProtocol and is the primary contact point for DFS clients in a
+ * federated cluster.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/CachedRecordStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/CachedRecordStore.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/CachedRecordStore.java
new file mode 100644
index 0000000..cdd4449
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/CachedRecordStore.java
@@ -0,0 +1,242 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.hadoop.hdfs.server.federation.metrics.StateStoreMetrics;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Record store that takes care of caching the records in memory.
+ *
+ * @param <R> Record to store by this interface.
+ */
+public abstract class CachedRecordStore<R extends BaseRecord>
+ extends RecordStore<R> implements StateStoreCache {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(CachedRecordStore.class);
+
+
+ /** Prevent loading the cache more than once every 500 ms. */
+ private static final long MIN_UPDATE_MS = 500;
+
+
+ /** Cached entries. */
+ private List<R> records = new ArrayList<>();
+
+ /** Time stamp of the cached entries. */
+ private long timestamp = -1;
+
+ /** If the cache is initialized. */
+ private boolean initialized = false;
+
+ /** Last time the cache was updated. */
+ private long lastUpdate = -1;
+
+ /** Lock to access the memory cache. */
+ private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ private final Lock readLock = readWriteLock.readLock();
+ private final Lock writeLock = readWriteLock.writeLock();
+
+ /** If it should override the expired values when loading the cache. */
+ private boolean override = false;
+
+
+ /**
+ * Create a new cached record store.
+ *
+ * @param clazz Class of the record to store.
+ * @param driver State Store driver.
+ */
+ protected CachedRecordStore(Class<R> clazz, StateStoreDriver driver) {
+ this(clazz, driver, false);
+ }
+
+ /**
+ * Create a new cached record store.
+ *
+ * @param clazz Class of the record to store.
+ * @param driver State Store driver.
+ * @param over If the entries should be override if they expire
+ */
+ protected CachedRecordStore(
+ Class<R> clazz, StateStoreDriver driver, boolean over) {
+ super(clazz, driver);
+
+ this.override = over;
+ }
+
+ /**
+ * Check that the cache of the State Store information is available.
+ *
+ * @throws StateStoreUnavailableException If the cache is not initialized.
+ */
+ private void checkCacheAvailable() throws StateStoreUnavailableException {
+ if (!this.initialized) {
+ throw new StateStoreUnavailableException(
+ "Cached State Store not initialized, " +
+ getRecordClass().getSimpleName() + " records not valid");
+ }
+ }
+
+ @Override
+ public boolean loadCache(boolean force) throws IOException {
+ // Prevent loading the cache too frequently
+ if (force || isUpdateTime()) {
+ List<R> newRecords = null;
+ long t = -1;
+ try {
+ QueryResult<R> result = getDriver().get(getRecordClass());
+ newRecords = result.getRecords();
+ t = result.getTimestamp();
+
+ // If we have any expired record, update the State Store
+ if (this.override) {
+ overrideExpiredRecords(result);
+ }
+ } catch (IOException e) {
+ LOG.error("Cannot get \"{}\" records from the State Store",
+ getRecordClass().getSimpleName());
+ this.initialized = false;
+ return false;
+ }
+
+ // Update cache atomically
+ writeLock.lock();
+ try {
+ this.records.clear();
+ this.records.addAll(newRecords);
+ this.timestamp = t;
+ this.initialized = true;
+ } finally {
+ writeLock.unlock();
+ }
+
+ // Update the metrics for the cache State Store size
+ StateStoreMetrics metrics = getDriver().getMetrics();
+ if (metrics != null) {
+ String recordName = getRecordClass().getSimpleName();
+ metrics.setCacheSize(recordName, this.records.size());
+ }
+
+ lastUpdate = Time.monotonicNow();
+ }
+ return true;
+ }
+
+ /**
+ * Check if it's time to update the cache. Update it it was never updated.
+ *
+ * @return If it's time to update this cache.
+ */
+ private boolean isUpdateTime() {
+ return Time.monotonicNow() - lastUpdate > MIN_UPDATE_MS;
+ }
+
+ /**
+ * Updates the state store with any record overrides we detected, such as an
+ * expired state.
+ *
+ * @param query RecordQueryResult containing the data to be inspected.
+ * @throws IOException
+ */
+ public void overrideExpiredRecords(QueryResult<R> query) throws IOException {
+ List<R> commitRecords = new ArrayList<>();
+ List<R> newRecords = query.getRecords();
+ long currentDriverTime = query.getTimestamp();
+ if (newRecords == null || currentDriverTime <= 0) {
+ LOG.error("Cannot check overrides for record");
+ return;
+ }
+ for (R record : newRecords) {
+ if (record.checkExpired(currentDriverTime)) {
+ String recordName = StateStoreUtils.getRecordName(record.getClass());
+ LOG.info("Override State Store record {}: {}", recordName, record);
+ commitRecords.add(record);
+ }
+ }
+ if (commitRecords.size() > 0) {
+ getDriver().putAll(commitRecords, true, false);
+ }
+ }
+
+ /**
+ * Updates the state store with any record overrides we detected, such as an
+ * expired state.
+ *
+ * @param record Record record to be updated.
+ * @throws IOException
+ */
+ public void overrideExpiredRecord(R record) throws IOException {
+ List<R> newRecords = Collections.singletonList(record);
+ long time = getDriver().getTime();
+ QueryResult<R> query = new QueryResult<>(newRecords, time);
+ overrideExpiredRecords(query);
+ }
+
+ /**
+ * Get all the cached records.
+ *
+ * @return Copy of the cached records.
+ * @throws StateStoreUnavailableException If the State store is not available.
+ */
+ public List<R> getCachedRecords() throws StateStoreUnavailableException {
+ checkCacheAvailable();
+
+ List<R> ret = new LinkedList<R>();
+ this.readLock.lock();
+ try {
+ ret.addAll(this.records);
+ } finally {
+ this.readLock.unlock();
+ }
+ return ret;
+ }
+
+ /**
+ * Get all the cached records and the time stamp of the cache.
+ *
+ * @return Copy of the cached records and the time stamp.
+ * @throws StateStoreUnavailableException If the State store is not available.
+ */
+ protected QueryResult<R> getCachedRecordsAndTimeStamp()
+ throws StateStoreUnavailableException {
+ checkCacheAvailable();
+
+ this.readLock.lock();
+ try {
+ return new QueryResult<R>(this.records, this.timestamp);
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MembershipStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MembershipStore.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MembershipStore.java
new file mode 100644
index 0000000..3e8ba6b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MembershipStore.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateNamenodeRegistrationRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateNamenodeRegistrationResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+
+/**
+ * Management API for NameNode registrations stored in
+ * {@link org.apache.hadoop.hdfs.server.federation.store.records.MembershipState
+ * MembershipState} records. The {@link org.apache.hadoop.hdfs.server.
+ * federation.router.RouterHeartbeatService RouterHeartbeatService} periodically
+ * polls each NN to update the NameNode metadata(addresses, operational) and HA
+ * state(active, standby). Each NameNode may be polled by multiple
+ * {@link org.apache.hadoop.hdfs.server.federation.router.Router Router}
+ * instances.
+ * <p>
+ * Once fetched from the
+ * {@link org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver
+ * StateStoreDriver}, NameNode registrations are cached until the next query.
+ * The fetched registration data is aggregated using a quorum to determine the
+ * best/most accurate state for each NameNode. The cache is periodically updated
+ * by the @{link StateStoreCacheUpdateService}.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public abstract class MembershipStore
+ extends CachedRecordStore<MembershipState> {
+
+ protected MembershipStore(StateStoreDriver driver) {
+ super(MembershipState.class, driver, true);
+ }
+
+ /**
+ * Inserts or updates a namenode membership entry into the table.
+ *
+ * @param request Fully populated NamenodeHeartbeatRequest request.
+ * @return True if successful, false otherwise.
+ * @throws StateStoreUnavailableException Throws exception if the data store
+ * is not initialized.
+ * @throws IOException if the data store could not be queried or the query is
+ * invalid.
+ */
+ public abstract NamenodeHeartbeatResponse namenodeHeartbeat(
+ NamenodeHeartbeatRequest request) throws IOException;
+
+ /**
+ * Queries for a single cached registration entry matching the given
+ * parameters. Possible keys are the names of data structure elements Possible
+ * values are matching SQL "LIKE" targets.
+ *
+ * @param request Fully populated GetNamenodeRegistrationsRequest request.
+ * @return Single matching FederationMembershipStateEntry or null if not found
+ * or more than one entry matches.
+ * @throws StateStoreUnavailableException Throws exception if the data store
+ * is not initialized.
+ * @throws IOException if the data store could not be queried or the query is
+ * invalid.
+ */
+ public abstract GetNamenodeRegistrationsResponse getNamenodeRegistrations(
+ GetNamenodeRegistrationsRequest request) throws IOException;
+
+ /**
+ * Get the expired registrations from the registration cache.
+ *
+ * @return Expired registrations or zero-length list if none are found.
+ * @throws StateStoreUnavailableException Throws exception if the data store
+ * is not initialized.
+ * @throws IOException if the data store could not be queried or the query is
+ * invalid.
+ */
+ public abstract GetNamenodeRegistrationsResponse
+ getExpiredNamenodeRegistrations(GetNamenodeRegistrationsRequest request)
+ throws IOException;
+
+ /**
+ * Retrieves a list of registered nameservices and their associated info.
+ *
+ * @param request
+ * @return Collection of information for each registered nameservice.
+ * @throws IOException if the data store could not be queried or the query is
+ * invalid.
+ */
+ public abstract GetNamespaceInfoResponse getNamespaceInfo(
+ GetNamespaceInfoRequest request) throws IOException;
+
+ /**
+ * Overrides a cached namenode state with an updated state.
+ *
+ * @param request Fully populated OverrideNamenodeRegistrationRequest request.
+ * @return OverrideNamenodeRegistrationResponse
+ * @throws StateStoreUnavailableException if the data store is not
+ * initialized.
+ * @throws IOException if the data store could not be queried or the query is
+ * invalid.
+ */
+ public abstract UpdateNamenodeRegistrationResponse updateNamenodeRegistration(
+ UpdateNamenodeRegistrationRequest request) throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MountTableStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MountTableStore.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MountTableStore.java
new file mode 100644
index 0000000..b439659
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MountTableStore.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+
+/**
+ * Management API for the HDFS mount table information stored in
+ * {@link org.apache.hadoop.hdfs.server.federation.store.records.MountTable
+ * MountTable} records. The mount table contains entries that map a particular
+ * global namespace path one or more HDFS nameservices (NN) + target path. It is
+ * possible to map mount locations for root folders, directories or individual
+ * files.
+ * <p>
+ * Once fetched from the
+ * {@link org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver
+ * StateStoreDriver}, MountTable records are cached in a tree for faster access.
+ * Each path in the global namespace is mapped to a nameserivce ID and local
+ * path upon request. The cache is periodically updated by the @{link
+ * StateStoreCacheUpdateService}.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public abstract class MountTableStore extends CachedRecordStore<MountTable>
+ implements MountTableManager {
+
+ public MountTableStore(StateStoreDriver driver) {
+ super(MountTable.class, driver);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RecordStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RecordStore.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RecordStore.java
new file mode 100644
index 0000000..53a8b82
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RecordStore.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import java.lang.reflect.Constructor;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+
+/**
+ * Store records in the State Store. Subclasses provide interfaces to operate on
+ * those records.
+ *
+ * @param <R> Record to store by this interface.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public abstract class RecordStore<R extends BaseRecord> {
+
+ private static final Log LOG = LogFactory.getLog(RecordStore.class);
+
+
+ /** Class of the record stored in this State Store. */
+ private final Class<R> recordClass;
+
+ /** State store driver backed by persistent storage. */
+ private final StateStoreDriver driver;
+
+
+ /**
+ * Create a new store for records.
+ *
+ * @param clazz Class of the record to store.
+ * @param stateStoreDriver Driver for the State Store.
+ */
+ protected RecordStore(Class<R> clazz, StateStoreDriver stateStoreDriver) {
+ this.recordClass = clazz;
+ this.driver = stateStoreDriver;
+ }
+
+ /**
+ * Report a required record to the data store. The data store uses this to
+ * create/maintain storage for the record.
+ *
+ * @return The class of the required record or null if no record is required
+ * for this interface.
+ */
+ public Class<R> getRecordClass() {
+ return this.recordClass;
+ }
+
+ /**
+ * Get the State Store driver.
+ *
+ * @return State Store driver.
+ */
+ protected StateStoreDriver getDriver() {
+ return this.driver;
+ }
+
+ /**
+ * Build a state store API implementation interface.
+ *
+ * @param clazz The specific interface implementation to create
+ * @param driver The {@link StateStoreDriver} implementation in use.
+ * @return An initialized instance of the specified state store API
+ * implementation.
+ */
+ public static <T extends RecordStore<?>> T newInstance(
+ final Class<T> clazz, final StateStoreDriver driver) {
+
+ try {
+ Constructor<T> constructor = clazz.getConstructor(StateStoreDriver.class);
+ T recordStore = constructor.newInstance(driver);
+ return recordStore;
+ } catch (Exception e) {
+ LOG.error("Cannot create new instance for " + clazz, e);
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RouterStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RouterStore.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RouterStore.java
new file mode 100644
index 0000000..c6a0dad
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/RouterStore.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationsRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationsResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+
+/**
+ * Management API for
+ * {@link org.apache.hadoop.hdfs.server.federation.store.records.RouterState
+ * RouterState} records in the state store. Accesses the data store via the
+ * {@link org.apache.hadoop.hdfs.server.federation.store.driver.
+ * StateStoreDriver StateStoreDriver} interface. No data is cached.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public abstract class RouterStore extends CachedRecordStore<RouterState> {
+
+ public RouterStore(StateStoreDriver driver) {
+ super(RouterState.class, driver, true);
+ }
+
+ /**
+ * Fetches the current router state object.
+ *
+ * @param request Fully populated request object.
+ * @return The matching router record or null if none exists.
+ * @throws IOException Throws exception if unable to query the data store or
+ * if more than one matching record is found.
+ */
+ public abstract GetRouterRegistrationResponse getRouterRegistration(
+ GetRouterRegistrationRequest request) throws IOException;
+
+ /**
+ * Fetches all router status objects.
+ *
+ * @param request Fully populated request object.
+ * @return List of Router records present in the data store.
+ * @throws IOException Throws exception if unable to query the data store
+ */
+ public abstract GetRouterRegistrationsResponse getRouterRegistrations(
+ GetRouterRegistrationsRequest request) throws IOException;
+
+ /**
+ * Update the state of this router in the State Store.
+ *
+ * @param request Fully populated request object.
+ * @return True if the update was successfully recorded, false otherwise.
+ * @throws IOException Throws exception if unable to query the data store
+ */
+ public abstract RouterHeartbeatResponse routerHeartbeat(
+ RouterHeartbeatRequest request) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreCache.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreCache.java
new file mode 100644
index 0000000..83fc501
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreCache.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import java.io.IOException;
+
+/**
+ * Interface for a cached copy of the State Store.
+ */
+public interface StateStoreCache {
+
+ /**
+ * Load the cache from the State Store. Called by the cache update service
+ * when the data has been reloaded.
+ *
+ * @param force If we force the load.
+ * @return If the cache was loaded successfully.
+ * @throws IOException If there was an error loading the cache.
+ */
+ boolean loadCache(boolean force) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreCacheUpdateService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreCacheUpdateService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreCacheUpdateService.java
new file mode 100644
index 0000000..6a4dc76
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreCacheUpdateService.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.router.PeriodicService;
+import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Service to periodically update the {@link StateStoreService}
+ * cached information in the
+ * {@link org.apache.hadoop.hdfs.server.federation.router.Router Router}.
+ * This is for performance and removes the State Store from the critical path
+ * in common operations.
+ */
+public class StateStoreCacheUpdateService extends PeriodicService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StateStoreCacheUpdateService.class);
+
+ /** The service that manages the State Store connection. */
+ private final StateStoreService stateStore;
+
+
+ /**
+ * Create a new Cache update service.
+ *
+ * @param stateStore Implementation of the state store
+ */
+ public StateStoreCacheUpdateService(StateStoreService stateStore) {
+ super(StateStoreCacheUpdateService.class.getSimpleName());
+ this.stateStore = stateStore;
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+
+ this.setIntervalMs(conf.getTimeDuration(
+ RBFConfigKeys.DFS_ROUTER_CACHE_TIME_TO_LIVE_MS,
+ RBFConfigKeys.DFS_ROUTER_CACHE_TIME_TO_LIVE_MS_DEFAULT,
+ TimeUnit.MILLISECONDS));
+
+ super.serviceInit(conf);
+ }
+
+ @Override
+ public void periodicInvoke() {
+ LOG.debug("Updating State Store cache");
+ stateStore.refreshCaches();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreConnectionMonitorService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreConnectionMonitorService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreConnectionMonitorService.java
new file mode 100644
index 0000000..8ebf4b8
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreConnectionMonitorService.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.router.PeriodicService;
+import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Service to periodically monitor the connection of the StateStore
+ * {@link StateStoreService} data store and to re-open the connection
+ * to the data store if required.
+ */
+public class StateStoreConnectionMonitorService extends PeriodicService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StateStoreConnectionMonitorService.class);
+
+ /** Service that maintains the State Store connection. */
+ private final StateStoreService stateStore;
+
+
+ /**
+ * Create a new service to monitor the connectivity of the state store driver.
+ *
+ * @param store Instance of the state store to be monitored.
+ */
+ public StateStoreConnectionMonitorService(StateStoreService store) {
+ super(StateStoreConnectionMonitorService.class.getSimpleName());
+ this.stateStore = store;
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ this.setIntervalMs(conf.getLong(
+ RBFConfigKeys.FEDERATION_STORE_CONNECTION_TEST_MS,
+ RBFConfigKeys.FEDERATION_STORE_CONNECTION_TEST_MS_DEFAULT));
+
+ super.serviceInit(conf);
+ }
+
+ @Override
+ public void periodicInvoke() {
+ LOG.debug("Checking state store connection");
+ if (!stateStore.isDriverReady()) {
+ LOG.info("Attempting to open state store driver.");
+ stateStore.loadDriver();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
new file mode 100644
index 0000000..ccbde09
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
@@ -0,0 +1,450 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import javax.management.NotCompliantMBeanException;
+import javax.management.ObjectName;
+import javax.management.StandardMBean;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.metrics.StateStoreMBean;
+import org.apache.hadoop.hdfs.server.federation.metrics.StateStoreMetrics;
+import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.impl.MembershipStoreImpl;
+import org.apache.hadoop.hdfs.server.federation.store.impl.MountTableStoreImpl;
+import org.apache.hadoop.hdfs.server.federation.store.impl.RouterStoreImpl;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+import org.apache.hadoop.metrics2.MetricsException;
+import org.apache.hadoop.metrics2.util.MBeans;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * A service to initialize a
+ * {@link org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver
+ * StateStoreDriver} and maintain the connection to the data store. There are
+ * multiple state store driver connections supported:
+ * <ul>
+ * <li>File
+ * {@link org.apache.hadoop.hdfs.server.federation.store.driver.impl.
+ * StateStoreFileImpl StateStoreFileImpl}
+ * <li>ZooKeeper
+ * {@link org.apache.hadoop.hdfs.server.federation.store.driver.impl.
+ * StateStoreZooKeeperImpl StateStoreZooKeeperImpl}
+ * </ul>
+ * <p>
+ * The service also supports the dynamic registration of record stores like:
+ * <ul>
+ * <li>{@link MembershipStore}: state of the Namenodes in the
+ * federation.
+ * <li>{@link MountTableStore}: Mount table between to subclusters.
+ * See {@link org.apache.hadoop.fs.viewfs.ViewFs ViewFs}.
+ * <li>{@link RebalancerStore}: Log of the rebalancing operations.
+ * <li>{@link RouterStore}: Router state in the federation.
+ * <li>{@link TokenStore}: Tokens in the federation.
+ * </ul>
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class StateStoreService extends CompositeService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StateStoreService.class);
+
+
+ /** State Store configuration. */
+ private Configuration conf;
+
+ /** Identifier for the service. */
+ private String identifier;
+
+ /** Driver for the back end connection. */
+ private StateStoreDriver driver;
+
+ /** Service to maintain data store connection. */
+ private StateStoreConnectionMonitorService monitorService;
+
+ /** StateStore metrics. */
+ private StateStoreMetrics metrics;
+
+ /** Supported record stores. */
+ private final Map<
+ Class<? extends BaseRecord>, RecordStore<? extends BaseRecord>>
+ recordStores;
+
+ /** Service to maintain State Store caches. */
+ private StateStoreCacheUpdateService cacheUpdater;
+ /** Time the cache was last successfully updated. */
+ private long cacheLastUpdateTime;
+ /** List of internal caches to update. */
+ private final List<StateStoreCache> cachesToUpdateInternal;
+ /** List of external caches to update. */
+ private final List<StateStoreCache> cachesToUpdateExternal;
+
+
+ public StateStoreService() {
+ super(StateStoreService.class.getName());
+
+ // Records and stores supported by this implementation
+ this.recordStores = new HashMap<>();
+
+ // Caches to maintain
+ this.cachesToUpdateInternal = new ArrayList<>();
+ this.cachesToUpdateExternal = new ArrayList<>();
+ }
+
+ /**
+ * Initialize the State Store and the connection to the backend.
+ *
+ * @param config Configuration for the State Store.
+ * @throws IOException
+ */
+ @Override
+ protected void serviceInit(Configuration config) throws Exception {
+ this.conf = config;
+
+ // Create implementation of State Store
+ Class<? extends StateStoreDriver> driverClass = this.conf.getClass(
+ RBFConfigKeys.FEDERATION_STORE_DRIVER_CLASS,
+ RBFConfigKeys.FEDERATION_STORE_DRIVER_CLASS_DEFAULT,
+ StateStoreDriver.class);
+ this.driver = ReflectionUtils.newInstance(driverClass, this.conf);
+
+ if (this.driver == null) {
+ throw new IOException("Cannot create driver for the State Store");
+ }
+
+ // Add supported record stores
+ addRecordStore(MembershipStoreImpl.class);
+ addRecordStore(MountTableStoreImpl.class);
+ addRecordStore(RouterStoreImpl.class);
+
+ // Check the connection to the State Store periodically
+ this.monitorService = new StateStoreConnectionMonitorService(this);
+ this.addService(monitorService);
+
+ // Set expirations intervals for each record
+ MembershipState.setExpirationMs(conf.getLong(
+ RBFConfigKeys.FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS,
+ RBFConfigKeys.FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS_DEFAULT));
+
+ RouterState.setExpirationMs(conf.getTimeDuration(
+ RBFConfigKeys.FEDERATION_STORE_ROUTER_EXPIRATION_MS,
+ RBFConfigKeys.FEDERATION_STORE_ROUTER_EXPIRATION_MS_DEFAULT,
+ TimeUnit.MILLISECONDS));
+
+ // Cache update service
+ this.cacheUpdater = new StateStoreCacheUpdateService(this);
+ addService(this.cacheUpdater);
+
+ // Create metrics for the State Store
+ this.metrics = StateStoreMetrics.create(conf);
+
+ // Adding JMX interface
+ try {
+ StandardMBean bean = new StandardMBean(metrics, StateStoreMBean.class);
+ ObjectName registeredObject =
+ MBeans.register("Router", "StateStore", bean);
+ LOG.info("Registered StateStoreMBean: {}", registeredObject);
+ } catch (NotCompliantMBeanException e) {
+ throw new RuntimeException("Bad StateStoreMBean setup", e);
+ } catch (MetricsException e) {
+ LOG.info("Failed to register State Store bean {}", e.getMessage());
+ }
+
+ super.serviceInit(this.conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ loadDriver();
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ closeDriver();
+
+ if (metrics != null) {
+ metrics.shutdown();
+ metrics = null;
+ }
+
+ super.serviceStop();
+ }
+
+ /**
+ * Add a record store to the State Store. It includes adding the store, the
+ * supported record and the cache management.
+ *
+ * @param clazz Class of the record store to track.
+ * @return New record store.
+ * @throws ReflectiveOperationException
+ */
+ private <T extends RecordStore<?>> void addRecordStore(
+ final Class<T> clazz) throws ReflectiveOperationException {
+
+ assert this.getServiceState() == STATE.INITED :
+ "Cannot add record to the State Store once started";
+
+ T recordStore = RecordStore.newInstance(clazz, this.getDriver());
+ Class<? extends BaseRecord> recordClass = recordStore.getRecordClass();
+ this.recordStores.put(recordClass, recordStore);
+
+ // Subscribe for cache updates
+ if (recordStore instanceof StateStoreCache) {
+ StateStoreCache cachedRecordStore = (StateStoreCache) recordStore;
+ this.cachesToUpdateInternal.add(cachedRecordStore);
+ }
+ }
+
+ /**
+ * Get the record store in this State Store for a given interface.
+ *
+ * @param recordStoreClass Class of the record store.
+ * @return Registered record store or null if not found.
+ */
+ public <T extends RecordStore<?>> T getRegisteredRecordStore(
+ final Class<T> recordStoreClass) {
+ for (RecordStore<? extends BaseRecord> recordStore :
+ this.recordStores.values()) {
+ if (recordStoreClass.isInstance(recordStore)) {
+ @SuppressWarnings("unchecked")
+ T recordStoreChecked = (T) recordStore;
+ return recordStoreChecked;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * List of records supported by this State Store.
+ *
+ * @return List of supported record classes.
+ */
+ public Collection<Class<? extends BaseRecord>> getSupportedRecords() {
+ return this.recordStores.keySet();
+ }
+
+ /**
+ * Load the State Store driver. If successful, refresh cached data tables.
+ */
+ public void loadDriver() {
+ synchronized (this.driver) {
+ if (!isDriverReady()) {
+ String driverName = this.driver.getClass().getSimpleName();
+ if (this.driver.init(
+ conf, getIdentifier(), getSupportedRecords(), metrics)) {
+ LOG.info("Connection to the State Store driver {} is open and ready",
+ driverName);
+ this.refreshCaches();
+ } else {
+ LOG.error("Cannot initialize State Store driver {}", driverName);
+ }
+ }
+ }
+ }
+
+ /**
+ * Check if the driver is ready to be used.
+ *
+ * @return If the driver is ready.
+ */
+ public boolean isDriverReady() {
+ return this.driver.isDriverReady();
+ }
+
+ /**
+ * Manually shuts down the driver.
+ *
+ * @throws Exception If the driver cannot be closed.
+ */
+ @VisibleForTesting
+ public void closeDriver() throws Exception {
+ if (this.driver != null) {
+ this.driver.close();
+ }
+ }
+
+ /**
+ * Get the state store driver.
+ *
+ * @return State store driver.
+ */
+ public StateStoreDriver getDriver() {
+ return this.driver;
+ }
+
+ /**
+ * Fetch a unique identifier for this state store instance. Typically it is
+ * the address of the router.
+ *
+ * @return Unique identifier for this store.
+ */
+ public String getIdentifier() {
+ return this.identifier;
+ }
+
+ /**
+ * Set a unique synchronization identifier for this store.
+ *
+ * @param id Unique identifier, typically the router's RPC address.
+ */
+ public void setIdentifier(String id) {
+ this.identifier = id;
+ }
+
+ //
+ // Cached state store data
+ //
+ /**
+ * The last time the state store cache was fully updated.
+ *
+ * @return Timestamp.
+ */
+ public long getCacheUpdateTime() {
+ return this.cacheLastUpdateTime;
+ }
+
+ /**
+ * Stops the cache update service.
+ */
+ @VisibleForTesting
+ public void stopCacheUpdateService() {
+ if (this.cacheUpdater != null) {
+ this.cacheUpdater.stop();
+ removeService(this.cacheUpdater);
+ this.cacheUpdater = null;
+ }
+ }
+
+ /**
+ * Register a cached record store for automatic periodic cache updates.
+ *
+ * @param client Client to the state store.
+ */
+ public void registerCacheExternal(StateStoreCache client) {
+ this.cachesToUpdateExternal.add(client);
+ }
+
+ /**
+ * Refresh the cache with information from the State Store. Called
+ * periodically by the CacheUpdateService to maintain data caches and
+ * versions.
+ */
+ public void refreshCaches() {
+ refreshCaches(false);
+ }
+
+ /**
+ * Refresh the cache with information from the State Store. Called
+ * periodically by the CacheUpdateService to maintain data caches and
+ * versions.
+ * @param force If we force the refresh.
+ */
+ public void refreshCaches(boolean force) {
+ boolean success = true;
+ if (isDriverReady()) {
+ List<StateStoreCache> cachesToUpdate = new LinkedList<>();
+ cachesToUpdate.addAll(cachesToUpdateInternal);
+ cachesToUpdate.addAll(cachesToUpdateExternal);
+ for (StateStoreCache cachedStore : cachesToUpdate) {
+ String cacheName = cachedStore.getClass().getSimpleName();
+ boolean result = false;
+ try {
+ result = cachedStore.loadCache(force);
+ } catch (IOException e) {
+ LOG.error("Error updating cache for {}", cacheName, e);
+ result = false;
+ }
+ if (!result) {
+ success = false;
+ LOG.error("Cache update failed for cache {}", cacheName);
+ }
+ }
+ } else {
+ success = false;
+ LOG.info("Skipping State Store cache update, driver is not ready.");
+ }
+ if (success) {
+ // Uses local time, not driver time.
+ this.cacheLastUpdateTime = Time.now();
+ }
+ }
+
+ /**
+ * Update the cache for a specific record store.
+ *
+ * @param clazz Class of the record store.
+ * @return If the cached was loaded.
+ * @throws IOException if the cache update failed.
+ */
+ public boolean loadCache(final Class<?> clazz) throws IOException {
+ return loadCache(clazz, false);
+ }
+
+ /**
+ * Update the cache for a specific record store.
+ *
+ * @param clazz Class of the record store.
+ * @param force Force the update ignoring cached periods.
+ * @return If the cached was loaded.
+ * @throws IOException if the cache update failed.
+ */
+ public boolean loadCache(Class<?> clazz, boolean force) throws IOException {
+ List<StateStoreCache> cachesToUpdate =
+ new LinkedList<StateStoreCache>();
+ cachesToUpdate.addAll(this.cachesToUpdateInternal);
+ cachesToUpdate.addAll(this.cachesToUpdateExternal);
+ for (StateStoreCache cachedStore : cachesToUpdate) {
+ if (clazz.isInstance(cachedStore)) {
+ return cachedStore.loadCache(force);
+ }
+ }
+ throw new IOException("Registered cache was not found for " + clazz);
+ }
+
+ /**
+ * Get the metrics for the State Store.
+ *
+ * @return State Store metrics.
+ */
+ public StateStoreMetrics getMetrics() {
+ return metrics;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUnavailableException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUnavailableException.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUnavailableException.java
new file mode 100644
index 0000000..4e6f8c8
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUnavailableException.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import java.io.IOException;
+
+/**
+ * Thrown when the state store is not reachable or available. Cached APIs and
+ * queries may succeed. Client should retry again later.
+ */
+public class StateStoreUnavailableException extends IOException {
+
+ private static final long serialVersionUID = 1L;
+
+ public StateStoreUnavailableException(String msg) {
+ super(msg);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUtils.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUtils.java
new file mode 100644
index 0000000..0a36619
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreUtils.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.Query;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Set of utility functions used to work with the State Store.
+ */
+public final class StateStoreUtils {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StateStoreUtils.class);
+
+
+ private StateStoreUtils() {
+ // Utility class
+ }
+
+ /**
+ * Get the base class for a record class. If we get an implementation of a
+ * record we will return the real parent record class.
+ *
+ * @param clazz Class of the data record to check.
+ * @return Base class for the record.
+ */
+ @SuppressWarnings("unchecked")
+ public static <T extends BaseRecord>
+ Class<? extends BaseRecord> getRecordClass(final Class<T> clazz) {
+
+ // We ignore the Impl classes and go to the super class
+ Class<? extends BaseRecord> actualClazz = clazz;
+ while (actualClazz.getSimpleName().endsWith("Impl")) {
+ actualClazz = (Class<? extends BaseRecord>) actualClazz.getSuperclass();
+ }
+
+ // Check if we went too far
+ if (actualClazz.equals(BaseRecord.class)) {
+ LOG.error("We went too far ({}) with {}", actualClazz, clazz);
+ actualClazz = clazz;
+ }
+ return actualClazz;
+ }
+
+ /**
+ * Get the base class for a record. If we get an implementation of a record we
+ * will return the real parent record class.
+ *
+ * @param record Record to check its main class.
+ * @return Base class for the record.
+ */
+ public static <T extends BaseRecord>
+ Class<? extends BaseRecord> getRecordClass(final T record) {
+ return getRecordClass(record.getClass());
+ }
+
+ /**
+ * Get the base class name for a record. If we get an implementation of a
+ * record we will return the real parent record class.
+ *
+ * @param clazz Class of the data record to check.
+ * @return Name of the base class for the record.
+ */
+ public static <T extends BaseRecord> String getRecordName(
+ final Class<T> clazz) {
+ return getRecordClass(clazz).getSimpleName();
+ }
+
+ /**
+ * Filters a list of records to find all records matching the query.
+ *
+ * @param query Map of field names and objects to use to filter results.
+ * @param records List of data records to filter.
+ * @return List of all records matching the query (or empty list if none
+ * match), null if the data set could not be filtered.
+ */
+ public static <T extends BaseRecord> List<T> filterMultiple(
+ final Query<T> query, final Iterable<T> records) {
+
+ List<T> matchingList = new ArrayList<>();
+ for (T record : records) {
+ if (query.matches(record)) {
+ matchingList.add(record);
+ }
+ }
+ return matchingList;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java
new file mode 100644
index 0000000..d595a97
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java
@@ -0,0 +1,203 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver;
+
+import java.net.InetAddress;
+import java.util.Collection;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.metrics.StateStoreMetrics;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreUnavailableException;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Driver class for an implementation of a {@link StateStoreService}
+ * provider. Driver implementations will extend this class and implement some of
+ * the default methods.
+ */
+public abstract class StateStoreDriver implements StateStoreRecordOperations {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StateStoreDriver.class);
+
+
+ /** State Store configuration. */
+ private Configuration conf;
+
+ /** Identifier for the driver. */
+ private String identifier;
+
+ /** State Store metrics. */
+ private StateStoreMetrics metrics;
+
+
+ /**
+ * Initialize the state store connection.
+ *
+ * @param config Configuration for the driver.
+ * @param id Identifier for the driver.
+ * @param records Records that are supported.
+ * @return If initialized and ready, false if failed to initialize driver.
+ */
+ public boolean init(final Configuration config, final String id,
+ final Collection<Class<? extends BaseRecord>> records,
+ final StateStoreMetrics stateStoreMetrics) {
+
+ this.conf = config;
+ this.identifier = id;
+ this.metrics = stateStoreMetrics;
+
+ if (this.identifier == null) {
+ LOG.warn("The identifier for the State Store connection is not set");
+ }
+
+ boolean success = initDriver();
+ if (!success) {
+ LOG.error("Cannot initialize driver for {}", getDriverName());
+ return false;
+ }
+
+ for (Class<? extends BaseRecord> cls : records) {
+ String recordString = StateStoreUtils.getRecordName(cls);
+ if (!initRecordStorage(recordString, cls)) {
+ LOG.error("Cannot initialize record store for {}", cls.getSimpleName());
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Get the State Store configuration.
+ *
+ * @return Configuration for the State Store.
+ */
+ protected Configuration getConf() {
+ return this.conf;
+ }
+
+ /**
+ * Gets a unique identifier for the running task/process. Typically the
+ * router address.
+ *
+ * @return Unique identifier for the running task.
+ */
+ public String getIdentifier() {
+ return this.identifier;
+ }
+
+ /**
+ * Get the metrics for the State Store.
+ *
+ * @return State Store metrics.
+ */
+ public StateStoreMetrics getMetrics() {
+ return this.metrics;
+ }
+
+ /**
+ * Prepare the driver to access data storage.
+ *
+ * @return True if the driver was successfully initialized. If false is
+ * returned, the state store will periodically attempt to
+ * re-initialize the driver and the router will remain in safe mode
+ * until the driver is initialized.
+ */
+ public abstract boolean initDriver();
+
+ /**
+ * Initialize storage for a single record class.
+ *
+ * @param className String reference of the record class to initialize,
+ * used to construct paths and file names for the record.
+ * Determined by configuration settings for the specific
+ * driver.
+ * @param clazz Record type corresponding to the provided name.
+ * @return True if successful, false otherwise.
+ */
+ public abstract <T extends BaseRecord> boolean initRecordStorage(
+ String className, Class<T> clazz);
+
+ /**
+ * Check if the driver is currently running and the data store connection is
+ * valid.
+ *
+ * @return True if the driver is initialized and the data store is ready.
+ */
+ public abstract boolean isDriverReady();
+
+ /**
+ * Check if the driver is ready to be used and throw an exception otherwise.
+ *
+ * @throws StateStoreUnavailableException If the driver is not ready.
+ */
+ public void verifyDriverReady() throws StateStoreUnavailableException {
+ if (!isDriverReady()) {
+ String driverName = getDriverName();
+ String hostname = getHostname();
+ throw new StateStoreUnavailableException("State Store driver " +
+ driverName + " in " + hostname + " is not ready.");
+ }
+ }
+
+ /**
+ * Close the State Store driver connection.
+ */
+ public abstract void close() throws Exception;
+
+ /**
+ * Returns the current time synchronization from the underlying store.
+ * Override for stores that supply a current date. The data store driver is
+ * responsible for maintaining the official synchronization time/date for all
+ * distributed components.
+ *
+ * @return Current time stamp, used for all synchronization dates.
+ */
+ public long getTime() {
+ return Time.now();
+ }
+
+ /**
+ * Get the name of the driver implementation for debugging.
+ *
+ * @return Name of the driver implementation.
+ */
+ private String getDriverName() {
+ return this.getClass().getSimpleName();
+ }
+
+ /**
+ * Get the host name of the machine running the driver for debugging.
+ *
+ * @return Host name of the machine running the driver.
+ */
+ private String getHostname() {
+ String hostname = "Unknown";
+ try {
+ hostname = InetAddress.getLocalHost().getHostName();
+ } catch (Exception e) {
+ LOG.error("Cannot get local address", e);
+ }
+ return hostname;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
new file mode 100644
index 0000000..443d46e
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.Query;
+import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
+import org.apache.hadoop.io.retry.AtMostOnce;
+import org.apache.hadoop.io.retry.Idempotent;
+
+/**
+ * Operations for a driver to manage records in the State Store.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface StateStoreRecordOperations {
+
+ /**
+ * Get all records of the requested record class from the data store. To use
+ * the default implementations in this class, getAll must return new instances
+ * of the records on each call. It is recommended to override the default
+ * implementations for better performance.
+ *
+ * @param clazz Class of record to fetch.
+ * @return List of all records that match the clazz.
+ * @throws IOException Throws exception if unable to query the data store.
+ */
+ @Idempotent
+ <T extends BaseRecord> QueryResult<T> get(Class<T> clazz) throws IOException;
+
+ /**
+ * Get a single record from the store that matches the query.
+ *
+ * @param clazz Class of record to fetch.
+ * @param query Query to filter results.
+ * @return A single record matching the query. Null if there are no matching
+ * records or more than one matching record in the store.
+ * @throws IOException If multiple records match or if the data store cannot
+ * be queried.
+ */
+ @Idempotent
+ <T extends BaseRecord> T get(Class<T> clazz, Query<T> query)
+ throws IOException;
+
+ /**
+ * Get multiple records from the store that match a query. This method
+ * assumes the underlying driver does not support filtering. If the driver
+ * supports filtering it should overwrite this method.
+ *
+ * @param clazz Class of record to fetch.
+ * @param query Query to filter results.
+ * @return Records of type clazz that match the query or empty list if none
+ * are found.
+ * @throws IOException Throws exception if unable to query the data store.
+ */
+ @Idempotent
+ <T extends BaseRecord> List<T> getMultiple(
+ Class<T> clazz, Query<T> query) throws IOException;
+
+ /**
+ * Creates a single record. Optionally updates an existing record with same
+ * primary key.
+ *
+ * @param record The record to insert or update.
+ * @param allowUpdate True if update of exiting record is allowed.
+ * @param errorIfExists True if an error should be returned when inserting
+ * an existing record. Only used if allowUpdate = false.
+ * @return True if the operation was successful.
+ *
+ * @throws IOException Throws exception if unable to query the data store.
+ */
+ @AtMostOnce
+ <T extends BaseRecord> boolean put(
+ T record, boolean allowUpdate, boolean errorIfExists) throws IOException;
+
+ /**
+ * Creates multiple records. Optionally updates existing records that have
+ * the same primary key.
+ *
+ * @param records List of data records to update or create. All records must
+ * be of class clazz.
+ * @param clazz Record class of records.
+ * @param allowUpdate True if update of exiting record is allowed.
+ * @param errorIfExists True if an error should be returned when inserting
+ * an existing record. Only used if allowUpdate = false.
+ * @return true if all operations were successful.
+ *
+ * @throws IOException Throws exception if unable to query the data store.
+ */
+ @AtMostOnce
+ <T extends BaseRecord> boolean putAll(
+ List<T> records, boolean allowUpdate, boolean errorIfExists)
+ throws IOException;
+
+ /**
+ * Remove a single record.
+ *
+ * @param record Record to be removed.
+ * @return true If the record was successfully removed. False if the record
+ * could not be removed or not stored.
+ * @throws IOException Throws exception if unable to query the data store.
+ */
+ @AtMostOnce
+ <T extends BaseRecord> boolean remove(T record) throws IOException;
+
+ /**
+ * Remove all records of this class from the store.
+ *
+ * @param clazz Class of records to remove.
+ * @return True if successful.
+ * @throws IOException Throws exception if unable to query the data store.
+ */
+ @AtMostOnce
+ <T extends BaseRecord> boolean removeAll(Class<T> clazz) throws IOException;
+
+ /**
+ * Remove multiple records of a specific class that match a query. Requires
+ * the getAll implementation to fetch fresh records on each call.
+ *
+ * @param query Query to filter what to remove.
+ * @return The number of records removed.
+ * @throws IOException Throws exception if unable to query the data store.
+ */
+ @AtMostOnce
+ <T extends BaseRecord> int remove(Class<T> clazz, Query<T> query)
+ throws IOException;
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreSerializer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreSerializer.java
new file mode 100644
index 0000000..666712f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreSerializer.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.driver;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.util.ReflectionUtils;
+
+/**
+ * Serializer to store and retrieve data in the State Store.
+ */
+public abstract class StateStoreSerializer {
+
+ /** Singleton for the serializer instance. */
+ private static StateStoreSerializer defaultSerializer;
+
+ /**
+ * Get the default serializer based.
+ * @return Singleton serializer.
+ */
+ public static StateStoreSerializer getSerializer() {
+ return getSerializer(null);
+ }
+
+ /**
+ * Get a serializer based on the provided configuration.
+ * @param conf Configuration. Default if null.
+ * @return Singleton serializer.
+ */
+ public static StateStoreSerializer getSerializer(Configuration conf) {
+ if (conf == null) {
+ synchronized (StateStoreSerializer.class) {
+ if (defaultSerializer == null) {
+ conf = new Configuration();
+ defaultSerializer = newSerializer(conf);
+ }
+ }
+ return defaultSerializer;
+ } else {
+ return newSerializer(conf);
+ }
+ }
+
+ private static StateStoreSerializer newSerializer(final Configuration conf) {
+ Class<? extends StateStoreSerializer> serializerName = conf.getClass(
+ RBFConfigKeys.FEDERATION_STORE_SERIALIZER_CLASS,
+ RBFConfigKeys.FEDERATION_STORE_SERIALIZER_CLASS_DEFAULT,
+ StateStoreSerializer.class);
+ return ReflectionUtils.newInstance(serializerName, conf);
+ }
+
+ /**
+ * Create a new record.
+ * @param clazz Class of the new record.
+ * @return New record.
+ */
+ public static <T> T newRecord(Class<T> clazz) {
+ return getSerializer(null).newRecordInstance(clazz);
+ }
+
+ /**
+ * Create a new record.
+ * @param clazz Class of the new record.
+ * @return New record.
+ */
+ public abstract <T> T newRecordInstance(Class<T> clazz);
+
+ /**
+ * Serialize a record into a byte array.
+ * @param record Record to serialize.
+ * @return Byte array with the serialized record.
+ */
+ public abstract byte[] serialize(BaseRecord record);
+
+ /**
+ * Serialize a record into a string.
+ * @param record Record to serialize.
+ * @return String with the serialized record.
+ */
+ public abstract String serializeString(BaseRecord record);
+
+ /**
+ * Deserialize a bytes array into a record.
+ * @param byteArray Byte array to deserialize.
+ * @param clazz Class of the record.
+ * @return New record.
+ * @throws IOException If it cannot deserialize the record.
+ */
+ public abstract <T extends BaseRecord> T deserialize(
+ byte[] byteArray, Class<T> clazz) throws IOException;
+
+ /**
+ * Deserialize a string into a record.
+ * @param data String with the data to deserialize.
+ * @param clazz Class of the record.
+ * @return New record.
+ * @throws IOException If it cannot deserialize the record.
+ */
+ public abstract <T extends BaseRecord> T deserialize(
+ String data, Class<T> clazz) throws IOException;
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
new file mode 100644
index 0000000..a80c3be
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
@@ -0,0 +1,766 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.metrics;
+
+import static org.apache.hadoop.util.Time.now;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import java.util.function.ToIntFunction;
+import java.util.function.ToLongFunction;
+import java.util.stream.Collectors;
+
+import javax.management.NotCompliantMBeanException;
+import javax.management.ObjectName;
+import javax.management.StandardMBean;
+
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.federation.router.Router;
+import org.apache.hadoop.hdfs.server.federation.router.RouterRpcServer;
+import org.apache.hadoop.hdfs.server.federation.store.MembershipStore;
+import org.apache.hadoop.hdfs.server.federation.store.MountTableStore;
+import org.apache.hadoop.hdfs.server.federation.store.RouterStore;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationsRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationsResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipStats;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+import org.apache.hadoop.hdfs.server.federation.store.records.StateStoreVersion;
+import org.apache.hadoop.metrics2.util.MBeans;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.VersionInfo;
+import org.codehaus.jettison.json.JSONObject;
+import org.eclipse.jetty.util.ajax.JSON;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Implementation of the Router metrics collector.
+ */
+public class FederationMetrics implements FederationMBean {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(FederationMetrics.class);
+
+ /** Format for a date. */
+ private static final String DATE_FORMAT = "yyyy/MM/dd HH:mm:ss";
+
+ /** Prevent holding the page from load too long. */
+ private static final long TIME_OUT = TimeUnit.SECONDS.toMillis(1);
+
+
+ /** Router interface. */
+ private final Router router;
+
+ /** FederationState JMX bean. */
+ private ObjectName beanName;
+
+ /** Resolve the namenode for each namespace. */
+ private final ActiveNamenodeResolver namenodeResolver;
+
+ /** State store. */
+ private final StateStoreService stateStore;
+ /** Membership state store. */
+ private MembershipStore membershipStore;
+ /** Mount table store. */
+ private MountTableStore mountTableStore;
+ /** Router state store. */
+ private RouterStore routerStore;
+
+
+ public FederationMetrics(Router router) throws IOException {
+ this.router = router;
+
+ try {
+ StandardMBean bean = new StandardMBean(this, FederationMBean.class);
+ this.beanName = MBeans.register("Router", "FederationState", bean);
+ LOG.info("Registered Router MBean: {}", this.beanName);
+ } catch (NotCompliantMBeanException e) {
+ throw new RuntimeException("Bad Router MBean setup", e);
+ }
+
+ // Resolve namenode for each nameservice
+ this.namenodeResolver = this.router.getNamenodeResolver();
+
+ // State store interfaces
+ this.stateStore = this.router.getStateStore();
+ if (this.stateStore == null) {
+ LOG.error("State store not available");
+ } else {
+ this.membershipStore = stateStore.getRegisteredRecordStore(
+ MembershipStore.class);
+ this.mountTableStore = stateStore.getRegisteredRecordStore(
+ MountTableStore.class);
+ this.routerStore = stateStore.getRegisteredRecordStore(
+ RouterStore.class);
+ }
+ }
+
+ /**
+ * Unregister the JMX beans.
+ */
+ public void close() {
+ if (this.beanName != null) {
+ MBeans.unregister(beanName);
+ }
+ }
+
+ @Override
+ public String getNamenodes() {
+ final Map<String, Map<String, Object>> info = new LinkedHashMap<>();
+ try {
+ // Get the values from the store
+ GetNamenodeRegistrationsRequest request =
+ GetNamenodeRegistrationsRequest.newInstance();
+ GetNamenodeRegistrationsResponse response =
+ membershipStore.getNamenodeRegistrations(request);
+
+ // Order the namenodes
+ final List<MembershipState> namenodes = response.getNamenodeMemberships();
+ if (namenodes == null || namenodes.size() == 0) {
+ return JSON.toString(info);
+ }
+ List<MembershipState> namenodesOrder = new ArrayList<>(namenodes);
+ Collections.sort(namenodesOrder, MembershipState.NAME_COMPARATOR);
+
+ // Dump namenodes information into JSON
+ for (MembershipState namenode : namenodesOrder) {
+ Map<String, Object> innerInfo = new HashMap<>();
+ Map<String, Object> map = getJson(namenode);
+ innerInfo.putAll(map);
+ long dateModified = namenode.getDateModified();
+ long lastHeartbeat = getSecondsSince(dateModified);
+ innerInfo.put("lastHeartbeat", lastHeartbeat);
+ MembershipStats stats = namenode.getStats();
+ long used = stats.getTotalSpace() - stats.getAvailableSpace();
+ innerInfo.put("used", used);
+ info.put(namenode.getNamenodeKey(),
+ Collections.unmodifiableMap(innerInfo));
+ }
+ } catch (IOException e) {
+ LOG.error("Enable to fetch json representation of namenodes {}",
+ e.getMessage());
+ return "{}";
+ }
+ return JSON.toString(info);
+ }
+
+ @Override
+ public String getNameservices() {
+ final Map<String, Map<String, Object>> info = new LinkedHashMap<>();
+ try {
+ final List<MembershipState> namenodes = getActiveNamenodeRegistrations();
+ List<MembershipState> namenodesOrder = new ArrayList<>(namenodes);
+ Collections.sort(namenodesOrder, MembershipState.NAME_COMPARATOR);
+
+ // Dump namenodes information into JSON
+ for (MembershipState namenode : namenodesOrder) {
+ Map<String, Object> innerInfo = new HashMap<>();
+ Map<String, Object> map = getJson(namenode);
+ innerInfo.putAll(map);
+ long dateModified = namenode.getDateModified();
+ long lastHeartbeat = getSecondsSince(dateModified);
+ innerInfo.put("lastHeartbeat", lastHeartbeat);
+ MembershipStats stats = namenode.getStats();
+ long used = stats.getTotalSpace() - stats.getAvailableSpace();
+ innerInfo.put("used", used);
+ info.put(namenode.getNamenodeKey(),
+ Collections.unmodifiableMap(innerInfo));
+ }
+ } catch (IOException e) {
+ LOG.error("Cannot retrieve nameservices for JMX: {}", e.getMessage());
+ return "{}";
+ }
+ return JSON.toString(info);
+ }
+
+ @Override
+ public String getMountTable() {
+ final List<Map<String, Object>> info = new LinkedList<>();
+
+ try {
+ // Get all the mount points in order
+ GetMountTableEntriesRequest request =
+ GetMountTableEntriesRequest.newInstance("/");
+ GetMountTableEntriesResponse response =
+ mountTableStore.getMountTableEntries(request);
+ final List<MountTable> mounts = response.getEntries();
+ List<MountTable> orderedMounts = new ArrayList<>(mounts);
+ Collections.sort(orderedMounts, MountTable.SOURCE_COMPARATOR);
+
+ // Dump mount table entries information into JSON
+ for (MountTable entry : orderedMounts) {
+ // Sumarize destinations
+ Set<String> nameservices = new LinkedHashSet<>();
+ Set<String> paths = new LinkedHashSet<>();
+ for (RemoteLocation location : entry.getDestinations()) {
+ nameservices.add(location.getNameserviceId());
+ paths.add(location.getDest());
+ }
+
+ Map<String, Object> map = getJson(entry);
+ // We add some values with a cleaner format
+ map.put("dateCreated", getDateString(entry.getDateCreated()));
+ map.put("dateModified", getDateString(entry.getDateModified()));
+
+ Map<String, Object> innerInfo = new HashMap<>();
+ innerInfo.putAll(map);
+ innerInfo.put("nameserviceId", StringUtils.join(",", nameservices));
+ innerInfo.put("path", StringUtils.join(",", paths));
+ if (nameservices.size() > 1) {
+ innerInfo.put("order", entry.getDestOrder().toString());
+ } else {
+ innerInfo.put("order", "");
+ }
+ innerInfo.put("readonly", entry.isReadOnly());
+ info.add(Collections.unmodifiableMap(innerInfo));
+ }
+ } catch (IOException e) {
+ LOG.error(
+ "Cannot generate JSON of mount table from store: {}", e.getMessage());
+ return "[]";
+ }
+ return JSON.toString(info);
+ }
+
+ @Override
+ public String getRouters() {
+ final Map<String, Map<String, Object>> info = new LinkedHashMap<>();
+ try {
+ // Get all the routers in order
+ GetRouterRegistrationsRequest request =
+ GetRouterRegistrationsRequest.newInstance();
+ GetRouterRegistrationsResponse response =
+ routerStore.getRouterRegistrations(request);
+ final List<RouterState> routers = response.getRouters();
+ List<RouterState> routersOrder = new ArrayList<>(routers);
+ Collections.sort(routersOrder);
+
+ // Dump router information into JSON
+ for (RouterState record : routersOrder) {
+ Map<String, Object> innerInfo = new HashMap<>();
+ Map<String, Object> map = getJson(record);
+ innerInfo.putAll(map);
+ long dateModified = record.getDateModified();
+ long lastHeartbeat = getSecondsSince(dateModified);
+ innerInfo.put("lastHeartbeat", lastHeartbeat);
+
+ StateStoreVersion stateStoreVersion = record.getStateStoreVersion();
+ if (stateStoreVersion == null) {
+ LOG.error("Cannot get State Store versions");
+ } else {
+ setStateStoreVersions(innerInfo, stateStoreVersion);
+ }
+
+ info.put(record.getPrimaryKey(),
+ Collections.unmodifiableMap(innerInfo));
+ }
+ } catch (IOException e) {
+ LOG.error("Cannot get Routers JSON from the State Store", e);
+ return "{}";
+ }
+ return JSON.toString(info);
+ }
+
+ /**
+ * Populate the map with the State Store versions.
+ *
+ * @param innerInfo Map with the information.
+ * @param version State Store versions.
+ */
+ private static void setStateStoreVersions(
+ Map<String, Object> map, StateStoreVersion version) {
+
+ long membershipVersion = version.getMembershipVersion();
+ String lastMembershipUpdate = getDateString(membershipVersion);
+ map.put("lastMembershipUpdate", lastMembershipUpdate);
+
+ long mountTableVersion = version.getMountTableVersion();
+ String lastMountTableDate = getDateString(mountTableVersion);
+ map.put("lastMountTableUpdate", lastMountTableDate);
+ }
+
+ @Override
+ public long getTotalCapacity() {
+ return getNameserviceAggregatedLong(MembershipStats::getTotalSpace);
+ }
+
+ @Override
+ public long getRemainingCapacity() {
+ return getNameserviceAggregatedLong(MembershipStats::getAvailableSpace);
+ }
+
+ @Override
+ public long getProvidedSpace() {
+ return getNameserviceAggregatedLong(MembershipStats::getProvidedSpace);
+ }
+
+ @Override
+ public long getUsedCapacity() {
+ return getTotalCapacity() - getRemainingCapacity();
+ }
+
+ @Override
+ public int getNumNameservices() {
+ try {
+ Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ return nss.size();
+ } catch (IOException e) {
+ LOG.error(
+ "Cannot fetch number of expired registrations from the store: {}",
+ e.getMessage());
+ return 0;
+ }
+ }
+
+ @Override
+ public int getNumNamenodes() {
+ try {
+ GetNamenodeRegistrationsRequest request =
+ GetNamenodeRegistrationsRequest.newInstance();
+ GetNamenodeRegistrationsResponse response =
+ membershipStore.getNamenodeRegistrations(request);
+ List<MembershipState> memberships = response.getNamenodeMemberships();
+ return memberships.size();
+ } catch (IOException e) {
+ LOG.error("Cannot retrieve numNamenodes for JMX: {}", e.getMessage());
+ return 0;
+ }
+ }
+
+ @Override
+ public int getNumExpiredNamenodes() {
+ try {
+ GetNamenodeRegistrationsRequest request =
+ GetNamenodeRegistrationsRequest.newInstance();
+ GetNamenodeRegistrationsResponse response =
+ membershipStore.getExpiredNamenodeRegistrations(request);
+ List<MembershipState> expiredMemberships =
+ response.getNamenodeMemberships();
+ return expiredMemberships.size();
+ } catch (IOException e) {
+ LOG.error(
+ "Cannot retrieve numExpiredNamenodes for JMX: {}", e.getMessage());
+ return 0;
+ }
+ }
+
+ @Override
+ public int getNumLiveNodes() {
+ return getNameserviceAggregatedInt(
+ MembershipStats::getNumOfActiveDatanodes);
+ }
+
+ @Override
+ public int getNumDeadNodes() {
+ return getNameserviceAggregatedInt(MembershipStats::getNumOfDeadDatanodes);
+ }
+
+ @Override
+ public int getNumDecommissioningNodes() {
+ return getNameserviceAggregatedInt(
+ MembershipStats::getNumOfDecommissioningDatanodes);
+ }
+
+ @Override
+ public int getNumDecomLiveNodes() {
+ return getNameserviceAggregatedInt(
+ MembershipStats::getNumOfDecomActiveDatanodes);
+ }
+
+ @Override
+ public int getNumDecomDeadNodes() {
+ return getNameserviceAggregatedInt(
+ MembershipStats::getNumOfDecomDeadDatanodes);
+ }
+
+ @Override // NameNodeMXBean
+ public String getNodeUsage() {
+ float median = 0;
+ float max = 0;
+ float min = 0;
+ float dev = 0;
+
+ final Map<String, Map<String, Object>> info = new HashMap<>();
+ try {
+ RouterRpcServer rpcServer = this.router.getRpcServer();
+ DatanodeInfo[] live = rpcServer.getDatanodeReport(
+ DatanodeReportType.LIVE, TIME_OUT);
+
+ if (live.length > 0) {
+ float totalDfsUsed = 0;
+ float[] usages = new float[live.length];
+ int i = 0;
+ for (DatanodeInfo dn : live) {
+ usages[i++] = dn.getDfsUsedPercent();
+ totalDfsUsed += dn.getDfsUsedPercent();
+ }
+ totalDfsUsed /= live.length;
+ Arrays.sort(usages);
+ median = usages[usages.length / 2];
+ max = usages[usages.length - 1];
+ min = usages[0];
+
+ for (i = 0; i < usages.length; i++) {
+ dev += (usages[i] - totalDfsUsed) * (usages[i] - totalDfsUsed);
+ }
+ dev = (float) Math.sqrt(dev / usages.length);
+ }
+ } catch (IOException e) {
+ LOG.info("Cannot get the live nodes: {}", e.getMessage());
+ }
+
+ final Map<String, Object> innerInfo = new HashMap<>();
+ innerInfo.put("min", StringUtils.format("%.2f%%", min));
+ innerInfo.put("median", StringUtils.format("%.2f%%", median));
+ innerInfo.put("max", StringUtils.format("%.2f%%", max));
+ innerInfo.put("stdDev", StringUtils.format("%.2f%%", dev));
+ info.put("nodeUsage", innerInfo);
+
+ return JSON.toString(info);
+ }
+
+ @Override
+ public long getNumBlocks() {
+ return getNameserviceAggregatedLong(MembershipStats::getNumOfBlocks);
+ }
+
+ @Override
+ public long getNumOfMissingBlocks() {
+ return getNameserviceAggregatedLong(MembershipStats::getNumOfBlocksMissing);
+ }
+
+ @Override
+ public long getNumOfBlocksPendingReplication() {
+ return getNameserviceAggregatedLong(
+ MembershipStats::getNumOfBlocksPendingReplication);
+ }
+
+ @Override
+ public long getNumOfBlocksUnderReplicated() {
+ return getNameserviceAggregatedLong(
+ MembershipStats::getNumOfBlocksUnderReplicated);
+ }
+
+ @Override
+ public long getNumOfBlocksPendingDeletion() {
+ return getNameserviceAggregatedLong(
+ MembershipStats::getNumOfBlocksPendingDeletion);
+ }
+
+ @Override
+ public long getNumFiles() {
+ return getNameserviceAggregatedLong(MembershipStats::getNumOfFiles);
+ }
+
+ @Override
+ public String getRouterStarted() {
+ long startTime = this.router.getStartTime();
+ return new Date(startTime).toString();
+ }
+
+ @Override
+ public String getVersion() {
+ return VersionInfo.getVersion() + ", r" + VersionInfo.getRevision();
+ }
+
+ @Override
+ public String getCompiledDate() {
+ return VersionInfo.getDate();
+ }
+
+ @Override
+ public String getCompileInfo() {
+ return VersionInfo.getDate() + " by " + VersionInfo.getUser() + " from "
+ + VersionInfo.getBranch();
+ }
+
+ @Override
+ public String getHostAndPort() {
+ InetSocketAddress address = this.router.getHttpServerAddress();
+ if (address != null) {
+ try {
+ String hostname = InetAddress.getLocalHost().getHostName();
+ int port = address.getPort();
+ return hostname + ":" + port;
+ } catch (UnknownHostException ignored) { }
+ }
+ return "Unknown";
+ }
+
+ @Override
+ public String getRouterId() {
+ return this.router.getRouterId();
+ }
+
+ @Override
+ public String getClusterId() {
+ try {
+ Collection<String> clusterIds =
+ getNamespaceInfo(FederationNamespaceInfo::getClusterId);
+ return clusterIds.toString();
+ } catch (IOException e) {
+ LOG.error("Cannot fetch cluster ID metrics: {}", e.getMessage());
+ return "";
+ }
+ }
+
+ @Override
+ public String getBlockPoolId() {
+ try {
+ Collection<String> blockpoolIds =
+ getNamespaceInfo(FederationNamespaceInfo::getBlockPoolId);
+ return blockpoolIds.toString();
+ } catch (IOException e) {
+ LOG.error("Cannot fetch block pool ID metrics: {}", e.getMessage());
+ return "";
+ }
+ }
+
+ @Override
+ public String getRouterStatus() {
+ return "RUNNING";
+ }
+
+ /**
+ * Build a set of unique values found in all namespaces.
+ *
+ * @param f Method reference of the appropriate FederationNamespaceInfo
+ * getter function
+ * @return Set of unique string values found in all discovered namespaces.
+ * @throws IOException if the query could not be executed.
+ */
+ private Collection<String> getNamespaceInfo(
+ Function<FederationNamespaceInfo, String> f) throws IOException {
+ GetNamespaceInfoRequest request = GetNamespaceInfoRequest.newInstance();
+ GetNamespaceInfoResponse response =
+ membershipStore.getNamespaceInfo(request);
+ return response.getNamespaceInfo().stream()
+ .map(f)
+ .collect(Collectors.toSet());
+ }
+
+ /**
+ * Get the aggregated value for a method for all nameservices.
+ * @param f Method reference
+ * @return Aggregated integer.
+ */
+ private int getNameserviceAggregatedInt(ToIntFunction<MembershipStats> f) {
+ try {
+ return getActiveNamenodeRegistrations().stream()
+ .map(MembershipState::getStats)
+ .collect(Collectors.summingInt(f));
+ } catch (IOException e) {
+ LOG.error("Unable to extract metrics: {}", e.getMessage());
+ return 0;
+ }
+ }
+
+ /**
+ * Get the aggregated value for a method for all nameservices.
+ * @param f Method reference
+ * @return Aggregated long.
+ */
+ private long getNameserviceAggregatedLong(ToLongFunction<MembershipStats> f) {
+ try {
+ return getActiveNamenodeRegistrations().stream()
+ .map(MembershipState::getStats)
+ .collect(Collectors.summingLong(f));
+ } catch (IOException e) {
+ LOG.error("Unable to extract metrics: {}", e.getMessage());
+ return 0;
+ }
+ }
+
+ /**
+ * Fetches the most active namenode memberships for all known nameservices.
+ * The fetched membership may not or may not be active. Excludes expired
+ * memberships.
+ * @throws IOException if the query could not be performed.
+ * @return List of the most active NNs from each known nameservice.
+ */
+ private List<MembershipState> getActiveNamenodeRegistrations()
+ throws IOException {
+
+ List<MembershipState> resultList = new ArrayList<>();
+ GetNamespaceInfoRequest request = GetNamespaceInfoRequest.newInstance();
+ GetNamespaceInfoResponse response =
+ membershipStore.getNamespaceInfo(request);
+ for (FederationNamespaceInfo nsInfo : response.getNamespaceInfo()) {
+ // Fetch the most recent namenode registration
+ String nsId = nsInfo.getNameserviceId();
+ List<? extends FederationNamenodeContext> nns =
+ namenodeResolver.getNamenodesForNameserviceId(nsId);
+ if (nns != null) {
+ FederationNamenodeContext nn = nns.get(0);
+ if (nn != null && nn instanceof MembershipState) {
+ resultList.add((MembershipState) nn);
+ }
+ }
+ }
+ return resultList;
+ }
+
+ /**
+ * Get time as a date string.
+ * @param time Seconds since 1970.
+ * @return String representing the date.
+ */
+ @VisibleForTesting
+ static String getDateString(long time) {
+ if (time <= 0) {
+ return "-";
+ }
+ Date date = new Date(time);
+
+ SimpleDateFormat sdf = new SimpleDateFormat(DATE_FORMAT);
+ return sdf.format(date);
+ }
+
+ /**
+ * Get the number of seconds passed since a date.
+ *
+ * @param timeMs to use as a reference.
+ * @return Seconds since the date.
+ */
+ private static long getSecondsSince(long timeMs) {
+ if (timeMs < 0) {
+ return -1;
+ }
+ return (now() - timeMs) / 1000;
+ }
+
+ /**
+ * Get JSON for this record.
+ *
+ * @return Map representing the data for the JSON representation.
+ */
+ private static Map<String, Object> getJson(BaseRecord record) {
+ Map<String, Object> json = new HashMap<>();
+ Map<String, Class<?>> fields = getFields(record);
+
+ for (String fieldName : fields.keySet()) {
+ if (!fieldName.equalsIgnoreCase("proto")) {
+ try {
+ Object value = getField(record, fieldName);
+ if (value instanceof BaseRecord) {
+ BaseRecord recordField = (BaseRecord) value;
+ json.putAll(getJson(recordField));
+ } else {
+ json.put(fieldName, value == null ? JSONObject.NULL : value);
+ }
+ } catch (Exception e) {
+ throw new IllegalArgumentException(
+ "Cannot serialize field " + fieldName + " into JSON");
+ }
+ }
+ }
+ return json;
+ }
+
+ /**
+ * Returns all serializable fields in the object.
+ *
+ * @return Map with the fields.
+ */
+ private static Map<String, Class<?>> getFields(BaseRecord record) {
+ Map<String, Class<?>> getters = new HashMap<>();
+ for (Method m : record.getClass().getDeclaredMethods()) {
+ if (m.getName().startsWith("get")) {
+ try {
+ Class<?> type = m.getReturnType();
+ char[] c = m.getName().substring(3).toCharArray();
+ c[0] = Character.toLowerCase(c[0]);
+ String key = new String(c);
+ getters.put(key, type);
+ } catch (Exception e) {
+ LOG.error("Cannot execute getter {} on {}", m.getName(), record);
+ }
+ }
+ }
+ return getters;
+ }
+
+ /**
+ * Fetches the value for a field name.
+ *
+ * @param fieldName the legacy name of the field.
+ * @return The field data or null if not found.
+ */
+ private static Object getField(BaseRecord record, String fieldName) {
+ Object result = null;
+ Method m = locateGetter(record, fieldName);
+ if (m != null) {
+ try {
+ result = m.invoke(record);
+ } catch (Exception e) {
+ LOG.error("Cannot get field {} on {}", fieldName, record);
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Finds the appropriate getter for a field name.
+ *
+ * @param fieldName The legacy name of the field.
+ * @return The matching getter or null if not found.
+ */
+ private static Method locateGetter(BaseRecord record, String fieldName) {
+ for (Method m : record.getClass().getMethods()) {
+ if (m.getName().equalsIgnoreCase("get" + fieldName)) {
+ return m;
+ }
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java
new file mode 100644
index 0000000..3e031fe
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.metrics;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * JMX interface for the RPC server.
+ * TODO use the default RPC MBean.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface FederationRPCMBean {
+
+ long getProxyOps();
+
+ double getProxyAvg();
+
+ long getProcessingOps();
+
+ double getProcessingAvg();
+
+ long getProxyOpFailureCommunicate();
+
+ long getProxyOpFailureStandby();
+
+ long getProxyOpNotImplemented();
+
+ long getProxyOpRetries();
+
+ long getRouterFailureStateStoreOps();
+
+ long getRouterFailureReadOnlyOps();
+
+ long getRouterFailureLockedOps();
+
+ long getRouterFailureSafemodeOps();
+
+ int getRpcServerCallQueue();
+
+ /**
+ * Get the number of RPC connections between the clients and the Router.
+ * @return Number of RPC connections between the clients and the Router.
+ */
+ int getRpcServerNumOpenConnections();
+
+ /**
+ * Get the number of RPC connections between the Router and the NNs.
+ * @return Number of RPC connections between the Router and the NNs.
+ */
+ int getRpcClientNumConnections();
+
+ /**
+ * Get the number of active RPC connections between the Router and the NNs.
+ * @return Number of active RPC connections between the Router and the NNs.
+ */
+ int getRpcClientNumActiveConnections();
+
+ /**
+ * Get the number of RPC connections to be created.
+ * @return Number of RPC connections to be created.
+ */
+ int getRpcClientNumCreatingConnections();
+
+ /**
+ * Get the number of connection pools between the Router and a NNs.
+ * @return Number of connection pools between the Router and a NNs.
+ */
+ int getRpcClientNumConnectionPools();
+
+ /**
+ * JSON representation of the RPC connections from the Router to the NNs.
+ * @return JSON string representation.
+ */
+ String getRpcClientConnections();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java
new file mode 100644
index 0000000..94d3383
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.metrics;
+
+import static org.apache.hadoop.metrics2.impl.MsInfo.ProcessName;
+import static org.apache.hadoop.metrics2.impl.MsInfo.SessionId;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.router.RouterRpcServer;
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableRate;
+
+/**
+ * Implementation of the RPC metrics collector.
+ */
+@Metrics(name = "RouterRPCActivity", about = "Router RPC Activity",
+ context = "dfs")
+public class FederationRPCMetrics implements FederationRPCMBean {
+
+ private final MetricsRegistry registry = new MetricsRegistry("router");
+
+ private RouterRpcServer rpcServer;
+
+ @Metric("Time for the router to process an operation internally")
+ private MutableRate processing;
+ @Metric("Number of operations the Router processed internally")
+ private MutableCounterLong processingOp;
+ @Metric("Time for the Router to proxy an operation to the Namenodes")
+ private MutableRate proxy;
+ @Metric("Number of operations the Router proxied to a Namenode")
+ private MutableCounterLong proxyOp;
+
+ @Metric("Number of operations to fail to reach NN")
+ private MutableCounterLong proxyOpFailureStandby;
+ @Metric("Number of operations to hit a standby NN")
+ private MutableCounterLong proxyOpFailureCommunicate;
+ @Metric("Number of operations not implemented")
+ private MutableCounterLong proxyOpNotImplemented;
+ @Metric("Number of operation retries")
+ private MutableCounterLong proxyOpRetries;
+
+ @Metric("Failed requests due to State Store unavailable")
+ private MutableCounterLong routerFailureStateStore;
+ @Metric("Failed requests due to read only mount point")
+ private MutableCounterLong routerFailureReadOnly;
+ @Metric("Failed requests due to locked path")
+ private MutableCounterLong routerFailureLocked;
+ @Metric("Failed requests due to safe mode")
+ private MutableCounterLong routerFailureSafemode;
+
+ public FederationRPCMetrics(Configuration conf, RouterRpcServer rpcServer) {
+ this.rpcServer = rpcServer;
+
+ registry.tag(SessionId, "RouterRPCSession");
+ registry.tag(ProcessName, "Router");
+ }
+
+ public static FederationRPCMetrics create(Configuration conf,
+ RouterRpcServer rpcServer) {
+ MetricsSystem ms = DefaultMetricsSystem.instance();
+ return ms.register(FederationRPCMetrics.class.getName(),
+ "HDFS Federation RPC Metrics",
+ new FederationRPCMetrics(conf, rpcServer));
+ }
+
+ /**
+ * Convert nanoseconds to milliseconds.
+ * @param ns Time in nanoseconds.
+ * @return Time in milliseconds.
+ */
+ private static double toMs(double ns) {
+ return ns / 1000000;
+ }
+
+ /**
+ * Reset the metrics system.
+ */
+ public static void reset() {
+ MetricsSystem ms = DefaultMetricsSystem.instance();
+ ms.unregisterSource(FederationRPCMetrics.class.getName());
+ }
+
+ public void incrProxyOpFailureStandby() {
+ proxyOpFailureStandby.incr();
+ }
+
+ @Override
+ public long getProxyOpFailureStandby() {
+ return proxyOpFailureStandby.value();
+ }
+
+ public void incrProxyOpFailureCommunicate() {
+ proxyOpFailureCommunicate.incr();
+ }
+
+ @Override
+ public long getProxyOpFailureCommunicate() {
+ return proxyOpFailureCommunicate.value();
+ }
+
+
+ public void incrProxyOpNotImplemented() {
+ proxyOpNotImplemented.incr();
+ }
+
+ @Override
+ public long getProxyOpNotImplemented() {
+ return proxyOpNotImplemented.value();
+ }
+
+ public void incrProxyOpRetries() {
+ proxyOpRetries.incr();
+ }
+
+ @Override
+ public long getProxyOpRetries() {
+ return proxyOpRetries.value();
+ }
+
+ public void incrRouterFailureStateStore() {
+ routerFailureStateStore.incr();
+ }
+
+ @Override
+ public long getRouterFailureStateStoreOps() {
+ return routerFailureStateStore.value();
+ }
+
+ public void incrRouterFailureSafemode() {
+ routerFailureSafemode.incr();
+ }
+
+ @Override
+ public long getRouterFailureSafemodeOps() {
+ return routerFailureSafemode.value();
+ }
+
+ public void incrRouterFailureReadOnly() {
+ routerFailureReadOnly.incr();
+ }
+
+ @Override
+ public long getRouterFailureReadOnlyOps() {
+ return routerFailureReadOnly.value();
+ }
+
+ public void incrRouterFailureLocked() {
+ routerFailureLocked.incr();
+ }
+
+ @Override
+ public long getRouterFailureLockedOps() {
+ return routerFailureLocked.value();
+ }
+
+ @Override
+ public int getRpcServerCallQueue() {
+ return rpcServer.getServer().getCallQueueLen();
+ }
+
+ @Override
+ public int getRpcServerNumOpenConnections() {
+ return rpcServer.getServer().getNumOpenConnections();
+ }
+
+ @Override
+ public int getRpcClientNumConnections() {
+ return rpcServer.getRPCClient().getNumConnections();
+ }
+
+ @Override
+ public int getRpcClientNumActiveConnections() {
+ return rpcServer.getRPCClient().getNumActiveConnections();
+ }
+
+ @Override
+ public int getRpcClientNumCreatingConnections() {
+ return rpcServer.getRPCClient().getNumCreatingConnections();
+ }
+
+ @Override
+ public int getRpcClientNumConnectionPools() {
+ return rpcServer.getRPCClient().getNumConnectionPools();
+ }
+
+ @Override
+ public String getRpcClientConnections() {
+ return rpcServer.getRPCClient().getJSON();
+ }
+
+ /**
+ * Add the time to proxy an operation from the moment the Router sends it to
+ * the Namenode until it replied.
+ * @param time Proxy time of an operation in nanoseconds.
+ */
+ public void addProxyTime(long time) {
+ proxy.add(time);
+ proxyOp.incr();
+ }
+
+ @Override
+ public double getProxyAvg() {
+ return toMs(proxy.lastStat().mean());
+ }
+
+ @Override
+ public long getProxyOps() {
+ return proxyOp.value();
+ }
+
+ /**
+ * Add the time to process a request in the Router from the time we receive
+ * the call until we send it to the Namenode.
+ * @param time Process time of an operation in nanoseconds.
+ */
+ public void addProcessingTime(long time) {
+ processing.add(time);
+ processingOp.incr();
+ }
+
+ @Override
+ public double getProcessingAvg() {
+ return toMs(processing.lastStat().mean());
+ }
+
+ @Override
+ public long getProcessingOps() {
+ return processingOp.value();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCPerformanceMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCPerformanceMonitor.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCPerformanceMonitor.java
new file mode 100644
index 0000000..547ebb5
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCPerformanceMonitor.java
@@ -0,0 +1,221 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.metrics;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+
+import javax.management.NotCompliantMBeanException;
+import javax.management.ObjectName;
+import javax.management.StandardMBean;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.router.RouterRpcMonitor;
+import org.apache.hadoop.hdfs.server.federation.router.RouterRpcServer;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.metrics2.util.MBeans;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+/**
+ * Customizable RPC performance monitor. Receives events from the RPC server
+ * and aggregates them via JMX.
+ */
+public class FederationRPCPerformanceMonitor implements RouterRpcMonitor {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(FederationRPCPerformanceMonitor.class);
+
+
+ /** Time for an operation to be received in the Router. */
+ private static final ThreadLocal<Long> START_TIME = new ThreadLocal<>();
+ /** Time for an operation to be send to the Namenode. */
+ private static final ThreadLocal<Long> PROXY_TIME = new ThreadLocal<>();
+
+ /** Configuration for the performance monitor. */
+ private Configuration conf;
+ /** RPC server for the Router. */
+ private RouterRpcServer server;
+ /** State Store. */
+ private StateStoreService store;
+
+ /** JMX interface to monitor the RPC metrics. */
+ private FederationRPCMetrics metrics;
+ private ObjectName registeredBean;
+
+ /** Thread pool for logging stats. */
+ private ExecutorService executor;
+
+
+ @Override
+ public void init(Configuration configuration, RouterRpcServer rpcServer,
+ StateStoreService stateStore) {
+
+ this.conf = configuration;
+ this.server = rpcServer;
+ this.store = stateStore;
+
+ // Create metrics
+ this.metrics = FederationRPCMetrics.create(conf, server);
+
+ // Create thread pool
+ ThreadFactory threadFactory = new ThreadFactoryBuilder()
+ .setNameFormat("Federation RPC Performance Monitor-%d").build();
+ this.executor = Executors.newFixedThreadPool(1, threadFactory);
+
+ // Adding JMX interface
+ try {
+ StandardMBean bean =
+ new StandardMBean(this.metrics, FederationRPCMBean.class);
+ registeredBean = MBeans.register("Router", "FederationRPC", bean);
+ LOG.info("Registered FederationRPCMBean: {}", registeredBean);
+ } catch (NotCompliantMBeanException e) {
+ throw new RuntimeException("Bad FederationRPCMBean setup", e);
+ }
+ }
+
+ @Override
+ public void close() {
+ if (registeredBean != null) {
+ MBeans.unregister(registeredBean);
+ registeredBean = null;
+ }
+ if (this.executor != null) {
+ this.executor.shutdown();
+ }
+ }
+
+ /**
+ * Resets all RPC service performance counters to their defaults.
+ */
+ public void resetPerfCounters() {
+ if (registeredBean != null) {
+ MBeans.unregister(registeredBean);
+ registeredBean = null;
+ }
+ if (metrics != null) {
+ FederationRPCMetrics.reset();
+ metrics = null;
+ }
+ init(conf, server, store);
+ }
+
+ @Override
+ public void startOp() {
+ START_TIME.set(this.getNow());
+ }
+
+ @Override
+ public long proxyOp() {
+ PROXY_TIME.set(this.getNow());
+ long processingTime = getProcessingTime();
+ if (processingTime >= 0) {
+ metrics.addProcessingTime(processingTime);
+ }
+ return Thread.currentThread().getId();
+ }
+
+ @Override
+ public void proxyOpComplete(boolean success) {
+ if (success) {
+ long proxyTime = getProxyTime();
+ if (proxyTime >= 0) {
+ metrics.addProxyTime(proxyTime);
+ }
+ }
+ }
+
+ @Override
+ public void proxyOpFailureStandby() {
+ metrics.incrProxyOpFailureStandby();
+ }
+
+ @Override
+ public void proxyOpFailureCommunicate() {
+ metrics.incrProxyOpFailureCommunicate();
+ }
+
+ @Override
+ public void proxyOpNotImplemented() {
+ metrics.incrProxyOpNotImplemented();
+ }
+
+ @Override
+ public void proxyOpRetries() {
+ metrics.incrProxyOpRetries();
+ }
+
+ @Override
+ public void routerFailureStateStore() {
+ metrics.incrRouterFailureStateStore();
+ }
+
+ @Override
+ public void routerFailureSafemode() {
+ metrics.incrRouterFailureSafemode();
+ }
+
+ @Override
+ public void routerFailureReadOnly() {
+ metrics.incrRouterFailureReadOnly();
+ }
+
+ @Override
+ public void routerFailureLocked() {
+ metrics.incrRouterFailureLocked();
+ }
+
+ /**
+ * Get current time.
+ * @return Current time in nanoseconds.
+ */
+ private long getNow() {
+ return System.nanoTime();
+ }
+
+ /**
+ * Get time between we receiving the operation and sending it to the Namenode.
+ * @return Processing time in nanoseconds.
+ */
+ private long getProcessingTime() {
+ if (START_TIME.get() != null && START_TIME.get() > 0 &&
+ PROXY_TIME.get() != null && PROXY_TIME.get() > 0) {
+ return PROXY_TIME.get() - START_TIME.get();
+ }
+ return -1;
+ }
+
+ /**
+ * Get time between now and when the operation was forwarded to the Namenode.
+ * @return Current proxy time in nanoseconds.
+ */
+ private long getProxyTime() {
+ if (PROXY_TIME.get() != null && PROXY_TIME.get() > 0) {
+ return getNow() - PROXY_TIME.get();
+ }
+ return -1;
+ }
+
+ @Override
+ public FederationRPCMetrics getRPCMetrics() {
+ return this.metrics;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
new file mode 100644
index 0000000..c4e5b5b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
@@ -0,0 +1,634 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.metrics;
+
+import static org.apache.hadoop.util.Time.now;
+
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import javax.management.NotCompliantMBeanException;
+import javax.management.ObjectName;
+import javax.management.StandardMBean;
+
+import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.hdfs.DFSUtilClient;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
+import org.apache.hadoop.hdfs.server.federation.router.Router;
+import org.apache.hadoop.hdfs.server.federation.router.RouterRpcServer;
+import org.apache.hadoop.hdfs.server.federation.store.MembershipStore;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoResponse;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeMXBean;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeStatusMXBean;
+import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean;
+import org.apache.hadoop.ipc.StandbyException;
+import org.apache.hadoop.metrics2.util.MBeans;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.util.VersionInfo;
+import org.eclipse.jetty.util.ajax.JSON;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Expose the Namenode metrics as the Router was one.
+ */
+public class NamenodeBeanMetrics
+ implements FSNamesystemMBean, NameNodeMXBean, NameNodeStatusMXBean {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NamenodeBeanMetrics.class);
+
+ private final Router router;
+
+ /** FSNamesystem bean. */
+ private ObjectName fsBeanName;
+ /** FSNamesystemState bean. */
+ private ObjectName fsStateBeanName;
+ /** NameNodeInfo bean. */
+ private ObjectName nnInfoBeanName;
+ /** NameNodeStatus bean. */
+ private ObjectName nnStatusBeanName;
+
+
+ public NamenodeBeanMetrics(Router router) {
+ this.router = router;
+
+ try {
+ // TODO this needs to be done with the Metrics from FSNamesystem
+ StandardMBean bean = new StandardMBean(this, FSNamesystemMBean.class);
+ this.fsBeanName = MBeans.register("NameNode", "FSNamesystem", bean);
+ LOG.info("Registered FSNamesystem MBean: {}", this.fsBeanName);
+ } catch (NotCompliantMBeanException e) {
+ throw new RuntimeException("Bad FSNamesystem MBean setup", e);
+ }
+
+ try {
+ StandardMBean bean = new StandardMBean(this, FSNamesystemMBean.class);
+ this.fsStateBeanName =
+ MBeans.register("NameNode", "FSNamesystemState", bean);
+ LOG.info("Registered FSNamesystemState MBean: {}", this.fsStateBeanName);
+ } catch (NotCompliantMBeanException e) {
+ throw new RuntimeException("Bad FSNamesystemState MBean setup", e);
+ }
+
+ try {
+ StandardMBean bean = new StandardMBean(this, NameNodeMXBean.class);
+ this.nnInfoBeanName = MBeans.register("NameNode", "NameNodeInfo", bean);
+ LOG.info("Registered NameNodeInfo MBean: {}", this.nnInfoBeanName);
+ } catch (NotCompliantMBeanException e) {
+ throw new RuntimeException("Bad NameNodeInfo MBean setup", e);
+ }
+
+ try {
+ StandardMBean bean = new StandardMBean(this, NameNodeStatusMXBean.class);
+ this.nnStatusBeanName =
+ MBeans.register("NameNode", "NameNodeStatus", bean);
+ LOG.info("Registered NameNodeStatus MBean: {}", this.nnStatusBeanName);
+ } catch (NotCompliantMBeanException e) {
+ throw new RuntimeException("Bad NameNodeStatus MBean setup", e);
+ }
+ }
+
+ /**
+ * De-register the JMX interfaces.
+ */
+ public void close() {
+ if (fsStateBeanName != null) {
+ MBeans.unregister(fsStateBeanName);
+ fsStateBeanName = null;
+ }
+ if (nnInfoBeanName != null) {
+ MBeans.unregister(nnInfoBeanName);
+ nnInfoBeanName = null;
+ }
+ // Remove the NameNode status bean
+ if (nnStatusBeanName != null) {
+ MBeans.unregister(nnStatusBeanName);
+ nnStatusBeanName = null;
+ }
+ }
+
+ private FederationMetrics getFederationMetrics() {
+ return this.router.getMetrics();
+ }
+
+ /////////////////////////////////////////////////////////
+ // NameNodeMXBean
+ /////////////////////////////////////////////////////////
+
+ @Override
+ public String getVersion() {
+ return VersionInfo.getVersion() + ", r" + VersionInfo.getRevision();
+ }
+
+ @Override
+ public String getSoftwareVersion() {
+ return VersionInfo.getVersion();
+ }
+
+ @Override
+ public long getUsed() {
+ return getFederationMetrics().getUsedCapacity();
+ }
+
+ @Override
+ public long getFree() {
+ return getFederationMetrics().getRemainingCapacity();
+ }
+
+ @Override
+ public long getTotal() {
+ return getFederationMetrics().getTotalCapacity();
+ }
+
+ @Override
+ public long getProvidedCapacity() {
+ return getFederationMetrics().getProvidedSpace();
+ }
+
+ @Override
+ public String getSafemode() {
+ // We assume that the global federated view is never in safe mode
+ return "";
+ }
+
+ @Override
+ public boolean isUpgradeFinalized() {
+ // We assume the upgrade is always finalized in a federated biew
+ return true;
+ }
+
+ @Override
+ public RollingUpgradeInfo.Bean getRollingUpgradeStatus() {
+ return null;
+ }
+
+ @Override
+ public long getNonDfsUsedSpace() {
+ return 0;
+ }
+
+ @Override
+ public float getPercentUsed() {
+ return DFSUtilClient.getPercentUsed(getCapacityUsed(), getCapacityTotal());
+ }
+
+ @Override
+ public float getPercentRemaining() {
+ return DFSUtilClient.getPercentUsed(
+ getCapacityRemaining(), getCapacityTotal());
+ }
+
+ @Override
+ public long getCacheUsed() {
+ return 0;
+ }
+
+ @Override
+ public long getCacheCapacity() {
+ return 0;
+ }
+
+ @Override
+ public long getBlockPoolUsedSpace() {
+ return 0;
+ }
+
+ @Override
+ public float getPercentBlockPoolUsed() {
+ return 0;
+ }
+
+ @Override
+ public long getTotalBlocks() {
+ return getFederationMetrics().getNumBlocks();
+ }
+
+ @Override
+ public long getNumberOfMissingBlocks() {
+ return getFederationMetrics().getNumOfMissingBlocks();
+ }
+
+ @Override
+ @Deprecated
+ public long getPendingReplicationBlocks() {
+ return getFederationMetrics().getNumOfBlocksPendingReplication();
+ }
+
+ @Override
+ public long getPendingReconstructionBlocks() {
+ return getFederationMetrics().getNumOfBlocksPendingReplication();
+ }
+
+ @Override
+ @Deprecated
+ public long getUnderReplicatedBlocks() {
+ return getFederationMetrics().getNumOfBlocksUnderReplicated();
+ }
+
+ @Override
+ public long getLowRedundancyBlocks() {
+ return getFederationMetrics().getNumOfBlocksUnderReplicated();
+ }
+
+ @Override
+ public long getPendingDeletionBlocks() {
+ return getFederationMetrics().getNumOfBlocksPendingDeletion();
+ }
+
+ @Override
+ public long getScheduledReplicationBlocks() {
+ return -1;
+ }
+
+ @Override
+ public long getNumberOfMissingBlocksWithReplicationFactorOne() {
+ return 0;
+ }
+
+ @Override
+ public String getCorruptFiles() {
+ return "N/A";
+ }
+
+ @Override
+ public int getThreads() {
+ return ManagementFactory.getThreadMXBean().getThreadCount();
+ }
+
+ @Override
+ public String getLiveNodes() {
+ return this.getNodes(DatanodeReportType.LIVE);
+ }
+
+ @Override
+ public String getDeadNodes() {
+ return this.getNodes(DatanodeReportType.DEAD);
+ }
+
+ @Override
+ public String getDecomNodes() {
+ return this.getNodes(DatanodeReportType.DECOMMISSIONING);
+ }
+
+ /**
+ * Get all the nodes in the federation from a particular type.
+ * TODO this is expensive, we may want to cache it.
+ * @param type Type of the datanodes to check.
+ * @return JSON with the nodes.
+ */
+ private String getNodes(DatanodeReportType type) {
+ final Map<String, Map<String, Object>> info = new HashMap<>();
+ try {
+ RouterRpcServer rpcServer = this.router.getRpcServer();
+ DatanodeInfo[] datanodes = rpcServer.getDatanodeReport(type);
+ for (DatanodeInfo node : datanodes) {
+ Map<String, Object> innerinfo = new HashMap<>();
+ innerinfo.put("infoAddr", node.getInfoAddr());
+ innerinfo.put("infoSecureAddr", node.getInfoSecureAddr());
+ innerinfo.put("xferaddr", node.getXferAddr());
+ innerinfo.put("location", node.getNetworkLocation());
+ innerinfo.put("lastContact", getLastContact(node));
+ innerinfo.put("usedSpace", node.getDfsUsed());
+ innerinfo.put("adminState", node.getAdminState().toString());
+ innerinfo.put("nonDfsUsedSpace", node.getNonDfsUsed());
+ innerinfo.put("capacity", node.getCapacity());
+ innerinfo.put("numBlocks", -1); // node.numBlocks()
+ innerinfo.put("version", (node.getSoftwareVersion() == null ?
+ "UNKNOWN" : node.getSoftwareVersion()));
+ innerinfo.put("used", node.getDfsUsed());
+ innerinfo.put("remaining", node.getRemaining());
+ innerinfo.put("blockScheduled", -1); // node.getBlocksScheduled()
+ innerinfo.put("blockPoolUsed", node.getBlockPoolUsed());
+ innerinfo.put("blockPoolUsedPercent", node.getBlockPoolUsedPercent());
+ innerinfo.put("volfails", -1); // node.getVolumeFailures()
+ info.put(node.getHostName() + ":" + node.getXferPort(),
+ Collections.unmodifiableMap(innerinfo));
+ }
+ } catch (StandbyException e) {
+ LOG.error("Cannot get {} nodes, Router in safe mode", type);
+ } catch (IOException e) {
+ LOG.error("Cannot get " + type + " nodes", e);
+ }
+ return JSON.toString(info);
+ }
+
+ @Override
+ public String getClusterId() {
+ try {
+ return getNamespaceInfo(FederationNamespaceInfo::getClusterId).toString();
+ } catch (IOException e) {
+ LOG.error("Cannot fetch cluster ID metrics {}", e.getMessage());
+ return "";
+ }
+ }
+
+ @Override
+ public String getBlockPoolId() {
+ try {
+ return
+ getNamespaceInfo(FederationNamespaceInfo::getBlockPoolId).toString();
+ } catch (IOException e) {
+ LOG.error("Cannot fetch block pool ID metrics {}", e.getMessage());
+ return "";
+ }
+ }
+
+ /**
+ * Build a set of unique values found in all namespaces.
+ *
+ * @param f Method reference of the appropriate FederationNamespaceInfo
+ * getter function
+ * @return Set of unique string values found in all discovered namespaces.
+ * @throws IOException if the query could not be executed.
+ */
+ private Collection<String> getNamespaceInfo(
+ Function<FederationNamespaceInfo, String> f) throws IOException {
+ StateStoreService stateStore = router.getStateStore();
+ MembershipStore membershipStore =
+ stateStore.getRegisteredRecordStore(MembershipStore.class);
+
+ GetNamespaceInfoRequest request = GetNamespaceInfoRequest.newInstance();
+ GetNamespaceInfoResponse response =
+ membershipStore.getNamespaceInfo(request);
+ return response.getNamespaceInfo().stream()
+ .map(f)
+ .collect(Collectors.toSet());
+ }
+
+ @Override
+ public String getNameDirStatuses() {
+ return "N/A";
+ }
+
+ @Override
+ public String getNodeUsage() {
+ return "N/A";
+ }
+
+ @Override
+ public String getNameJournalStatus() {
+ return "N/A";
+ }
+
+ @Override
+ public String getJournalTransactionInfo() {
+ return "N/A";
+ }
+
+ @Override
+ public long getNNStartedTimeInMillis() {
+ return this.router.getStartTime();
+ }
+
+ @Override
+ public String getCompileInfo() {
+ return VersionInfo.getDate() + " by " + VersionInfo.getUser() +
+ " from " + VersionInfo.getBranch();
+ }
+
+ @Override
+ public int getDistinctVersionCount() {
+ return 0;
+ }
+
+ @Override
+ public Map<String, Integer> getDistinctVersions() {
+ return null;
+ }
+
+ /////////////////////////////////////////////////////////
+ // FSNamesystemMBean
+ /////////////////////////////////////////////////////////
+
+ @Override
+ public String getFSState() {
+ // We assume is not in safe mode
+ return "Operational";
+ }
+
+ @Override
+ public long getBlocksTotal() {
+ return this.getTotalBlocks();
+ }
+
+ @Override
+ public long getCapacityTotal() {
+ return this.getTotal();
+ }
+
+ @Override
+ public long getCapacityRemaining() {
+ return this.getFree();
+ }
+
+ @Override
+ public long getCapacityUsed() {
+ return this.getUsed();
+ }
+
+ @Override
+ public long getProvidedCapacityTotal() {
+ return getProvidedCapacity();
+ }
+
+ @Override
+ public long getFilesTotal() {
+ return getFederationMetrics().getNumFiles();
+ }
+
+ @Override
+ public int getTotalLoad() {
+ return -1;
+ }
+
+ @Override
+ public int getNumLiveDataNodes() {
+ return this.router.getMetrics().getNumLiveNodes();
+ }
+
+ @Override
+ public int getNumDeadDataNodes() {
+ return this.router.getMetrics().getNumDeadNodes();
+ }
+
+ @Override
+ public int getNumStaleDataNodes() {
+ return -1;
+ }
+
+ @Override
+ public int getNumDecomLiveDataNodes() {
+ return this.router.getMetrics().getNumDecomLiveNodes();
+ }
+
+ @Override
+ public int getNumDecomDeadDataNodes() {
+ return this.router.getMetrics().getNumDecomDeadNodes();
+ }
+
+ @Override
+ public int getNumDecommissioningDataNodes() {
+ return this.router.getMetrics().getNumDecommissioningNodes();
+ }
+
+ @Override
+ public int getNumInMaintenanceLiveDataNodes() {
+ return 0;
+ }
+
+ @Override
+ public int getNumInMaintenanceDeadDataNodes() {
+ return 0;
+ }
+
+ @Override
+ public int getNumEnteringMaintenanceDataNodes() {
+ return 0;
+ }
+
+ @Override
+ public int getVolumeFailuresTotal() {
+ return 0;
+ }
+
+ @Override
+ public long getEstimatedCapacityLostTotal() {
+ return 0;
+ }
+
+ @Override
+ public String getSnapshotStats() {
+ return null;
+ }
+
+ @Override
+ public long getMaxObjects() {
+ return 0;
+ }
+
+ @Override
+ public long getBlockDeletionStartTime() {
+ return -1;
+ }
+
+ @Override
+ public int getNumStaleStorages() {
+ return -1;
+ }
+
+ @Override
+ public String getTopUserOpCounts() {
+ return "N/A";
+ }
+
+ @Override
+ public int getFsLockQueueLength() {
+ return 0;
+ }
+
+ @Override
+ public long getTotalSyncCount() {
+ return 0;
+ }
+
+ @Override
+ public String getTotalSyncTimes() {
+ return "";
+ }
+
+ private long getLastContact(DatanodeInfo node) {
+ return (now() - node.getLastUpdate()) / 1000;
+ }
+
+ /////////////////////////////////////////////////////////
+ // NameNodeStatusMXBean
+ /////////////////////////////////////////////////////////
+
+ @Override
+ public String getNNRole() {
+ return NamenodeRole.NAMENODE.toString();
+ }
+
+ @Override
+ public String getState() {
+ return HAServiceState.ACTIVE.toString();
+ }
+
+ @Override
+ public String getHostAndPort() {
+ return NetUtils.getHostPortString(router.getRpcServerAddress());
+ }
+
+ @Override
+ public boolean isSecurityEnabled() {
+ return false;
+ }
+
+ @Override
+ public long getLastHATransitionTime() {
+ return 0;
+ }
+
+ @Override
+ public long getBytesWithFutureGenerationStamps() {
+ return 0;
+ }
+
+ @Override
+ public String getSlowPeersReport() {
+ return "N/A";
+ }
+
+ @Override
+ public String getSlowDisksReport() {
+ return "N/A";
+ }
+
+ @Override
+ public long getNumberOfSnapshottableDirs() {
+ return 0;
+ }
+
+ @Override
+ public String getEnteringMaintenanceNodes() {
+ return "N/A";
+ }
+
+ @Override
+ public String getNameDirSize() {
+ return "N/A";
+ }
+
+ @Override
+ public int getNumEncryptionZones() {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMBean.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMBean.java
new file mode 100644
index 0000000..5e4ccab
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMBean.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.metrics;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * JMX interface for the State Store metrics.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface StateStoreMBean {
+
+ long getReadOps();
+
+ double getReadAvg();
+
+ long getWriteOps();
+
+ double getWriteAvg();
+
+ long getFailureOps();
+
+ double getFailureAvg();
+
+ long getRemoveOps();
+
+ double getRemoveAvg();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
new file mode 100644
index 0000000..09253a2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.metrics;
+
+import static org.apache.hadoop.metrics2.impl.MsInfo.ProcessName;
+import static org.apache.hadoop.metrics2.impl.MsInfo.SessionId;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
+import org.apache.hadoop.metrics2.lib.MutableRate;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Implementations of the JMX interface for the State Store metrics.
+ */
+@Metrics(name = "StateStoreActivity", about = "Router metrics",
+ context = "dfs")
+public final class StateStoreMetrics implements StateStoreMBean {
+
+ private final MetricsRegistry registry = new MetricsRegistry("router");
+
+ @Metric("GET transactions")
+ private MutableRate reads;
+ @Metric("PUT transactions")
+ private MutableRate writes;
+ @Metric("REMOVE transactions")
+ private MutableRate removes;
+ @Metric("Failed transactions")
+ private MutableRate failures;
+
+ private Map<String, MutableGaugeInt> cacheSizes;
+
+ private StateStoreMetrics(Configuration conf) {
+ registry.tag(SessionId, "RouterSession");
+ registry.tag(ProcessName, "Router");
+ cacheSizes = new HashMap<>();
+ }
+
+ public static StateStoreMetrics create(Configuration conf) {
+ MetricsSystem ms = DefaultMetricsSystem.instance();
+ return ms.register(new StateStoreMetrics(conf));
+ }
+
+ public void shutdown() {
+ DefaultMetricsSystem.shutdown();
+ reset();
+ }
+
+ public void addRead(long latency) {
+ reads.add(latency);
+ }
+
+ public long getReadOps() {
+ return reads.lastStat().numSamples();
+ }
+
+ public double getReadAvg() {
+ return reads.lastStat().mean();
+ }
+
+ public void addWrite(long latency) {
+ writes.add(latency);
+ }
+
+ public long getWriteOps() {
+ return writes.lastStat().numSamples();
+ }
+
+ public double getWriteAvg() {
+ return writes.lastStat().mean();
+ }
+
+ public void addFailure(long latency) {
+ failures.add(latency);
+ }
+
+ public long getFailureOps() {
+ return failures.lastStat().numSamples();
+ }
+
+ public double getFailureAvg() {
+ return failures.lastStat().mean();
+ }
+
+ public void addRemove(long latency) {
+ removes.add(latency);
+ }
+
+ public long getRemoveOps() {
+ return removes.lastStat().numSamples();
+ }
+
+ public double getRemoveAvg() {
+ return removes.lastStat().mean();
+ }
+
+ /**
+ * Set the size of the cache for a State Store interface.
+ *
+ * @param name Name of the record to cache.
+ * @param size Number of records.
+ */
+ public void setCacheSize(String name, int size) {
+ String counterName = "Cache" + name + "Size";
+ MutableGaugeInt counter = cacheSizes.get(counterName);
+ if (counter == null) {
+ counter = registry.newGauge(counterName, name, size);
+ cacheSizes.put(counterName, counter);
+ }
+ counter.set(size);
+ }
+
+ @VisibleForTesting
+ public void reset() {
+ reads.resetMinMax();
+ writes.resetMinMax();
+ removes.resetMinMax();
+ failures.resetMinMax();
+
+ reads.lastStat().reset();
+ writes.lastStat().reset();
+ removes.lastStat().reset();
+ failures.lastStat().reset();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/package-info.java
new file mode 100644
index 0000000..c56c823
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/package-info.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Report metrics for Router-based Federation.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+package org.apache.hadoop.hdfs.server.federation.metrics;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java
new file mode 100644
index 0000000..1773b34
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Locates the most active NN for a given nameservice ID or blockpool ID. This
+ * interface is used by the {@link org.apache.hadoop.hdfs.server.federation.
+ * router.RouterRpcServer RouterRpcServer} to:
+ * <ul>
+ * <li>Determine the target NN for a given subcluster.
+ * <li>List of all namespaces discovered/active in the federation.
+ * <li>Update the currently active NN empirically.
+ * </ul>
+ * The interface is also used by the {@link org.apache.hadoop.hdfs.server.
+ * federation.router.NamenodeHeartbeatService NamenodeHeartbeatService} to
+ * register a discovered NN.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface ActiveNamenodeResolver {
+
+ /**
+ * Report a successful, active NN address for a nameservice or blockPool.
+ *
+ * @param ns Nameservice identifier.
+ * @param successfulAddress The address the successful responded to the
+ * command.
+ * @throws IOException If the state store cannot be accessed.
+ */
+ void updateActiveNamenode(
+ String ns, InetSocketAddress successfulAddress) throws IOException;
+
+ /**
+ * Returns a prioritized list of the most recent cached registration entries
+ * for a single nameservice ID.
+ * Returns an empty list if none are found. Returns entries in preference of:
+ * <ul>
+ * <li>The most recent ACTIVE NN
+ * <li>The most recent STANDBY NN
+ * <li>The most recent UNAVAILABLE NN
+ * </ul>
+ *
+ * @param nameserviceId Nameservice identifier.
+ * @return Prioritized list of namenode contexts.
+ * @throws IOException If the state store cannot be accessed.
+ */
+ List<? extends FederationNamenodeContext>
+ getNamenodesForNameserviceId(String nameserviceId) throws IOException;
+
+ /**
+ * Returns a prioritized list of the most recent cached registration entries
+ * for a single block pool ID.
+ * Returns an empty list if none are found. Returns entries in preference of:
+ * <ul>
+ * <li>The most recent ACTIVE NN
+ * <li>The most recent STANDBY NN
+ * <li>The most recent UNAVAILABLE NN
+ * </ul>
+ *
+ * @param blockPoolId Block pool identifier for the nameservice.
+ * @return Prioritized list of namenode contexts.
+ * @throws IOException If the state store cannot be accessed.
+ */
+ List<? extends FederationNamenodeContext>
+ getNamenodesForBlockPoolId(String blockPoolId) throws IOException;
+
+ /**
+ * Register a namenode in the State Store.
+ *
+ * @param report Namenode status report.
+ * @return True if the node was registered and successfully committed to the
+ * data store.
+ * @throws IOException Throws exception if the namenode could not be
+ * registered.
+ */
+ boolean registerNamenode(NamenodeStatusReport report) throws IOException;
+
+ /**
+ * Get a list of all namespaces that are registered and active in the
+ * federation.
+ *
+ * @return List of name spaces in the federation
+ * @throws IOException Throws exception if the namespace list is not
+ * available.
+ */
+ Set<FederationNamespaceInfo> getNamespaces() throws IOException;
+
+ /**
+ * Assign a unique identifier for the parent router service.
+ * Required to report the status to the namenode resolver.
+ *
+ * @param routerId Unique string identifier for the router.
+ */
+ void setRouterId(String routerId);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeContext.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeContext.java
new file mode 100644
index 0000000..68ef02a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeContext.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+/**
+ * Interface for a discovered NN and its current server endpoints.
+ */
+public interface FederationNamenodeContext {
+
+ /**
+ * Get the RPC server address of the namenode.
+ *
+ * @return RPC server address in the form of host:port.
+ */
+ String getRpcAddress();
+
+ /**
+ * Get the Service RPC server address of the namenode.
+ *
+ * @return Service RPC server address in the form of host:port.
+ */
+ String getServiceAddress();
+
+ /**
+ * Get the Lifeline RPC server address of the namenode.
+ *
+ * @return Lifeline RPC server address in the form of host:port.
+ */
+ String getLifelineAddress();
+
+ /**
+ * Get the HTTP server address of the namenode.
+ *
+ * @return HTTP address in the form of host:port.
+ */
+ String getWebAddress();
+
+ /**
+ * Get the unique key representing the namenode.
+ *
+ * @return Combination of the nameservice and the namenode IDs.
+ */
+ String getNamenodeKey();
+
+ /**
+ * Identifier for the nameservice/namespace.
+ *
+ * @return Namenode nameservice identifier.
+ */
+ String getNameserviceId();
+
+ /**
+ * Identifier for the namenode.
+ *
+ * @return String
+ */
+ String getNamenodeId();
+
+ /**
+ * The current state of the namenode (active, standby, etc).
+ *
+ * @return FederationNamenodeServiceState State of the namenode.
+ */
+ FederationNamenodeServiceState getState();
+
+ /**
+ * The update date.
+ *
+ * @return Long with the update date.
+ */
+ long getDateModified();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java
new file mode 100644
index 0000000..c773f82
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+
+/**
+ * Namenode state in the federation. The order of this enum is used to evaluate
+ * NN priority for RPC calls.
+ */
+public enum FederationNamenodeServiceState {
+ ACTIVE, // HAServiceState.ACTIVE or operational.
+ STANDBY, // HAServiceState.STANDBY.
+ UNAVAILABLE, // When the namenode cannot be reached.
+ EXPIRED; // When the last update is too old.
+
+ public static FederationNamenodeServiceState getState(HAServiceState state) {
+ switch(state) {
+ case ACTIVE:
+ return FederationNamenodeServiceState.ACTIVE;
+ case STANDBY:
+ return FederationNamenodeServiceState.STANDBY;
+ case INITIALIZING:
+ return FederationNamenodeServiceState.UNAVAILABLE;
+ case STOPPING:
+ return FederationNamenodeServiceState.UNAVAILABLE;
+ default:
+ return FederationNamenodeServiceState.UNAVAILABLE;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java
new file mode 100644
index 0000000..edcd308
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import org.apache.hadoop.hdfs.server.federation.router.RemoteLocationContext;
+
+/**
+ * Represents information about a single nameservice/namespace in a federated
+ * HDFS cluster.
+ */
+public class FederationNamespaceInfo extends RemoteLocationContext {
+
+ /** Block pool identifier. */
+ private final String blockPoolId;
+ /** Cluster identifier. */
+ private final String clusterId;
+ /** Nameservice identifier. */
+ private final String nameserviceId;
+
+ public FederationNamespaceInfo(String bpId, String clId, String nsId) {
+ this.blockPoolId = bpId;
+ this.clusterId = clId;
+ this.nameserviceId = nsId;
+ }
+
+ @Override
+ public String getNameserviceId() {
+ return this.nameserviceId;
+ }
+
+ @Override
+ public String getDest() {
+ return this.nameserviceId;
+ }
+
+ /**
+ * The HDFS cluster id for this namespace.
+ *
+ * @return Cluster identifier.
+ */
+ public String getClusterId() {
+ return this.clusterId;
+ }
+
+ /**
+ * The HDFS block pool id for this namespace.
+ *
+ * @return Block pool identifier.
+ */
+ public String getBlockPoolId() {
+ return this.blockPoolId;
+ }
+
+ @Override
+ public String toString() {
+ return this.nameserviceId + "->" + this.blockPoolId + ":" + this.clusterId;
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
deleted file mode 100644
index 2b12a25..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
+++ /dev/null
@@ -1,1072 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.addDirectory;
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.countContents;
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createFile;
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.deleteFile;
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.getFileStatus;
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyFileExists;
-import static org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.TEST_STRING;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.lang.reflect.Method;
-import java.net.URISyntaxException;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.EnumSet;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Random;
-import java.util.Set;
-import java.util.TreeSet;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.crypto.CryptoProtocolVersion;
-import org.apache.hadoop.fs.CreateFlag;
-import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Options;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.DFSClient;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
-import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse;
-import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
-import org.apache.hadoop.hdfs.protocol.ClientProtocol;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-import org.apache.hadoop.hdfs.protocol.DirectoryListing;
-import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats;
-import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
-import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
-import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
-import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
-import org.apache.hadoop.hdfs.protocol.LocatedBlock;
-import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
-import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster;
-import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
-import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
-import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
-import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
-import org.apache.hadoop.io.EnumSetWritable;
-import org.apache.hadoop.io.erasurecode.ECSchema;
-import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.Service.STATE;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Supplier;
-import com.google.common.collect.Maps;
-
-/**
- * The the RPC interface of the {@link Router} implemented by
- * {@link RouterRpcServer}.
- */
-public class TestRouterRpc {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(TestRouterRpc.class);
-
- private static final Comparator<ErasureCodingPolicyInfo> EC_POLICY_CMP =
- new Comparator<ErasureCodingPolicyInfo>() {
- public int compare(
- ErasureCodingPolicyInfo ec0,
- ErasureCodingPolicyInfo ec1) {
- String name0 = ec0.getPolicy().getName();
- String name1 = ec1.getPolicy().getName();
- return name0.compareTo(name1);
- }
- };
-
- /** Federated HDFS cluster. */
- private static RouterDFSCluster cluster;
-
- /** Random Router for this federated cluster. */
- private RouterContext router;
-
- /** Random nameservice in the federated cluster. */
- private String ns;
- /** First namenode in the nameservice. */
- private NamenodeContext namenode;
-
- /** Client interface to the Router. */
- private ClientProtocol routerProtocol;
- /** Client interface to the Namenode. */
- private ClientProtocol nnProtocol;
-
- /** Filesystem interface to the Router. */
- private FileSystem routerFS;
- /** Filesystem interface to the Namenode. */
- private FileSystem nnFS;
-
- /** File in the Router. */
- private String routerFile;
- /** File in the Namenode. */
- private String nnFile;
-
-
- @BeforeClass
- public static void globalSetUp() throws Exception {
- cluster = new RouterDFSCluster(false, 2);
- // We need 6 DNs to test Erasure Coding with RS-6-3-64k
- cluster.setNumDatanodesPerNameservice(6);
-
- // Start NNs and DNs and wait until ready
- cluster.startCluster();
-
- // Start routers with only an RPC service
- cluster.addRouterOverrides((new RouterConfigBuilder()).rpc().build());
- cluster.startRouters();
-
- // Register and verify all NNs with all routers
- cluster.registerNamenodes();
- cluster.waitNamenodeRegistration();
- }
-
- @AfterClass
- public static void tearDown() {
- cluster.shutdown();
- }
-
- @Before
- public void testSetup() throws Exception {
-
- // Create mock locations
- cluster.installMockLocations();
-
- // Delete all files via the NNs and verify
- cluster.deleteAllFiles();
-
- // Create test fixtures on NN
- cluster.createTestDirectoriesNamenode();
-
- // Wait to ensure NN has fully created its test directories
- Thread.sleep(100);
-
- // Default namenode and random router for this test
- this.router = cluster.getRandomRouter();
- this.ns = cluster.getNameservices().get(0);
- this.namenode = cluster.getNamenode(ns, null);
-
- // Handles to the ClientProtocol interface
- this.routerProtocol = router.getClient().getNamenode();
- this.nnProtocol = namenode.getClient().getNamenode();
-
- // Handles to the filesystem client
- this.nnFS = namenode.getFileSystem();
- this.routerFS = router.getFileSystem();
-
- // Create a test file on the NN
- Random r = new Random();
- String randomFile = "testfile-" + r.nextInt();
- this.nnFile =
- cluster.getNamenodeTestDirectoryForNS(ns) + "/" + randomFile;
- this.routerFile =
- cluster.getFederatedTestDirectoryForNS(ns) + "/" + randomFile;
-
- createFile(nnFS, nnFile, 32);
- verifyFileExists(nnFS, nnFile);
- }
-
- @Test
- public void testRpcService() throws IOException {
- Router testRouter = new Router();
- List<String> nss = cluster.getNameservices();
- String ns0 = nss.get(0);
- Configuration routerConfig = cluster.generateRouterConfiguration(ns0, null);
- RouterRpcServer server = new RouterRpcServer(routerConfig, testRouter,
- testRouter.getNamenodeResolver(), testRouter.getSubclusterResolver());
- server.init(routerConfig);
- assertEquals(STATE.INITED, server.getServiceState());
- server.start();
- assertEquals(STATE.STARTED, server.getServiceState());
- server.stop();
- assertEquals(STATE.STOPPED, server.getServiceState());
- server.close();
- testRouter.close();
- }
-
- protected RouterDFSCluster getCluster() {
- return TestRouterRpc.cluster;
- }
-
- protected RouterContext getRouterContext() {
- return this.router;
- }
-
- protected void setRouter(RouterContext r)
- throws IOException, URISyntaxException {
- this.router = r;
- this.routerProtocol = r.getClient().getNamenode();
- this.routerFS = r.getFileSystem();
- }
-
- protected FileSystem getRouterFileSystem() {
- return this.routerFS;
- }
-
- protected FileSystem getNamenodeFileSystem() {
- return this.nnFS;
- }
-
- protected ClientProtocol getRouterProtocol() {
- return this.routerProtocol;
- }
-
- protected ClientProtocol getNamenodeProtocol() {
- return this.nnProtocol;
- }
-
- protected NamenodeContext getNamenode() {
- return this.namenode;
- }
-
- protected void setNamenodeFile(String filename) {
- this.nnFile = filename;
- }
-
- protected String getNamenodeFile() {
- return this.nnFile;
- }
-
- protected void setRouterFile(String filename) {
- this.routerFile = filename;
- }
-
- protected String getRouterFile() {
- return this.routerFile;
- }
-
- protected void setNamenode(NamenodeContext nn)
- throws IOException, URISyntaxException {
- this.namenode = nn;
- this.nnProtocol = nn.getClient().getNamenode();
- this.nnFS = nn.getFileSystem();
- }
-
- protected String getNs() {
- return this.ns;
- }
-
- protected void setNs(String nameservice) {
- this.ns = nameservice;
- }
-
- protected static void compareResponses(
- ClientProtocol protocol1, ClientProtocol protocol2,
- Method m, Object[] paramList) {
-
- Object return1 = null;
- Exception exception1 = null;
- try {
- return1 = m.invoke(protocol1, paramList);
- } catch (Exception ex) {
- exception1 = ex;
- }
-
- Object return2 = null;
- Exception exception2 = null;
- try {
- return2 = m.invoke(protocol2, paramList);
- } catch (Exception ex) {
- exception2 = ex;
- }
-
- assertEquals(return1, return2);
- if (exception1 == null && exception2 == null) {
- return;
- }
-
- assertEquals(
- exception1.getCause().getClass(),
- exception2.getCause().getClass());
- }
-
- @Test
- public void testProxyListFiles() throws IOException, InterruptedException,
- URISyntaxException, NoSuchMethodException, SecurityException {
-
- // Verify that the root listing is a union of the mount table destinations
- // and the files stored at all nameservices mounted at the root (ns0 + ns1)
- //
- // / -->
- // /ns0 (from mount table)
- // /ns1 (from mount table)
- // all items in / of ns0 (default NS)
-
- // Collect the mount table entries from the root mount point
- Set<String> requiredPaths = new TreeSet<>();
- FileSubclusterResolver fileResolver =
- router.getRouter().getSubclusterResolver();
- for (String mount : fileResolver.getMountPoints("/")) {
- requiredPaths.add(mount);
- }
-
- // Collect all files/dirs on the root path of the default NS
- String defaultNs = cluster.getNameservices().get(0);
- NamenodeContext nn = cluster.getNamenode(defaultNs, null);
- FileStatus[] iterator = nn.getFileSystem().listStatus(new Path("/"));
- for (FileStatus file : iterator) {
- requiredPaths.add(file.getPath().getName());
- }
-
- // Fetch listing
- DirectoryListing listing =
- routerProtocol.getListing("/", HdfsFileStatus.EMPTY_NAME, false);
- Iterator<String> requiredPathsIterator = requiredPaths.iterator();
- // Match each path returned and verify order returned
- for(HdfsFileStatus f : listing.getPartialListing()) {
- String fileName = requiredPathsIterator.next();
- String currentFile = f.getFullPath(new Path("/")).getName();
- assertEquals(currentFile, fileName);
- }
-
- // Verify the total number of results found/matched
- assertEquals(requiredPaths.size(), listing.getPartialListing().length);
-
- // List a path that doesn't exist and validate error response with NN
- // behavior.
- Method m = ClientProtocol.class.getMethod(
- "getListing", String.class, byte[].class, boolean.class);
- String badPath = "/unknownlocation/unknowndir";
- compareResponses(routerProtocol, nnProtocol, m,
- new Object[] {badPath, HdfsFileStatus.EMPTY_NAME, false});
- }
-
- @Test
- public void testProxyListFilesWithConflict()
- throws IOException, InterruptedException {
-
- // Add a directory to the namespace that conflicts with a mount point
- NamenodeContext nn = cluster.getNamenode(ns, null);
- FileSystem nnFs = nn.getFileSystem();
- addDirectory(nnFs, cluster.getFederatedTestDirectoryForNS(ns));
-
- FileSystem routerFs = router.getFileSystem();
- int initialCount = countContents(routerFs, "/");
-
- // Root file system now for NS X:
- // / ->
- // /ns0 (mount table)
- // /ns1 (mount table)
- // /target-ns0 (the target folder for the NS0 mapped to /
- // /nsX (local directory that duplicates mount table)
- int newCount = countContents(routerFs, "/");
- assertEquals(initialCount, newCount);
-
- // Verify that each root path is readable and contains one test directory
- assertEquals(1, countContents(routerFs, cluster.getFederatedPathForNS(ns)));
-
- // Verify that real folder for the ns contains a single test directory
- assertEquals(1, countContents(nnFs, cluster.getNamenodePathForNS(ns)));
-
- }
-
- protected void testRename(RouterContext testRouter, String filename,
- String renamedFile, boolean exceptionExpected) throws IOException {
-
- createFile(testRouter.getFileSystem(), filename, 32);
- // verify
- verifyFileExists(testRouter.getFileSystem(), filename);
- // rename
- boolean exceptionThrown = false;
- try {
- DFSClient client = testRouter.getClient();
- ClientProtocol clientProtocol = client.getNamenode();
- clientProtocol.rename(filename, renamedFile);
- } catch (Exception ex) {
- exceptionThrown = true;
- }
- if (exceptionExpected) {
- // Error was expected
- assertTrue(exceptionThrown);
- FileContext fileContext = testRouter.getFileContext();
- assertTrue(fileContext.delete(new Path(filename), true));
- } else {
- // No error was expected
- assertFalse(exceptionThrown);
- // verify
- assertTrue(verifyFileExists(testRouter.getFileSystem(), renamedFile));
- // delete
- FileContext fileContext = testRouter.getFileContext();
- assertTrue(fileContext.delete(new Path(renamedFile), true));
- }
- }
-
- protected void testRename2(RouterContext testRouter, String filename,
- String renamedFile, boolean exceptionExpected) throws IOException {
- createFile(testRouter.getFileSystem(), filename, 32);
- // verify
- verifyFileExists(testRouter.getFileSystem(), filename);
- // rename
- boolean exceptionThrown = false;
- try {
- DFSClient client = testRouter.getClient();
- ClientProtocol clientProtocol = client.getNamenode();
- clientProtocol.rename2(filename, renamedFile, new Options.Rename[] {});
- } catch (Exception ex) {
- exceptionThrown = true;
- }
- assertEquals(exceptionExpected, exceptionThrown);
- if (exceptionExpected) {
- // Error was expected
- FileContext fileContext = testRouter.getFileContext();
- assertTrue(fileContext.delete(new Path(filename), true));
- } else {
- // verify
- assertTrue(verifyFileExists(testRouter.getFileSystem(), renamedFile));
- // delete
- FileContext fileContext = testRouter.getFileContext();
- assertTrue(fileContext.delete(new Path(renamedFile), true));
- }
- }
-
- @Test
- public void testProxyRenameFiles() throws IOException, InterruptedException {
-
- Thread.sleep(5000);
- List<String> nss = cluster.getNameservices();
- String ns0 = nss.get(0);
- String ns1 = nss.get(1);
-
- // Rename within the same namespace
- // /ns0/testdir/testrename -> /ns0/testdir/testrename-append
- String filename =
- cluster.getFederatedTestDirectoryForNS(ns0) + "/testrename";
- String renamedFile = filename + "-append";
- testRename(router, filename, renamedFile, false);
- testRename2(router, filename, renamedFile, false);
-
- // Rename a file to a destination that is in a different namespace (fails)
- filename = cluster.getFederatedTestDirectoryForNS(ns0) + "/testrename";
- renamedFile = cluster.getFederatedTestDirectoryForNS(ns1) + "/testrename";
- testRename(router, filename, renamedFile, true);
- testRename2(router, filename, renamedFile, true);
- }
-
- @Test
- public void testProxyChownFiles() throws Exception {
-
- String newUsername = "TestUser";
- String newGroup = "TestGroup";
-
- // change owner
- routerProtocol.setOwner(routerFile, newUsername, newGroup);
-
- // Verify with NN
- FileStatus file = getFileStatus(namenode.getFileSystem(), nnFile);
- assertEquals(file.getOwner(), newUsername);
- assertEquals(file.getGroup(), newGroup);
-
- // Bad request and validate router response matches NN response.
- Method m = ClientProtocol.class.getMethod("setOwner", String.class,
- String.class, String.class);
- String badPath = "/unknownlocation/unknowndir";
- compareResponses(routerProtocol, nnProtocol, m,
- new Object[] {badPath, newUsername, newGroup});
- }
-
- @Test
- public void testProxyGetStats() throws Exception {
- // Some of the statistics are out of sync because of the mini cluster
- Supplier<Boolean> check = new Supplier<Boolean>() {
- @Override
- public Boolean get() {
- try {
- long[] combinedData = routerProtocol.getStats();
- long[] individualData = getAggregateStats();
- int len = Math.min(combinedData.length, individualData.length);
- for (int i = 0; i < len; i++) {
- if (combinedData[i] != individualData[i]) {
- LOG.error("Stats for {} don't match: {} != {}",
- i, combinedData[i], individualData[i]);
- return false;
- }
- }
- return true;
- } catch (Exception e) {
- LOG.error("Cannot get stats: {}", e.getMessage());
- return false;
- }
- }
- };
- GenericTestUtils.waitFor(check, 500, 5 * 1000);
- }
-
- /**
- * Get the sum of each subcluster statistics.
- * @return Aggregated statistics.
- * @throws Exception If it cannot get the stats from the Router or Namenode.
- */
- private long[] getAggregateStats() throws Exception {
- long[] individualData = new long[10];
- for (String nameservice : cluster.getNameservices()) {
- NamenodeContext n = cluster.getNamenode(nameservice, null);
- DFSClient client = n.getClient();
- ClientProtocol clientProtocol = client.getNamenode();
- long[] data = clientProtocol.getStats();
- for (int i = 0; i < data.length; i++) {
- individualData[i] += data[i];
- }
- }
- return individualData;
- }
-
- @Test
- public void testProxyGetDatanodeReport() throws Exception {
-
- DatanodeInfo[] combinedData =
- routerProtocol.getDatanodeReport(DatanodeReportType.ALL);
-
- Set<Integer> individualData = new HashSet<Integer>();
- for (String nameservice : cluster.getNameservices()) {
- NamenodeContext n = cluster.getNamenode(nameservice, null);
- DFSClient client = n.getClient();
- ClientProtocol clientProtocol = client.getNamenode();
- DatanodeInfo[] data =
- clientProtocol.getDatanodeReport(DatanodeReportType.ALL);
- for (int i = 0; i < data.length; i++) {
- // Collect unique DNs based on their xfer port
- DatanodeInfo info = data[i];
- individualData.add(info.getXferPort());
- }
- }
- assertEquals(combinedData.length, individualData.size());
- }
-
- @Test
- public void testProxyGetDatanodeStorageReport()
- throws IOException, InterruptedException, URISyntaxException {
-
- DatanodeStorageReport[] combinedData =
- routerProtocol.getDatanodeStorageReport(DatanodeReportType.ALL);
-
- Set<String> individualData = new HashSet<>();
- for (String nameservice : cluster.getNameservices()) {
- NamenodeContext n = cluster.getNamenode(nameservice, null);
- DFSClient client = n.getClient();
- ClientProtocol clientProtocol = client.getNamenode();
- DatanodeStorageReport[] data =
- clientProtocol.getDatanodeStorageReport(DatanodeReportType.ALL);
- for (DatanodeStorageReport report : data) {
- // Determine unique DN instances
- DatanodeInfo dn = report.getDatanodeInfo();
- individualData.add(dn.toString());
- }
- }
- assertEquals(combinedData.length, individualData.size());
- }
-
- @Test
- public void testProxyMkdir() throws Exception {
-
- // Check the initial folders
- FileStatus[] filesInitial = routerFS.listStatus(new Path("/"));
-
- // Create a directory via the router at the root level
- String dirPath = "/testdir";
- FsPermission permission = new FsPermission("705");
- routerProtocol.mkdirs(dirPath, permission, false);
-
- // Verify the root listing has the item via the router
- FileStatus[] files = routerFS.listStatus(new Path("/"));
- assertEquals(Arrays.toString(files) + " should be " +
- Arrays.toString(filesInitial) + " + " + dirPath,
- filesInitial.length + 1, files.length);
- assertTrue(verifyFileExists(routerFS, dirPath));
-
- // Verify the directory is present in only 1 Namenode
- int foundCount = 0;
- for (NamenodeContext n : cluster.getNamenodes()) {
- if (verifyFileExists(n.getFileSystem(), dirPath)) {
- foundCount++;
- }
- }
- assertEquals(1, foundCount);
- assertTrue(deleteFile(routerFS, dirPath));
-
- // Validate router failure response matches NN failure response.
- Method m = ClientProtocol.class.getMethod("mkdirs", String.class,
- FsPermission.class, boolean.class);
- String badPath = "/unknownlocation/unknowndir";
- compareResponses(routerProtocol, nnProtocol, m,
- new Object[] {badPath, permission, false});
- }
-
- @Test
- public void testProxyChmodFiles() throws Exception {
-
- FsPermission permission = new FsPermission("444");
-
- // change permissions
- routerProtocol.setPermission(routerFile, permission);
-
- // Validate permissions NN
- FileStatus file = getFileStatus(namenode.getFileSystem(), nnFile);
- assertEquals(permission, file.getPermission());
-
- // Validate router failure response matches NN failure response.
- Method m = ClientProtocol.class.getMethod(
- "setPermission", String.class, FsPermission.class);
- String badPath = "/unknownlocation/unknowndir";
- compareResponses(routerProtocol, nnProtocol, m,
- new Object[] {badPath, permission});
- }
-
- @Test
- public void testProxySetReplication() throws Exception {
-
- // Check current replication via NN
- FileStatus file = getFileStatus(nnFS, nnFile);
- assertEquals(1, file.getReplication());
-
- // increment replication via router
- routerProtocol.setReplication(routerFile, (short) 2);
-
- // Verify via NN
- file = getFileStatus(nnFS, nnFile);
- assertEquals(2, file.getReplication());
-
- // Validate router failure response matches NN failure response.
- Method m = ClientProtocol.class.getMethod(
- "setReplication", String.class, short.class);
- String badPath = "/unknownlocation/unknowndir";
- compareResponses(routerProtocol, nnProtocol, m,
- new Object[] {badPath, (short) 2});
- }
-
- @Test
- public void testProxyTruncateFile() throws Exception {
-
- // Check file size via NN
- FileStatus file = getFileStatus(nnFS, nnFile);
- assertTrue(file.getLen() > 0);
-
- // Truncate to 0 bytes via router
- routerProtocol.truncate(routerFile, 0, "testclient");
-
- // Verify via NN
- file = getFileStatus(nnFS, nnFile);
- assertEquals(0, file.getLen());
-
- // Validate router failure response matches NN failure response.
- Method m = ClientProtocol.class.getMethod(
- "truncate", String.class, long.class, String.class);
- String badPath = "/unknownlocation/unknowndir";
- compareResponses(routerProtocol, nnProtocol, m,
- new Object[] {badPath, (long) 0, "testclient"});
- }
-
- @Test
- public void testProxyGetBlockLocations() throws Exception {
-
- // Fetch block locations via router
- LocatedBlocks locations =
- routerProtocol.getBlockLocations(routerFile, 0, 1024);
- assertEquals(1, locations.getLocatedBlocks().size());
-
- // Validate router failure response matches NN failure response.
- Method m = ClientProtocol.class.getMethod(
- "getBlockLocations", String.class, long.class, long.class);
- String badPath = "/unknownlocation/unknowndir";
- compareResponses(routerProtocol, nnProtocol,
- m, new Object[] {badPath, (long) 0, (long) 0});
- }
-
- @Test
- public void testProxyStoragePolicy() throws Exception {
-
- // Query initial policy via NN
- HdfsFileStatus status = namenode.getClient().getFileInfo(nnFile);
-
- // Set a random policy via router
- BlockStoragePolicy[] policies = namenode.getClient().getStoragePolicies();
- BlockStoragePolicy policy = policies[0];
-
- while (policy.isCopyOnCreateFile()) {
- // Pick a non copy on create policy
- Random rand = new Random();
- int randIndex = rand.nextInt(policies.length);
- policy = policies[randIndex];
- }
- routerProtocol.setStoragePolicy(routerFile, policy.getName());
-
- // Verify policy via NN
- HdfsFileStatus newStatus = namenode.getClient().getFileInfo(nnFile);
- assertTrue(newStatus.getStoragePolicy() == policy.getId());
- assertTrue(newStatus.getStoragePolicy() != status.getStoragePolicy());
-
- // Validate router failure response matches NN failure response.
- Method m = ClientProtocol.class.getMethod("setStoragePolicy", String.class,
- String.class);
- String badPath = "/unknownlocation/unknowndir";
- compareResponses(routerProtocol, nnProtocol,
- m, new Object[] {badPath, "badpolicy"});
- }
-
- @Test
- public void testProxyGetPreferedBlockSize() throws Exception {
-
- // Query via NN and Router and verify
- long namenodeSize = nnProtocol.getPreferredBlockSize(nnFile);
- long routerSize = routerProtocol.getPreferredBlockSize(routerFile);
- assertEquals(routerSize, namenodeSize);
-
- // Validate router failure response matches NN failure response.
- Method m = ClientProtocol.class.getMethod(
- "getPreferredBlockSize", String.class);
- String badPath = "/unknownlocation/unknowndir";
- compareResponses(
- routerProtocol, nnProtocol, m, new Object[] {badPath});
- }
-
- private void testConcat(
- String source, String target, boolean failureExpected) {
- boolean failure = false;
- try {
- // Concat test file with fill block length file via router
- routerProtocol.concat(target, new String[] {source});
- } catch (IOException ex) {
- failure = true;
- }
- assertEquals(failureExpected, failure);
- }
-
- @Test
- public void testProxyConcatFile() throws Exception {
-
- // Create a stub file in the primary ns
- String sameNameservice = ns;
- String existingFile =
- cluster.getFederatedTestDirectoryForNS(sameNameservice) +
- "_concatfile";
- int existingFileSize = 32;
- createFile(routerFS, existingFile, existingFileSize);
-
- // Identify an alternate nameservice that doesn't match the existing file
- String alternateNameservice = null;
- for (String n : cluster.getNameservices()) {
- if (!n.equals(sameNameservice)) {
- alternateNameservice = n;
- break;
- }
- }
-
- // Create new files, must be a full block to use concat. One file is in the
- // same namespace as the target file, the other is in a different namespace.
- String altRouterFile =
- cluster.getFederatedTestDirectoryForNS(alternateNameservice) +
- "_newfile";
- String sameRouterFile =
- cluster.getFederatedTestDirectoryForNS(sameNameservice) +
- "_newfile";
- createFile(routerFS, altRouterFile, DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
- createFile(routerFS, sameRouterFile, DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
-
- // Concat in different namespaces, fails
- testConcat(existingFile, altRouterFile, true);
-
- // Concat in same namespaces, succeeds
- testConcat(existingFile, sameRouterFile, false);
-
- // Check target file length
- FileStatus status = getFileStatus(routerFS, sameRouterFile);
- assertEquals(
- existingFileSize + DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT,
- status.getLen());
-
- // Validate router failure response matches NN failure response.
- Method m = ClientProtocol.class.getMethod(
- "concat", String.class, String[].class);
- String badPath = "/unknownlocation/unknowndir";
- compareResponses(routerProtocol, nnProtocol, m,
- new Object[] {badPath, new String[] {routerFile}});
- }
-
- @Test
- public void testProxyAppend() throws Exception {
-
- // Append a test string via router
- EnumSet<CreateFlag> createFlag = EnumSet.of(CreateFlag.APPEND);
- DFSClient routerClient = getRouterContext().getClient();
- HdfsDataOutputStream stream =
- routerClient.append(routerFile, 1024, createFlag, null, null);
- stream.writeBytes(TEST_STRING);
- stream.close();
-
- // Verify file size via NN
- FileStatus status = getFileStatus(nnFS, nnFile);
- assertTrue(status.getLen() > TEST_STRING.length());
-
- // Validate router failure response matches NN failure response.
- Method m = ClientProtocol.class.getMethod("append", String.class,
- String.class, EnumSetWritable.class);
- String badPath = "/unknownlocation/unknowndir";
- EnumSetWritable<CreateFlag> createFlagWritable =
- new EnumSetWritable<CreateFlag>(createFlag);
- compareResponses(routerProtocol, nnProtocol, m,
- new Object[] {badPath, "testClient", createFlagWritable});
- }
-
- @Test
- public void testProxyGetAdditionalDatanode()
- throws IOException, InterruptedException, URISyntaxException {
-
- // Use primitive APIs to open a file, add a block, and get datanode location
- EnumSet<CreateFlag> createFlag = EnumSet.of(CreateFlag.CREATE);
- String clientName = getRouterContext().getClient().getClientName();
- String newRouterFile = routerFile + "_additionalDatanode";
- HdfsFileStatus status = routerProtocol.create(
- newRouterFile, new FsPermission("777"), clientName,
- new EnumSetWritable<CreateFlag>(createFlag), true, (short) 1,
- (long) 1024, CryptoProtocolVersion.supported(), null);
-
- // Add a block via router (requires client to have same lease)
- LocatedBlock block = routerProtocol.addBlock(
- newRouterFile, clientName, null, null,
- status.getFileId(), null, null);
-
- DatanodeInfo[] exclusions = new DatanodeInfo[0];
- LocatedBlock newBlock = routerProtocol.getAdditionalDatanode(
- newRouterFile, status.getFileId(), block.getBlock(),
- block.getLocations(), block.getStorageIDs(), exclusions, 1, clientName);
- assertNotNull(newBlock);
- }
-
- @Test
- public void testProxyCreateFileAlternateUser()
- throws IOException, URISyntaxException, InterruptedException {
-
- // Create via Router
- String routerDir = cluster.getFederatedTestDirectoryForNS(ns);
- String namenodeDir = cluster.getNamenodeTestDirectoryForNS(ns);
- String newRouterFile = routerDir + "/unknownuser";
- String newNamenodeFile = namenodeDir + "/unknownuser";
- String username = "unknownuser";
-
- // Allow all user access to dir
- namenode.getFileContext().setPermission(
- new Path(namenodeDir), new FsPermission("777"));
-
- UserGroupInformation ugi = UserGroupInformation.createRemoteUser(username);
- DFSClient client = getRouterContext().getClient(ugi);
- client.create(newRouterFile, true);
-
- // Fetch via NN and check user
- FileStatus status = getFileStatus(nnFS, newNamenodeFile);
- assertEquals(status.getOwner(), username);
- }
-
- @Test
- public void testProxyGetFileInfoAcessException() throws IOException {
-
- UserGroupInformation ugi =
- UserGroupInformation.createRemoteUser("unknownuser");
-
- // List files from the NN and trap the exception
- Exception nnFailure = null;
- try {
- String testFile = cluster.getNamenodeTestFileForNS(ns);
- namenode.getClient(ugi).getLocatedBlocks(testFile, 0);
- } catch (Exception e) {
- nnFailure = e;
- }
- assertNotNull(nnFailure);
-
- // List files from the router and trap the exception
- Exception routerFailure = null;
- try {
- String testFile = cluster.getFederatedTestFileForNS(ns);
- getRouterContext().getClient(ugi).getLocatedBlocks(testFile, 0);
- } catch (Exception e) {
- routerFailure = e;
- }
- assertNotNull(routerFailure);
-
- assertEquals(routerFailure.getClass(), nnFailure.getClass());
- }
-
- @Test
- public void testErasureCoding() throws IOException {
-
- LOG.info("List the available erasurce coding policies");
- ErasureCodingPolicyInfo[] policies = checkErasureCodingPolicies();
- for (ErasureCodingPolicyInfo policy : policies) {
- LOG.info(" {}", policy);
- }
-
- LOG.info("List the erasure coding codecs");
- Map<String, String> codecsRouter = routerProtocol.getErasureCodingCodecs();
- Map<String, String> codecsNamenode = nnProtocol.getErasureCodingCodecs();
- assertTrue(Maps.difference(codecsRouter, codecsNamenode).areEqual());
- for (Entry<String, String> entry : codecsRouter.entrySet()) {
- LOG.info(" {}: {}", entry.getKey(), entry.getValue());
- }
-
- LOG.info("Create a testing directory via the router at the root level");
- String dirPath = "/testec";
- String filePath1 = dirPath + "/testfile1";
- FsPermission permission = new FsPermission("755");
- routerProtocol.mkdirs(dirPath, permission, false);
- createFile(routerFS, filePath1, 32);
- assertTrue(verifyFileExists(routerFS, filePath1));
- DFSClient file1Protocol = getFileDFSClient(filePath1);
-
- LOG.info("The policy for the new file should not be set");
- assertNull(routerProtocol.getErasureCodingPolicy(filePath1));
- assertNull(file1Protocol.getErasureCodingPolicy(filePath1));
-
- String policyName = "RS-6-3-1024k";
- LOG.info("Set policy \"{}\" for \"{}\"", policyName, dirPath);
- routerProtocol.setErasureCodingPolicy(dirPath, policyName);
-
- String filePath2 = dirPath + "/testfile2";
- LOG.info("Create {} in the path with the new EC policy", filePath2);
- createFile(routerFS, filePath2, 32);
- assertTrue(verifyFileExists(routerFS, filePath2));
- DFSClient file2Protocol = getFileDFSClient(filePath2);
-
- LOG.info("Check that the policy is set for {}", filePath2);
- ErasureCodingPolicy policyRouter1 =
- routerProtocol.getErasureCodingPolicy(filePath2);
- ErasureCodingPolicy policyNamenode1 =
- file2Protocol.getErasureCodingPolicy(filePath2);
- assertNotNull(policyRouter1);
- assertEquals(policyName, policyRouter1.getName());
- assertEquals(policyName, policyNamenode1.getName());
-
- LOG.info("Create a new erasure coding policy");
- String newPolicyName = "RS-6-3-128k";
- ECSchema ecSchema = new ECSchema(ErasureCodeConstants.RS_CODEC_NAME, 6, 3);
- ErasureCodingPolicy ecPolicy = new ErasureCodingPolicy(
- newPolicyName,
- ecSchema,
- 128 * 1024,
- (byte) -1);
- ErasureCodingPolicy[] newPolicies = new ErasureCodingPolicy[] {
- ecPolicy
- };
- AddErasureCodingPolicyResponse[] responses =
- routerProtocol.addErasureCodingPolicies(newPolicies);
- assertEquals(1, responses.length);
- assertTrue(responses[0].isSucceed());
- routerProtocol.disableErasureCodingPolicy(newPolicyName);
-
- LOG.info("The new policy should be there and disabled");
- policies = checkErasureCodingPolicies();
- boolean found = false;
- for (ErasureCodingPolicyInfo policy : policies) {
- LOG.info(" {}" + policy);
- if (policy.getPolicy().getName().equals(newPolicyName)) {
- found = true;
- assertEquals(ErasureCodingPolicyState.DISABLED, policy.getState());
- break;
- }
- }
- assertTrue(found);
-
- LOG.info("Set the test folder to use the new policy");
- routerProtocol.enableErasureCodingPolicy(newPolicyName);
- routerProtocol.setErasureCodingPolicy(dirPath, newPolicyName);
-
- LOG.info("Create a file in the path with the new EC policy");
- String filePath3 = dirPath + "/testfile3";
- createFile(routerFS, filePath3, 32);
- assertTrue(verifyFileExists(routerFS, filePath3));
- DFSClient file3Protocol = getFileDFSClient(filePath3);
-
- ErasureCodingPolicy policyRouterFile3 =
- routerProtocol.getErasureCodingPolicy(filePath3);
- assertEquals(newPolicyName, policyRouterFile3.getName());
- ErasureCodingPolicy policyNamenodeFile3 =
- file3Protocol.getErasureCodingPolicy(filePath3);
- assertEquals(newPolicyName, policyNamenodeFile3.getName());
-
- LOG.info("Remove the policy and check the one for the test folder");
- routerProtocol.removeErasureCodingPolicy(newPolicyName);
- ErasureCodingPolicy policyRouter3 =
- routerProtocol.getErasureCodingPolicy(filePath3);
- assertEquals(newPolicyName, policyRouter3.getName());
- ErasureCodingPolicy policyNamenode3 =
- file3Protocol.getErasureCodingPolicy(filePath3);
- assertEquals(newPolicyName, policyNamenode3.getName());
-
- LOG.info("Check the stats");
- ECBlockGroupStats statsRouter = routerProtocol.getECBlockGroupStats();
- ECBlockGroupStats statsNamenode = nnProtocol.getECBlockGroupStats();
- assertEquals(statsNamenode.toString(), statsRouter.toString());
- }
-
- /**
- * Check the erasure coding policies in the Router and the Namenode.
- * @return The erasure coding policies.
- */
- private ErasureCodingPolicyInfo[] checkErasureCodingPolicies()
- throws IOException {
- ErasureCodingPolicyInfo[] policiesRouter =
- routerProtocol.getErasureCodingPolicies();
- assertNotNull(policiesRouter);
- ErasureCodingPolicyInfo[] policiesNamenode =
- nnProtocol.getErasureCodingPolicies();
- Arrays.sort(policiesRouter, EC_POLICY_CMP);
- Arrays.sort(policiesNamenode, EC_POLICY_CMP);
- assertArrayEquals(policiesRouter, policiesNamenode);
- return policiesRouter;
- }
-
- /**
- * Find the Namenode for a particular file and return the DFSClient.
- * @param path Path of the file to check.
- * @return The DFSClient to the Namenode holding the file.
- */
- private DFSClient getFileDFSClient(final String path) {
- for (String nsId : cluster.getNameservices()) {
- LOG.info("Checking {} for {}", nsId, path);
- NamenodeContext nn = cluster.getNamenode(nsId, null);
- try {
- DFSClient nnClientProtocol = nn.getClient();
- if (nnClientProtocol.getFileInfo(path) != null) {
- return nnClientProtocol;
- }
- } catch (Exception ignore) {
- // ignore
- }
- }
- return null;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
deleted file mode 100644
index 5489691..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createFile;
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyFileExists;
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.lang.reflect.Method;
-import java.net.URISyntaxException;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
-import java.util.TreeSet;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.protocol.ClientProtocol;
-import org.apache.hadoop.hdfs.protocol.DirectoryListing;
-import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
-import org.apache.hadoop.hdfs.server.federation.MockResolver;
-import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster;
-import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
-import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
-import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
-import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
-
-/**
- * The the RPC interface of the {@link getRouter()} implemented by
- * {@link RouterRpcServer}.
- */
-public class TestRouterRpcMultiDestination extends TestRouterRpc {
-
- @Override
- public void testSetup() throws Exception {
-
- RouterDFSCluster cluster = getCluster();
-
- // Create mock locations
- getCluster().installMockLocations();
- List<RouterContext> routers = cluster.getRouters();
-
- // Add extra location to the root mount / such that the root mount points:
- // /
- // ns0 -> /
- // ns1 -> /
- for (RouterContext rc : routers) {
- Router router = rc.getRouter();
- MockResolver resolver = (MockResolver) router.getSubclusterResolver();
- resolver.addLocation("/", cluster.getNameservices().get(1), "/");
- }
-
- // Create a mount that points to 2 dirs in the same ns:
- // /same
- // ns0 -> /
- // ns0 -> /target-ns0
- for (RouterContext rc : routers) {
- Router router = rc.getRouter();
- MockResolver resolver = (MockResolver) router.getSubclusterResolver();
- List<String> nss = cluster.getNameservices();
- String ns0 = nss.get(0);
- resolver.addLocation("/same", ns0, "/");
- resolver.addLocation("/same", ns0, cluster.getNamenodePathForNS(ns0));
- }
-
- // Delete all files via the NNs and verify
- cluster.deleteAllFiles();
-
- // Create test fixtures on NN
- cluster.createTestDirectoriesNamenode();
-
- // Wait to ensure NN has fully created its test directories
- Thread.sleep(100);
-
- // Pick a NS, namenode and getRouter() for this test
- RouterContext router = cluster.getRandomRouter();
- this.setRouter(router);
-
- String ns = cluster.getRandomNameservice();
- this.setNs(ns);
- this.setNamenode(cluster.getNamenode(ns, null));
-
- // Create a test file on a single NN that is accessed via a getRouter() path
- // with 2 destinations. All tests should failover to the alternate
- // destination if the wrong NN is attempted first.
- Random r = new Random();
- String randomString = "testfile-" + r.nextInt();
- setNamenodeFile("/" + randomString);
- setRouterFile("/" + randomString);
-
- FileSystem nnFs = getNamenodeFileSystem();
- FileSystem routerFs = getRouterFileSystem();
- createFile(nnFs, getNamenodeFile(), 32);
-
- verifyFileExists(nnFs, getNamenodeFile());
- verifyFileExists(routerFs, getRouterFile());
- }
-
- private void testListing(String path) throws IOException {
-
- // Collect the mount table entries for this path
- Set<String> requiredPaths = new TreeSet<>();
- RouterContext rc = getRouterContext();
- Router router = rc.getRouter();
- FileSubclusterResolver subclusterResolver = router.getSubclusterResolver();
- for (String mount : subclusterResolver.getMountPoints(path)) {
- requiredPaths.add(mount);
- }
-
- // Get files/dirs from the Namenodes
- PathLocation location = subclusterResolver.getDestinationForPath(path);
- for (RemoteLocation loc : location.getDestinations()) {
- String nsId = loc.getNameserviceId();
- String dest = loc.getDest();
- NamenodeContext nn = getCluster().getNamenode(nsId, null);
- FileSystem fs = nn.getFileSystem();
- FileStatus[] files = fs.listStatus(new Path(dest));
- for (FileStatus file : files) {
- String pathName = file.getPath().getName();
- requiredPaths.add(pathName);
- }
- }
-
- // Get files/dirs from the Router
- DirectoryListing listing =
- getRouterProtocol().getListing(path, HdfsFileStatus.EMPTY_NAME, false);
- Iterator<String> requiredPathsIterator = requiredPaths.iterator();
-
- // Match each path returned and verify order returned
- HdfsFileStatus[] partialListing = listing.getPartialListing();
- for (HdfsFileStatus fileStatus : listing.getPartialListing()) {
- String fileName = requiredPathsIterator.next();
- String currentFile = fileStatus.getFullPath(new Path(path)).getName();
- assertEquals(currentFile, fileName);
- }
-
- // Verify the total number of results found/matched
- assertEquals(
- requiredPaths + " doesn't match " + Arrays.toString(partialListing),
- requiredPaths.size(), partialListing.length);
- }
-
- @Override
- public void testProxyListFiles() throws IOException, InterruptedException,
- URISyntaxException, NoSuchMethodException, SecurityException {
-
- // Verify that the root listing is a union of the mount table destinations
- // and the files stored at all nameservices mounted at the root (ns0 + ns1)
- // / -->
- // /ns0 (from mount table)
- // /ns1 (from mount table)
- // /same (from the mount table)
- // all items in / of ns0 from mapping of / -> ns0:::/)
- // all items in / of ns1 from mapping of / -> ns1:::/)
- testListing("/");
-
- // Verify that the "/same" mount point lists the contents of both dirs in
- // the same ns
- // /same -->
- // /target-ns0 (from root of ns0)
- // /testdir (from contents of /target-ns0)
- testListing("/same");
-
- // List a non-existing path and validate error response with NN behavior
- ClientProtocol namenodeProtocol =
- getCluster().getRandomNamenode().getClient().getNamenode();
- Method m = ClientProtocol.class.getMethod(
- "getListing", String.class, byte[].class, boolean.class);
- String badPath = "/unknownlocation/unknowndir";
- compareResponses(getRouterProtocol(), namenodeProtocol, m,
- new Object[] {badPath, HdfsFileStatus.EMPTY_NAME, false});
- }
-
- @Override
- public void testProxyRenameFiles() throws IOException, InterruptedException {
-
- super.testProxyRenameFiles();
-
- List<String> nss = getCluster().getNameservices();
- String ns0 = nss.get(0);
- String ns1 = nss.get(1);
-
- // Rename a file from ns0 into the root (mapped to both ns0 and ns1)
- String testDir0 = getCluster().getFederatedTestDirectoryForNS(ns0);
- String filename0 = testDir0 + "/testrename";
- String renamedFile = "/testrename";
- testRename(getRouterContext(), filename0, renamedFile, false);
- testRename2(getRouterContext(), filename0, renamedFile, false);
-
- // Rename a file from ns1 into the root (mapped to both ns0 and ns1)
- String testDir1 = getCluster().getFederatedTestDirectoryForNS(ns1);
- String filename1 = testDir1 + "/testrename";
- testRename(getRouterContext(), filename1, renamedFile, false);
- testRename2(getRouterContext(), filename1, renamedFile, false);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterSafemode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterSafemode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterSafemode.java
deleted file mode 100644
index e05f727..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterSafemode.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_CACHE_TIME_TO_LIVE_MS;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_SAFEMODE_EXPIRATION;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_SAFEMODE_EXTENSION;
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.deleteStateStore;
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.getStateStoreConfiguration;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
-import org.apache.hadoop.service.Service.STATE;
-import org.apache.hadoop.util.Time;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- * Test the safe mode for the {@link Router} controlled by
- * {@link RouterSafemodeService}.
- */
-public class TestRouterSafemode {
-
- private Router router;
- private static Configuration conf;
-
- @BeforeClass
- public static void create() throws IOException {
- // Wipe state store
- deleteStateStore();
- // Configuration that supports the state store
- conf = getStateStoreConfiguration();
- // 2 sec startup standby
- conf.setTimeDuration(DFS_ROUTER_SAFEMODE_EXTENSION,
- TimeUnit.SECONDS.toMillis(2), TimeUnit.MILLISECONDS);
- // 1 sec cache refresh
- conf.setTimeDuration(DFS_ROUTER_CACHE_TIME_TO_LIVE_MS,
- TimeUnit.SECONDS.toMillis(1), TimeUnit.MILLISECONDS);
- // 2 sec post cache update before entering safemode (2 intervals)
- conf.setTimeDuration(DFS_ROUTER_SAFEMODE_EXPIRATION,
- TimeUnit.SECONDS.toMillis(2), TimeUnit.MILLISECONDS);
-
- conf.set(DFSConfigKeys.DFS_ROUTER_RPC_BIND_HOST_KEY, "0.0.0.0");
- conf.set(DFSConfigKeys.DFS_ROUTER_RPC_ADDRESS_KEY, "127.0.0.1:0");
- conf.set(DFSConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_KEY, "127.0.0.1:0");
- conf.set(DFSConfigKeys.DFS_ROUTER_ADMIN_BIND_HOST_KEY, "0.0.0.0");
- conf.set(DFSConfigKeys.DFS_ROUTER_HTTP_ADDRESS_KEY, "127.0.0.1:0");
- conf.set(DFSConfigKeys.DFS_ROUTER_HTTPS_ADDRESS_KEY, "127.0.0.1:0");
-
- // RPC + State Store + Safe Mode only
- conf = new RouterConfigBuilder(conf)
- .rpc()
- .safemode()
- .stateStore()
- .metrics()
- .build();
- }
-
- @AfterClass
- public static void destroy() {
- }
-
- @Before
- public void setup() throws IOException, URISyntaxException {
- router = new Router();
- router.init(conf);
- router.start();
- }
-
- @After
- public void cleanup() throws IOException {
- if (router != null) {
- router.stop();
- router = null;
- }
- }
-
- @Test
- public void testSafemodeService() throws IOException {
- RouterSafemodeService server = new RouterSafemodeService(router);
- server.init(conf);
- assertEquals(STATE.INITED, server.getServiceState());
- server.start();
- assertEquals(STATE.STARTED, server.getServiceState());
- server.stop();
- assertEquals(STATE.STOPPED, server.getServiceState());
- server.close();
- }
-
- @Test
- public void testRouterExitSafemode()
- throws InterruptedException, IllegalStateException, IOException {
-
- assertTrue(router.getRpcServer().isInSafeMode());
- verifyRouter(RouterServiceState.SAFEMODE);
-
- // Wait for initial time in milliseconds
- long interval =
- conf.getTimeDuration(DFS_ROUTER_SAFEMODE_EXTENSION,
- TimeUnit.SECONDS.toMillis(2), TimeUnit.MILLISECONDS) +
- conf.getTimeDuration(DFS_ROUTER_CACHE_TIME_TO_LIVE_MS,
- TimeUnit.SECONDS.toMillis(1), TimeUnit.MILLISECONDS);
- Thread.sleep(interval);
-
- assertFalse(router.getRpcServer().isInSafeMode());
- verifyRouter(RouterServiceState.RUNNING);
- }
-
- @Test
- public void testRouterEnterSafemode()
- throws IllegalStateException, IOException, InterruptedException {
-
- // Verify starting state
- assertTrue(router.getRpcServer().isInSafeMode());
- verifyRouter(RouterServiceState.SAFEMODE);
-
- // We should be in safe mode for DFS_ROUTER_SAFEMODE_EXTENSION time
- long interval0 = conf.getTimeDuration(DFS_ROUTER_SAFEMODE_EXTENSION,
- TimeUnit.SECONDS.toMillis(2), TimeUnit.MILLISECONDS) - 1000;
- long t0 = Time.now();
- while (Time.now() - t0 < interval0) {
- verifyRouter(RouterServiceState.SAFEMODE);
- Thread.sleep(100);
- }
-
- // We wait some time for the state to propagate
- long interval1 = 1000 + 2 * conf.getTimeDuration(
- DFS_ROUTER_CACHE_TIME_TO_LIVE_MS, TimeUnit.SECONDS.toMillis(1),
- TimeUnit.MILLISECONDS);
- Thread.sleep(interval1);
-
- // Running
- assertFalse(router.getRpcServer().isInSafeMode());
- verifyRouter(RouterServiceState.RUNNING);
-
- // Disable cache
- router.getStateStore().stopCacheUpdateService();
-
- // Wait until the State Store cache is stale in milliseconds
- long interval2 =
- conf.getTimeDuration(DFS_ROUTER_SAFEMODE_EXPIRATION,
- TimeUnit.SECONDS.toMillis(2), TimeUnit.MILLISECONDS) +
- conf.getTimeDuration(DFS_ROUTER_CACHE_TIME_TO_LIVE_MS,
- TimeUnit.SECONDS.toMillis(1), TimeUnit.MILLISECONDS);
- Thread.sleep(interval2);
-
- // Safemode
- assertTrue(router.getRpcServer().isInSafeMode());
- verifyRouter(RouterServiceState.SAFEMODE);
- }
-
- @Test
- public void testRouterRpcSafeMode()
- throws IllegalStateException, IOException {
-
- assertTrue(router.getRpcServer().isInSafeMode());
- verifyRouter(RouterServiceState.SAFEMODE);
-
- // If the Router is in Safe Mode, we should get a SafeModeException
- boolean exception = false;
- try {
- router.getRpcServer().delete("/testfile.txt", true);
- fail("We should have thrown a safe mode exception");
- } catch (RouterSafeModeException sme) {
- exception = true;
- }
- assertTrue("We should have thrown a safe mode exception", exception);
- }
-
- private void verifyRouter(RouterServiceState status)
- throws IllegalStateException, IOException {
- assertEquals(status, router.getRouterState());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
deleted file mode 100644
index def3935..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
+++ /dev/null
@@ -1,274 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store;
-
-import static org.apache.hadoop.hdfs.DFSConfigKeys.FEDERATION_STORE_DRIVER_CLASS;
-import static org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl.FEDERATION_STORE_FILE_DIRECTORY;
-import static org.junit.Assert.assertNotNull;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
-import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileBaseImpl;
-import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl;
-import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
-import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
-import org.apache.hadoop.hdfs.server.federation.store.records.MembershipStats;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-import org.apache.hadoop.util.Time;
-
-/**
- * Utilities to test the State Store.
- */
-public final class FederationStateStoreTestUtils {
-
- /** The State Store Driver implementation class for testing .*/
- private static final Class<? extends StateStoreDriver>
- FEDERATION_STORE_DRIVER_CLASS_FOR_TEST = StateStoreFileImpl.class;
-
- private FederationStateStoreTestUtils() {
- // Utility Class
- }
-
- /**
- * Get the State Store driver implementation for testing.
- *
- * @return Class of the State Store driver implementation.
- */
- public static Class<? extends StateStoreDriver> getTestDriverClass() {
- return FEDERATION_STORE_DRIVER_CLASS_FOR_TEST;
- }
-
- /**
- * Create a default State Store configuration.
- *
- * @return State Store configuration.
- */
- public static Configuration getStateStoreConfiguration() {
- Class<? extends StateStoreDriver> clazz = getTestDriverClass();
- return getStateStoreConfiguration(clazz);
- }
-
- /**
- * Create a new State Store configuration for a particular driver.
- *
- * @param clazz Class of the driver to create.
- * @return State Store configuration.
- */
- public static Configuration getStateStoreConfiguration(
- Class<? extends StateStoreDriver> clazz) {
- Configuration conf = new HdfsConfiguration(false);
-
- conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, true);
- conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, "hdfs://test");
-
- conf.setClass(FEDERATION_STORE_DRIVER_CLASS, clazz, StateStoreDriver.class);
-
- if (clazz.isAssignableFrom(StateStoreFileBaseImpl.class)) {
- setFileConfiguration(conf);
- }
- return conf;
- }
-
- /**
- * Create a new State Store based on a configuration.
- *
- * @param configuration Configuration for the State Store.
- * @return New State Store service.
- * @throws IOException If it cannot create the State Store.
- * @throws InterruptedException If we cannot wait for the store to start.
- */
- public static StateStoreService newStateStore(
- Configuration configuration) throws IOException, InterruptedException {
-
- StateStoreService stateStore = new StateStoreService();
- assertNotNull(stateStore);
-
- // Set unique identifier, this is normally the router address
- String identifier = UUID.randomUUID().toString();
- stateStore.setIdentifier(identifier);
-
- stateStore.init(configuration);
- stateStore.start();
-
- // Wait for state store to connect
- waitStateStore(stateStore, TimeUnit.SECONDS.toMillis(10));
-
- return stateStore;
- }
-
- /**
- * Wait for the State Store to initialize its driver.
- *
- * @param stateStore State Store.
- * @param timeoutMs Time out in milliseconds.
- * @throws IOException If the State Store cannot be reached.
- * @throws InterruptedException If the sleep is interrupted.
- */
- public static void waitStateStore(StateStoreService stateStore,
- long timeoutMs) throws IOException, InterruptedException {
- long startingTime = Time.monotonicNow();
- while (!stateStore.isDriverReady()) {
- Thread.sleep(100);
- if (Time.monotonicNow() - startingTime > timeoutMs) {
- throw new IOException("Timeout waiting for State Store to connect");
- }
- }
- }
-
- /**
- * Delete the default State Store.
- *
- * @throws IOException
- */
- public static void deleteStateStore() throws IOException {
- Class<? extends StateStoreDriver> driverClass = getTestDriverClass();
- deleteStateStore(driverClass);
- }
-
- /**
- * Delete the State Store.
- * @param driverClass Class of the State Store driver implementation.
- * @throws IOException If it cannot be deleted.
- */
- public static void deleteStateStore(
- Class<? extends StateStoreDriver> driverClass) throws IOException {
-
- if (StateStoreFileBaseImpl.class.isAssignableFrom(driverClass)) {
- String workingDirectory = System.getProperty("user.dir");
- File dir = new File(workingDirectory + "/statestore");
- if (dir.exists()) {
- FileUtils.cleanDirectory(dir);
- }
- }
- }
-
- /**
- * Set the default configuration for drivers based on files.
- *
- * @param conf Configuration to extend.
- */
- public static void setFileConfiguration(Configuration conf) {
- String workingPath = System.getProperty("user.dir");
- String stateStorePath = workingPath + "/statestore";
- conf.set(FEDERATION_STORE_FILE_DIRECTORY, stateStorePath);
- }
-
- /**
- * Clear all the records from the State Store.
- *
- * @param store State Store to remove records from.
- * @return If the State Store was cleared.
- * @throws IOException If it cannot clear the State Store.
- */
- public static boolean clearAllRecords(StateStoreService store)
- throws IOException {
- Collection<Class<? extends BaseRecord>> allRecords =
- store.getSupportedRecords();
- for (Class<? extends BaseRecord> recordType : allRecords) {
- if (!clearRecords(store, recordType)) {
- return false;
- }
- }
- return true;
- }
-
- /**
- * Clear records from a certain type from the State Store.
- *
- * @param store State Store to remove records from.
- * @param recordClass Class of the records to remove.
- * @return If the State Store was cleared.
- * @throws IOException If it cannot clear the State Store.
- */
- public static <T extends BaseRecord> boolean clearRecords(
- StateStoreService store, Class<T> recordClass) throws IOException {
- List<T> emptyList = new ArrayList<>();
- if (!synchronizeRecords(store, emptyList, recordClass)) {
- return false;
- }
- store.refreshCaches(true);
- return true;
- }
-
- /**
- * Synchronize a set of records. Remove all and keep the ones specified.
- *
- * @param stateStore State Store service managing the driver.
- * @param records Records to add.
- * @param clazz Class of the record to synchronize.
- * @return If the synchronization succeeded.
- * @throws IOException If it cannot connect to the State Store.
- */
- public static <T extends BaseRecord> boolean synchronizeRecords(
- StateStoreService stateStore, List<T> records, Class<T> clazz)
- throws IOException {
- StateStoreDriver driver = stateStore.getDriver();
- driver.verifyDriverReady();
- if (driver.removeAll(clazz)) {
- if (driver.putAll(records, true, false)) {
- return true;
- }
- }
- return false;
- }
-
- public static List<MountTable> createMockMountTable(
- List<String> nameservices) throws IOException {
- // create table entries
- List<MountTable> entries = new ArrayList<>();
- for (String ns : nameservices) {
- Map<String, String> destMap = new HashMap<>();
- destMap.put(ns, "/target-" + ns);
- MountTable entry = MountTable.newInstance("/" + ns, destMap);
- entries.add(entry);
- }
- return entries;
- }
-
- public static MembershipState createMockRegistrationForNamenode(
- String nameserviceId, String namenodeId,
- FederationNamenodeServiceState state) throws IOException {
- MembershipState entry = MembershipState.newInstance(
- "routerId", nameserviceId, namenodeId, "clusterId", "test",
- "0.0.0.0:0", "0.0.0.0:0", "0.0.0.0:0", "0.0.0.0:0", state, false);
- MembershipStats stats = MembershipStats.newInstance();
- stats.setNumOfActiveDatanodes(100);
- stats.setNumOfDeadDatanodes(10);
- stats.setNumOfDecommissioningDatanodes(20);
- stats.setNumOfDecomActiveDatanodes(15);
- stats.setNumOfDecomDeadDatanodes(5);
- stats.setNumOfBlocks(10);
- entry.setStats(stats);
- return entry;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreBase.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreBase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreBase.java
deleted file mode 100644
index 7f6704e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreBase.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store;
-
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.newStateStore;
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.getStateStoreConfiguration;
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.waitStateStore;
-import static org.junit.Assert.assertNotNull;
-
-import java.io.IOException;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-
-/**
- * Test the basic {@link StateStoreService} {@link MountTableStore}
- * functionality.
- */
-public class TestStateStoreBase {
-
- private static StateStoreService stateStore;
- private static Configuration conf;
-
- protected static StateStoreService getStateStore() {
- return stateStore;
- }
-
- protected static Configuration getConf() {
- return conf;
- }
-
- @BeforeClass
- public static void createBase() throws IOException, InterruptedException {
-
- conf = getStateStoreConfiguration();
-
- // Disable auto-reconnect to data store
- conf.setLong(DFSConfigKeys.FEDERATION_STORE_CONNECTION_TEST_MS,
- TimeUnit.HOURS.toMillis(1));
- }
-
- @AfterClass
- public static void destroyBase() throws Exception {
- if (stateStore != null) {
- stateStore.stop();
- stateStore.close();
- stateStore = null;
- }
- }
-
- @Before
- public void setupBase() throws IOException, InterruptedException,
- InstantiationException, IllegalAccessException {
- if (stateStore == null) {
- stateStore = newStateStore(conf);
- assertNotNull(stateStore);
- }
- // Wait for state store to connect
- stateStore.loadDriver();
- waitStateStore(stateStore, TimeUnit.SECONDS.toMillis(10));
- }
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
new file mode 100644
index 0000000..a1adf77
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
@@ -0,0 +1,362 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HEARTBEAT_INTERVAL_MS;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HEARTBEAT_INTERVAL_MS_DEFAULT;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ha.HAServiceProtocol;
+import org.apache.hadoop.ha.HAServiceStatus;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.NameNodeProxies;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport;
+import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
+import org.apache.hadoop.hdfs.tools.NNHAServiceTarget;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The {@link Router} periodically checks the state of a Namenode (usually on
+ * the same server) and reports their high availability (HA) state and
+ * load/space status to the
+ * {@link org.apache.hadoop.hdfs.server.federation.store.StateStoreService}
+ * . Note that this is an optional role as a Router can be independent of any
+ * subcluster.
+ * <p>
+ * For performance with Namenode HA, the Router uses the high availability state
+ * information in the State Store to forward the request to the Namenode that is
+ * most likely to be active.
+ * <p>
+ * Note that this service can be embedded into the Namenode itself to simplify
+ * the operation.
+ */
+public class NamenodeHeartbeatService extends PeriodicService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NamenodeHeartbeatService.class);
+
+
+ /** Configuration for the heartbeat. */
+ private Configuration conf;
+
+ /** Router performing the heartbeating. */
+ private final ActiveNamenodeResolver resolver;
+
+ /** Interface to the tracked NN. */
+ private final String nameserviceId;
+ private final String namenodeId;
+
+ /** Namenode HA target. */
+ private NNHAServiceTarget localTarget;
+ /** RPC address for the namenode. */
+ private String rpcAddress;
+ /** Service RPC address for the namenode. */
+ private String serviceAddress;
+ /** Service RPC address for the namenode. */
+ private String lifelineAddress;
+ /** HTTP address for the namenode. */
+ private String webAddress;
+
+ /**
+ * Create a new Namenode status updater.
+ * @param resolver Namenode resolver service to handle NN registration.
+ * @param nsId Identifier of the nameservice.
+ * @param nnId Identifier of the namenode in HA.
+ */
+ public NamenodeHeartbeatService(
+ ActiveNamenodeResolver resolver, String nsId, String nnId) {
+ super(NamenodeHeartbeatService.class.getSimpleName() +
+ (nsId == null ? "" : " " + nsId) +
+ (nnId == null ? "" : " " + nnId));
+
+ this.resolver = resolver;
+
+ this.nameserviceId = nsId;
+ this.namenodeId = nnId;
+
+ }
+
+ @Override
+ protected void serviceInit(Configuration configuration) throws Exception {
+
+ this.conf = configuration;
+
+ String nnDesc = nameserviceId;
+ if (this.namenodeId != null && !this.namenodeId.isEmpty()) {
+ this.localTarget = new NNHAServiceTarget(
+ conf, nameserviceId, namenodeId);
+ nnDesc += "-" + namenodeId;
+ } else {
+ this.localTarget = null;
+ }
+
+ // Get the RPC address for the clients to connect
+ this.rpcAddress = getRpcAddress(conf, nameserviceId, namenodeId);
+ LOG.info("{} RPC address: {}", nnDesc, rpcAddress);
+
+ // Get the Service RPC address for monitoring
+ this.serviceAddress =
+ DFSUtil.getNamenodeServiceAddr(conf, nameserviceId, namenodeId);
+ if (this.serviceAddress == null) {
+ LOG.error("Cannot locate RPC service address for NN {}, " +
+ "using RPC address {}", nnDesc, this.rpcAddress);
+ this.serviceAddress = this.rpcAddress;
+ }
+ LOG.info("{} Service RPC address: {}", nnDesc, serviceAddress);
+
+ // Get the Lifeline RPC address for faster monitoring
+ this.lifelineAddress =
+ DFSUtil.getNamenodeLifelineAddr(conf, nameserviceId, namenodeId);
+ if (this.lifelineAddress == null) {
+ this.lifelineAddress = this.serviceAddress;
+ }
+ LOG.info("{} Lifeline RPC address: {}", nnDesc, lifelineAddress);
+
+ // Get the Web address for UI
+ this.webAddress =
+ DFSUtil.getNamenodeWebAddr(conf, nameserviceId, namenodeId);
+ LOG.info("{} Web address: {}", nnDesc, webAddress);
+
+ this.setIntervalMs(conf.getLong(
+ DFS_ROUTER_HEARTBEAT_INTERVAL_MS,
+ DFS_ROUTER_HEARTBEAT_INTERVAL_MS_DEFAULT));
+
+
+ super.serviceInit(configuration);
+ }
+
+ @Override
+ public void periodicInvoke() {
+ updateState();
+ }
+
+ /**
+ * Get the RPC address for a Namenode.
+ * @param conf Configuration.
+ * @param nsId Name service identifier.
+ * @param nnId Name node identifier.
+ * @return RPC address in format hostname:1234.
+ */
+ private static String getRpcAddress(
+ Configuration conf, String nsId, String nnId) {
+
+ // Get it from the regular RPC setting
+ String confKey = DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY;
+ String ret = conf.get(confKey);
+
+ if (nsId != null || nnId != null) {
+ // Get if for the proper nameservice and namenode
+ confKey = DFSUtil.addKeySuffixes(confKey, nsId, nnId);
+ ret = conf.get(confKey);
+
+ // If not available, get it from the map
+ if (ret == null) {
+ Map<String, InetSocketAddress> rpcAddresses =
+ DFSUtil.getRpcAddressesForNameserviceId(conf, nsId, null);
+ InetSocketAddress sockAddr = null;
+ if (nnId != null) {
+ sockAddr = rpcAddresses.get(nnId);
+ } else if (rpcAddresses.size() == 1) {
+ // Get the only namenode in the namespace
+ sockAddr = rpcAddresses.values().iterator().next();
+ }
+ if (sockAddr != null) {
+ InetAddress addr = sockAddr.getAddress();
+ ret = addr.getHostName() + ":" + sockAddr.getPort();
+ }
+ }
+ }
+ return ret;
+ }
+
+ /**
+ * Update the state of the Namenode.
+ */
+ private void updateState() {
+ NamenodeStatusReport report = getNamenodeStatusReport();
+ if (!report.registrationValid()) {
+ // Not operational
+ LOG.error("Namenode is not operational: {}", getNamenodeDesc());
+ } else if (report.haStateValid()) {
+ // block and HA status available
+ LOG.debug("Received service state: {} from HA namenode: {}",
+ report.getState(), getNamenodeDesc());
+ } else if (localTarget == null) {
+ // block info available, HA status not expected
+ LOG.debug(
+ "Reporting non-HA namenode as operational: " + getNamenodeDesc());
+ } else {
+ // block info available, HA status should be available, but was not
+ // fetched do nothing and let the current state stand
+ return;
+ }
+ try {
+ if (!resolver.registerNamenode(report)) {
+ LOG.warn("Cannot register namenode {}", report);
+ }
+ } catch (IOException e) {
+ LOG.info("Cannot register namenode in the State Store");
+ } catch (Exception ex) {
+ LOG.error("Unhandled exception updating NN registration for {}",
+ getNamenodeDesc(), ex);
+ }
+ }
+
+ /**
+ * Get the status report for the Namenode monitored by this heartbeater.
+ * @return Namenode status report.
+ */
+ protected NamenodeStatusReport getNamenodeStatusReport() {
+ NamenodeStatusReport report = new NamenodeStatusReport(nameserviceId,
+ namenodeId, rpcAddress, serviceAddress, lifelineAddress, webAddress);
+
+ try {
+ LOG.debug("Probing NN at service address: {}", serviceAddress);
+
+ URI serviceURI = new URI("hdfs://" + serviceAddress);
+ // Read the filesystem info from RPC (required)
+ NamenodeProtocol nn = NameNodeProxies
+ .createProxy(this.conf, serviceURI, NamenodeProtocol.class)
+ .getProxy();
+
+ if (nn != null) {
+ NamespaceInfo info = nn.versionRequest();
+ if (info != null) {
+ report.setNamespaceInfo(info);
+ }
+ }
+ if (!report.registrationValid()) {
+ return report;
+ }
+
+ // Check for safemode from the client protocol. Currently optional, but
+ // should be required at some point for QoS
+ try {
+ ClientProtocol client = NameNodeProxies
+ .createProxy(this.conf, serviceURI, ClientProtocol.class)
+ .getProxy();
+ if (client != null) {
+ boolean isSafeMode = client.setSafeMode(
+ SafeModeAction.SAFEMODE_GET, false);
+ report.setSafeMode(isSafeMode);
+ }
+ } catch (Exception e) {
+ LOG.error("Cannot fetch safemode state for {}", getNamenodeDesc(), e);
+ }
+
+ // Read the stats from JMX (optional)
+ updateJMXParameters(webAddress, report);
+
+ if (localTarget != null) {
+ // Try to get the HA status
+ try {
+ // Determine if NN is active
+ // TODO: dynamic timeout
+ HAServiceProtocol haProtocol = localTarget.getProxy(conf, 30*1000);
+ HAServiceStatus status = haProtocol.getServiceStatus();
+ report.setHAServiceState(status.getState());
+ } catch (Throwable e) {
+ if (e.getMessage().startsWith("HA for namenode is not enabled")) {
+ LOG.error("HA for {} is not enabled", getNamenodeDesc());
+ localTarget = null;
+ } else {
+ // Failed to fetch HA status, ignoring failure
+ LOG.error("Cannot fetch HA status for {}: {}",
+ getNamenodeDesc(), e.getMessage(), e);
+ }
+ }
+ }
+ } catch(IOException e) {
+ LOG.error("Cannot communicate with {}: {}",
+ getNamenodeDesc(), e.getMessage());
+ } catch(Throwable e) {
+ // Generic error that we don't know about
+ LOG.error("Unexpected exception while communicating with {}: {}",
+ getNamenodeDesc(), e.getMessage(), e);
+ }
+ return report;
+ }
+
+ /**
+ * Get the description of the Namenode to monitor.
+ * @return Description of the Namenode to monitor.
+ */
+ public String getNamenodeDesc() {
+ if (namenodeId != null && !namenodeId.isEmpty()) {
+ return nameserviceId + "-" + namenodeId + ":" + serviceAddress;
+ } else {
+ return nameserviceId + ":" + serviceAddress;
+ }
+ }
+
+ /**
+ * Get the parameters for a Namenode from JMX and add them to the report.
+ * @param address Web interface of the Namenode to monitor.
+ * @param report Namenode status report to update with JMX data.
+ */
+ private void updateJMXParameters(
+ String address, NamenodeStatusReport report) {
+ try {
+ // TODO part of this should be moved to its own utility
+ String query = "Hadoop:service=NameNode,name=FSNamesystem*";
+ JSONArray aux = FederationUtil.getJmx(query, address);
+ if (aux != null) {
+ for (int i = 0; i < aux.length(); i++) {
+ JSONObject jsonObject = aux.getJSONObject(i);
+ String name = jsonObject.getString("name");
+ if (name.equals("Hadoop:service=NameNode,name=FSNamesystemState")) {
+ report.setDatanodeInfo(
+ jsonObject.getInt("NumLiveDataNodes"),
+ jsonObject.getInt("NumDeadDataNodes"),
+ jsonObject.getInt("NumDecommissioningDataNodes"),
+ jsonObject.getInt("NumDecomLiveDataNodes"),
+ jsonObject.getInt("NumDecomDeadDataNodes"));
+ } else if (name.equals(
+ "Hadoop:service=NameNode,name=FSNamesystem")) {
+ report.setNamesystemInfo(
+ jsonObject.getLong("CapacityRemaining"),
+ jsonObject.getLong("CapacityTotal"),
+ jsonObject.getLong("FilesTotal"),
+ jsonObject.getLong("BlocksTotal"),
+ jsonObject.getLong("MissingBlocks"),
+ jsonObject.getLong("PendingReplicationBlocks"),
+ jsonObject.getLong("UnderReplicatedBlocks"),
+ jsonObject.getLong("PendingDeletionBlocks"),
+ jsonObject.getLong("ProvidedCapacityTotal"));
+ }
+ }
+ }
+ } catch (Exception e) {
+ LOG.error("Cannot get stat from {} using JMX", getNamenodeDesc(), e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java
new file mode 100644
index 0000000..5e12222
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java
@@ -0,0 +1,198 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.service.ServiceStateException;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+/**
+ * Service to periodically execute a runnable.
+ */
+public abstract class PeriodicService extends AbstractService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(PeriodicService.class);
+
+ /** Default interval in milliseconds for the periodic service. */
+ private static final long DEFAULT_INTERVAL_MS = TimeUnit.MINUTES.toMillis(1);
+
+
+ /** Interval for running the periodic service in milliseconds. */
+ private long intervalMs;
+ /** Name of the service. */
+ private final String serviceName;
+
+ /** Scheduler for the periodic service. */
+ private final ScheduledExecutorService scheduler;
+
+ /** If the service is running. */
+ private volatile boolean isRunning = false;
+
+ /** How many times we run. */
+ private long runCount;
+ /** How many errors we got. */
+ private long errorCount;
+ /** When was the last time we executed this service successfully. */
+ private long lastRun;
+
+ /**
+ * Create a new periodic update service.
+ *
+ * @param name Name of the service.
+ */
+ public PeriodicService(String name) {
+ this(name, DEFAULT_INTERVAL_MS);
+ }
+
+ /**
+ * Create a new periodic update service.
+ *
+ * @param name Name of the service.
+ * @param interval Interval for the periodic service in milliseconds.
+ */
+ public PeriodicService(String name, long interval) {
+ super(name);
+ this.serviceName = name;
+ this.intervalMs = interval;
+
+ ThreadFactory threadFactory = new ThreadFactoryBuilder()
+ .setNameFormat(this.getName() + "-%d")
+ .build();
+ this.scheduler = Executors.newScheduledThreadPool(1, threadFactory);
+ }
+
+ /**
+ * Set the interval for the periodic service.
+ *
+ * @param interval Interval in milliseconds.
+ */
+ protected void setIntervalMs(long interval) {
+ if (getServiceState() == STATE.STARTED) {
+ throw new ServiceStateException("Periodic service already started");
+ } else {
+ this.intervalMs = interval;
+ }
+ }
+
+ /**
+ * Get the interval for the periodic service.
+ *
+ * @return Interval in milliseconds.
+ */
+ protected long getIntervalMs() {
+ return this.intervalMs;
+ }
+
+ /**
+ * Get how many times we failed to run the periodic service.
+ *
+ * @return Times we failed to run the periodic service.
+ */
+ protected long getErrorCount() {
+ return this.errorCount;
+ }
+
+ /**
+ * Get how many times we run the periodic service.
+ *
+ * @return Times we run the periodic service.
+ */
+ protected long getRunCount() {
+ return this.runCount;
+ }
+
+ /**
+ * Get the last time the periodic service was executed.
+ *
+ * @return Last time the periodic service was executed.
+ */
+ protected long getLastUpdate() {
+ return this.lastRun;
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ super.serviceStart();
+ LOG.info("Starting periodic service {}", this.serviceName);
+ startPeriodic();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ stopPeriodic();
+ LOG.info("Stopping periodic service {}", this.serviceName);
+ super.serviceStop();
+ }
+
+ /**
+ * Stop the periodic task.
+ */
+ protected synchronized void stopPeriodic() {
+ if (this.isRunning) {
+ LOG.info("{} is shutting down", this.serviceName);
+ this.isRunning = false;
+ this.scheduler.shutdownNow();
+ }
+ }
+
+ /**
+ * Start the periodic execution.
+ */
+ protected synchronized void startPeriodic() {
+ stopPeriodic();
+
+ // Create the runnable service
+ Runnable updateRunnable = new Runnable() {
+ @Override
+ public void run() {
+ LOG.debug("Running {} update task", serviceName);
+ try {
+ if (!isRunning) {
+ return;
+ }
+ periodicInvoke();
+ runCount++;
+ lastRun = Time.now();
+ } catch (Exception ex) {
+ errorCount++;
+ LOG.warn(serviceName + " service threw an exception", ex);
+ }
+ }
+ };
+
+ // Start the execution of the periodic service
+ this.isRunning = true;
+ this.scheduler.scheduleWithFixedDelay(
+ updateRunnable, 0, this.intervalMs, TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Method that the service will run periodically.
+ */
+ protected abstract void periodicInvoke();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java
new file mode 100644
index 0000000..dbb6ffa
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java
@@ -0,0 +1,208 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.fs.QuotaUsage;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Module that implements the quota relevant RPC calls
+ * {@link ClientProtocol#setQuota(String, long, long, StorageType)}
+ * and
+ * {@link ClientProtocol#getQuotaUsage(String)}
+ * in the {@link RouterRpcServer}.
+ */
+public class Quota {
+ private static final Logger LOG = LoggerFactory.getLogger(Quota.class);
+
+ /** RPC server to receive client calls. */
+ private final RouterRpcServer rpcServer;
+ /** RPC clients to connect to the Namenodes. */
+ private final RouterRpcClient rpcClient;
+ /** Router used in RouterRpcServer. */
+ private final Router router;
+
+ public Quota(Router router, RouterRpcServer server) {
+ this.router = router;
+ this.rpcServer = server;
+ this.rpcClient = server.getRPCClient();
+ }
+
+ /**
+ * Set quota for the federation path.
+ * @param path Federation path.
+ * @param namespaceQuota Name space quota.
+ * @param storagespaceQuota Storage space quota.
+ * @param type StorageType that the space quota is intended to be set on.
+ * @throws IOException
+ */
+ public void setQuota(String path, long namespaceQuota,
+ long storagespaceQuota, StorageType type) throws IOException {
+ rpcServer.checkOperation(OperationCategory.WRITE);
+
+ // Set quota for current path and its children mount table path.
+ final List<RemoteLocation> locations = getQuotaRemoteLocations(path);
+ if (LOG.isDebugEnabled()) {
+ for (RemoteLocation loc : locations) {
+ LOG.debug("Set quota for path: nsId: {}, dest: {}.",
+ loc.getNameserviceId(), loc.getDest());
+ }
+ }
+
+ RemoteMethod method = new RemoteMethod("setQuota",
+ new Class<?>[] {String.class, long.class, long.class,
+ StorageType.class},
+ new RemoteParam(), namespaceQuota, storagespaceQuota, type);
+ rpcClient.invokeConcurrent(locations, method, false, false);
+ }
+
+ /**
+ * Get quota usage for the federation path.
+ * @param path Federation path.
+ * @return Aggregated quota.
+ * @throws IOException
+ */
+ public QuotaUsage getQuotaUsage(String path) throws IOException {
+ final List<RemoteLocation> quotaLocs = getValidQuotaLocations(path);
+ RemoteMethod method = new RemoteMethod("getQuotaUsage",
+ new Class<?>[] {String.class}, new RemoteParam());
+ Map<RemoteLocation, QuotaUsage> results = rpcClient.invokeConcurrent(
+ quotaLocs, method, true, false, QuotaUsage.class);
+
+ return aggregateQuota(results);
+ }
+
+ /**
+ * Get valid quota remote locations used in {@link #getQuotaUsage(String)}.
+ * Differentiate the method {@link #getQuotaRemoteLocations(String)}, this
+ * method will do some additional filtering.
+ * @param path Federation path.
+ * @return List of valid quota remote locations.
+ * @throws IOException
+ */
+ private List<RemoteLocation> getValidQuotaLocations(String path)
+ throws IOException {
+ final List<RemoteLocation> locations = getQuotaRemoteLocations(path);
+
+ // NameService -> Locations
+ Map<String, List<RemoteLocation>> validLocations = new HashMap<>();
+ for (RemoteLocation loc : locations) {
+ String nsId = loc.getNameserviceId();
+ List<RemoteLocation> dests = validLocations.get(nsId);
+ if (dests == null) {
+ dests = new LinkedList<>();
+ dests.add(loc);
+ validLocations.put(nsId, dests);
+ } else {
+ // Ensure the paths in the same nameservice is different.
+ // Don't include parent-child paths.
+ boolean isChildPath = false;
+ for (RemoteLocation d : dests) {
+ if (loc.getDest().startsWith(d.getDest())) {
+ isChildPath = true;
+ break;
+ }
+ }
+
+ if (!isChildPath) {
+ dests.add(loc);
+ }
+ }
+ }
+
+ List<RemoteLocation> quotaLocs = new LinkedList<>();
+ for (List<RemoteLocation> locs : validLocations.values()) {
+ quotaLocs.addAll(locs);
+ }
+
+ return quotaLocs;
+ }
+
+ /**
+ * Aggregate quota that queried from sub-clusters.
+ * @param results Quota query result.
+ * @return Aggregated Quota.
+ */
+ private QuotaUsage aggregateQuota(Map<RemoteLocation, QuotaUsage> results) {
+ long nsCount = 0;
+ long ssCount = 0;
+ boolean hasQuotaUnSet = false;
+
+ for (Map.Entry<RemoteLocation, QuotaUsage> entry : results.entrySet()) {
+ RemoteLocation loc = entry.getKey();
+ QuotaUsage usage = entry.getValue();
+ if (usage != null) {
+ // If quota is not set in real FileSystem, the usage
+ // value will return -1.
+ if (usage.getQuota() == -1 && usage.getSpaceQuota() == -1) {
+ hasQuotaUnSet = true;
+ }
+
+ nsCount += usage.getFileAndDirectoryCount();
+ ssCount += usage.getSpaceConsumed();
+ LOG.debug(
+ "Get quota usage for path: nsId: {}, dest: {},"
+ + " nsCount: {}, ssCount: {}.",
+ loc.getNameserviceId(), loc.getDest(),
+ usage.getFileAndDirectoryCount(), usage.getSpaceConsumed());
+ }
+ }
+
+ QuotaUsage.Builder builder = new QuotaUsage.Builder()
+ .fileAndDirectoryCount(nsCount).spaceConsumed(ssCount);
+ if (hasQuotaUnSet) {
+ builder.quota(HdfsConstants.QUOTA_DONT_SET);
+ }
+
+ return builder.build();
+ }
+
+ /**
+ * Get all quota remote locations across subclusters under given
+ * federation path.
+ * @param path Federation path.
+ * @return List of quota remote locations.
+ * @throws IOException
+ */
+ private List<RemoteLocation> getQuotaRemoteLocations(String path)
+ throws IOException {
+ List<RemoteLocation> locations = new LinkedList<>();
+ RouterQuotaManager manager = this.router.getQuotaManager();
+ if (manager != null) {
+ Set<String> childrenPaths = manager.getPaths(path);
+ for (String childPath : childrenPaths) {
+ locations.addAll(rpcServer.getLocationsForPath(childPath, true));
+ }
+ }
+
+ return locations;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
new file mode 100644
index 0000000..170b876
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
@@ -0,0 +1,229 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCPerformanceMonitor;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreSerializerPBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Config fields for router-based hdfs federation.
+ */
+@InterfaceAudience.Private
+public class RBFConfigKeys extends CommonConfigurationKeysPublic {
+
+ // HDFS Router-based federation
+ public static final String FEDERATION_ROUTER_PREFIX =
+ "dfs.federation.router.";
+ public static final String DFS_ROUTER_DEFAULT_NAMESERVICE =
+ FEDERATION_ROUTER_PREFIX + "default.nameserviceId";
+ public static final String DFS_ROUTER_HANDLER_COUNT_KEY =
+ FEDERATION_ROUTER_PREFIX + "handler.count";
+ public static final int DFS_ROUTER_HANDLER_COUNT_DEFAULT = 10;
+ public static final String DFS_ROUTER_READER_QUEUE_SIZE_KEY =
+ FEDERATION_ROUTER_PREFIX + "reader.queue.size";
+ public static final int DFS_ROUTER_READER_QUEUE_SIZE_DEFAULT = 100;
+ public static final String DFS_ROUTER_READER_COUNT_KEY =
+ FEDERATION_ROUTER_PREFIX + "reader.count";
+ public static final int DFS_ROUTER_READER_COUNT_DEFAULT = 1;
+ public static final String DFS_ROUTER_HANDLER_QUEUE_SIZE_KEY =
+ FEDERATION_ROUTER_PREFIX + "handler.queue.size";
+ public static final int DFS_ROUTER_HANDLER_QUEUE_SIZE_DEFAULT = 100;
+ public static final String DFS_ROUTER_RPC_BIND_HOST_KEY =
+ FEDERATION_ROUTER_PREFIX + "rpc-bind-host";
+ public static final int DFS_ROUTER_RPC_PORT_DEFAULT = 8888;
+ public static final String DFS_ROUTER_RPC_ADDRESS_KEY =
+ FEDERATION_ROUTER_PREFIX + "rpc-address";
+ public static final String DFS_ROUTER_RPC_ADDRESS_DEFAULT =
+ "0.0.0.0:" + DFS_ROUTER_RPC_PORT_DEFAULT;
+ public static final String DFS_ROUTER_RPC_ENABLE =
+ FEDERATION_ROUTER_PREFIX + "rpc.enable";
+ public static final boolean DFS_ROUTER_RPC_ENABLE_DEFAULT = true;
+
+ public static final String DFS_ROUTER_METRICS_ENABLE =
+ FEDERATION_ROUTER_PREFIX + "metrics.enable";
+ public static final boolean DFS_ROUTER_METRICS_ENABLE_DEFAULT = true;
+ public static final String DFS_ROUTER_METRICS_CLASS =
+ FEDERATION_ROUTER_PREFIX + "metrics.class";
+ public static final Class<? extends RouterRpcMonitor>
+ DFS_ROUTER_METRICS_CLASS_DEFAULT =
+ FederationRPCPerformanceMonitor.class;
+
+ // HDFS Router heartbeat
+ public static final String DFS_ROUTER_HEARTBEAT_ENABLE =
+ FEDERATION_ROUTER_PREFIX + "heartbeat.enable";
+ public static final boolean DFS_ROUTER_HEARTBEAT_ENABLE_DEFAULT = true;
+ public static final String DFS_ROUTER_HEARTBEAT_INTERVAL_MS =
+ FEDERATION_ROUTER_PREFIX + "heartbeat.interval";
+ public static final long DFS_ROUTER_HEARTBEAT_INTERVAL_MS_DEFAULT =
+ TimeUnit.SECONDS.toMillis(5);
+ public static final String DFS_ROUTER_MONITOR_NAMENODE =
+ FEDERATION_ROUTER_PREFIX + "monitor.namenode";
+ public static final String DFS_ROUTER_MONITOR_LOCAL_NAMENODE =
+ FEDERATION_ROUTER_PREFIX + "monitor.localnamenode.enable";
+ public static final boolean DFS_ROUTER_MONITOR_LOCAL_NAMENODE_DEFAULT = true;
+ public static final String DFS_ROUTER_HEARTBEAT_STATE_INTERVAL_MS =
+ FEDERATION_ROUTER_PREFIX + "heartbeat-state.interval";
+ public static final long DFS_ROUTER_HEARTBEAT_STATE_INTERVAL_MS_DEFAULT =
+ TimeUnit.SECONDS.toMillis(5);
+
+ // HDFS Router NN client
+ public static final String DFS_ROUTER_NAMENODE_CONNECTION_POOL_SIZE =
+ FEDERATION_ROUTER_PREFIX + "connection.pool-size";
+ public static final int DFS_ROUTER_NAMENODE_CONNECTION_POOL_SIZE_DEFAULT =
+ 64;
+ public static final String DFS_ROUTER_NAMENODE_CONNECTION_POOL_CLEAN =
+ FEDERATION_ROUTER_PREFIX + "connection.pool.clean.ms";
+ public static final long DFS_ROUTER_NAMENODE_CONNECTION_POOL_CLEAN_DEFAULT =
+ TimeUnit.MINUTES.toMillis(1);
+ public static final String DFS_ROUTER_NAMENODE_CONNECTION_CLEAN_MS =
+ FEDERATION_ROUTER_PREFIX + "connection.clean.ms";
+ public static final long DFS_ROUTER_NAMENODE_CONNECTION_CLEAN_MS_DEFAULT =
+ TimeUnit.SECONDS.toMillis(10);
+
+ // HDFS Router RPC client
+ public static final String DFS_ROUTER_CLIENT_THREADS_SIZE =
+ FEDERATION_ROUTER_PREFIX + "client.thread-size";
+ public static final int DFS_ROUTER_CLIENT_THREADS_SIZE_DEFAULT = 32;
+ public static final String DFS_ROUTER_CLIENT_MAX_ATTEMPTS =
+ FEDERATION_ROUTER_PREFIX + "client.retry.max.attempts";
+ public static final int DFS_ROUTER_CLIENT_MAX_ATTEMPTS_DEFAULT = 3;
+
+ // HDFS Router State Store connection
+ public static final String FEDERATION_FILE_RESOLVER_CLIENT_CLASS =
+ FEDERATION_ROUTER_PREFIX + "file.resolver.client.class";
+ public static final Class<? extends FileSubclusterResolver>
+ FEDERATION_FILE_RESOLVER_CLIENT_CLASS_DEFAULT =
+ MountTableResolver.class;
+ public static final String FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS =
+ FEDERATION_ROUTER_PREFIX + "namenode.resolver.client.class";
+ public static final Class<? extends ActiveNamenodeResolver>
+ FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS_DEFAULT =
+ MembershipNamenodeResolver.class;
+
+ // HDFS Router-based federation State Store
+ public static final String FEDERATION_STORE_PREFIX =
+ FEDERATION_ROUTER_PREFIX + "store.";
+
+ public static final String DFS_ROUTER_STORE_ENABLE =
+ FEDERATION_STORE_PREFIX + "enable";
+ public static final boolean DFS_ROUTER_STORE_ENABLE_DEFAULT = true;
+
+ public static final String FEDERATION_STORE_SERIALIZER_CLASS =
+ FEDERATION_STORE_PREFIX + "serializer";
+ public static final Class<StateStoreSerializerPBImpl>
+ FEDERATION_STORE_SERIALIZER_CLASS_DEFAULT =
+ StateStoreSerializerPBImpl.class;
+
+ public static final String FEDERATION_STORE_DRIVER_CLASS =
+ FEDERATION_STORE_PREFIX + "driver.class";
+ public static final Class<? extends StateStoreDriver>
+ FEDERATION_STORE_DRIVER_CLASS_DEFAULT = StateStoreZooKeeperImpl.class;
+
+ public static final String FEDERATION_STORE_CONNECTION_TEST_MS =
+ FEDERATION_STORE_PREFIX + "connection.test";
+ public static final long FEDERATION_STORE_CONNECTION_TEST_MS_DEFAULT =
+ TimeUnit.MINUTES.toMillis(1);
+
+ public static final String DFS_ROUTER_CACHE_TIME_TO_LIVE_MS =
+ FEDERATION_ROUTER_PREFIX + "cache.ttl";
+ public static final long DFS_ROUTER_CACHE_TIME_TO_LIVE_MS_DEFAULT =
+ TimeUnit.MINUTES.toMillis(1);
+
+ public static final String FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS =
+ FEDERATION_STORE_PREFIX + "membership.expiration";
+ public static final long FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS_DEFAULT =
+ TimeUnit.MINUTES.toMillis(5);
+ public static final String FEDERATION_STORE_ROUTER_EXPIRATION_MS =
+ FEDERATION_STORE_PREFIX + "router.expiration";
+ public static final long FEDERATION_STORE_ROUTER_EXPIRATION_MS_DEFAULT =
+ TimeUnit.MINUTES.toMillis(5);
+
+ // HDFS Router safe mode
+ public static final String DFS_ROUTER_SAFEMODE_ENABLE =
+ FEDERATION_ROUTER_PREFIX + "safemode.enable";
+ public static final boolean DFS_ROUTER_SAFEMODE_ENABLE_DEFAULT = true;
+ public static final String DFS_ROUTER_SAFEMODE_EXTENSION =
+ FEDERATION_ROUTER_PREFIX + "safemode.extension";
+ public static final long DFS_ROUTER_SAFEMODE_EXTENSION_DEFAULT =
+ TimeUnit.SECONDS.toMillis(30);
+ public static final String DFS_ROUTER_SAFEMODE_EXPIRATION =
+ FEDERATION_ROUTER_PREFIX + "safemode.expiration";
+ public static final long DFS_ROUTER_SAFEMODE_EXPIRATION_DEFAULT =
+ 3 * DFS_ROUTER_CACHE_TIME_TO_LIVE_MS_DEFAULT;
+
+ // HDFS Router-based federation mount table entries
+ /** Maximum number of cache entries to have. */
+ public static final String FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE =
+ FEDERATION_ROUTER_PREFIX + "mount-table.max-cache-size";
+ /** Remove cache entries if we have more than 10k. */
+ public static final int FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE_DEFAULT = 10000;
+
+ // HDFS Router-based federation admin
+ public static final String DFS_ROUTER_ADMIN_HANDLER_COUNT_KEY =
+ FEDERATION_ROUTER_PREFIX + "admin.handler.count";
+ public static final int DFS_ROUTER_ADMIN_HANDLER_COUNT_DEFAULT = 1;
+ public static final int DFS_ROUTER_ADMIN_PORT_DEFAULT = 8111;
+ public static final String DFS_ROUTER_ADMIN_ADDRESS_KEY =
+ FEDERATION_ROUTER_PREFIX + "admin-address";
+ public static final String DFS_ROUTER_ADMIN_ADDRESS_DEFAULT =
+ "0.0.0.0:" + DFS_ROUTER_ADMIN_PORT_DEFAULT;
+ public static final String DFS_ROUTER_ADMIN_BIND_HOST_KEY =
+ FEDERATION_ROUTER_PREFIX + "admin-bind-host";
+ public static final String DFS_ROUTER_ADMIN_ENABLE =
+ FEDERATION_ROUTER_PREFIX + "admin.enable";
+ public static final boolean DFS_ROUTER_ADMIN_ENABLE_DEFAULT = true;
+
+ // HDFS Router-based federation web
+ public static final String DFS_ROUTER_HTTP_ENABLE =
+ FEDERATION_ROUTER_PREFIX + "http.enable";
+ public static final boolean DFS_ROUTER_HTTP_ENABLE_DEFAULT = true;
+ public static final String DFS_ROUTER_HTTP_ADDRESS_KEY =
+ FEDERATION_ROUTER_PREFIX + "http-address";
+ public static final int DFS_ROUTER_HTTP_PORT_DEFAULT = 50071;
+ public static final String DFS_ROUTER_HTTP_BIND_HOST_KEY =
+ FEDERATION_ROUTER_PREFIX + "http-bind-host";
+ public static final String DFS_ROUTER_HTTP_ADDRESS_DEFAULT =
+ "0.0.0.0:" + DFS_ROUTER_HTTP_PORT_DEFAULT;
+ public static final String DFS_ROUTER_HTTPS_ADDRESS_KEY =
+ FEDERATION_ROUTER_PREFIX + "https-address";
+ public static final int DFS_ROUTER_HTTPS_PORT_DEFAULT = 50072;
+ public static final String DFS_ROUTER_HTTPS_BIND_HOST_KEY =
+ FEDERATION_ROUTER_PREFIX + "https-bind-host";
+ public static final String DFS_ROUTER_HTTPS_ADDRESS_DEFAULT =
+ "0.0.0.0:" + DFS_ROUTER_HTTPS_PORT_DEFAULT;
+
+ // HDFS Router-based federation quota
+ public static final String DFS_ROUTER_QUOTA_ENABLE =
+ FEDERATION_ROUTER_PREFIX + "quota.enable";
+ public static final boolean DFS_ROUTER_QUOTA_ENABLED_DEFAULT = false;
+ public static final String DFS_ROUTER_QUOTA_CACHE_UPATE_INTERVAL =
+ FEDERATION_ROUTER_PREFIX + "quota-cache.update.interval";
+ public static final long DFS_ROUTER_QUOTA_CACHE_UPATE_INTERVAL_DEFAULT =
+ 60000;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
new file mode 100644
index 0000000..a90c460
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.commons.lang.builder.HashCodeBuilder;
+
+/**
+ * Base class for objects that are unique to a namespace.
+ */
+public abstract class RemoteLocationContext
+ implements Comparable<RemoteLocationContext> {
+
+ /**
+ * Returns an identifier for a unique namespace.
+ *
+ * @return Namespace identifier.
+ */
+ public abstract String getNameserviceId();
+
+ /**
+ * Destination in this location. For example the path in a remote namespace.
+ *
+ * @return Destination in this location.
+ */
+ public abstract String getDest();
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder(17, 31)
+ .append(getNameserviceId())
+ .append(getDest())
+ .toHashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof RemoteLocationContext) {
+ RemoteLocationContext other = (RemoteLocationContext) obj;
+ return this.getNameserviceId().equals(other.getNameserviceId()) &&
+ this.getDest().equals(other.getDest());
+ }
+ return false;
+ }
+
+ @Override
+ public int compareTo(RemoteLocationContext info) {
+ int ret = this.getNameserviceId().compareTo(info.getNameserviceId());
+ if (ret == 0) {
+ ret = this.getDest().compareTo(info.getDest());
+ }
+ return ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java
new file mode 100644
index 0000000..cd57d45
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.Arrays;
+
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Determines the remote client protocol method and the parameter list for a
+ * specific location.
+ */
+public class RemoteMethod {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RemoteMethod.class);
+
+
+ /** List of parameters: static and dynamic values, matchings types. */
+ private final Object[] params;
+ /** List of method parameters types, matches parameters. */
+ private final Class<?>[] types;
+ /** String name of the ClientProtocol method. */
+ private final String methodName;
+
+ /**
+ * Create a method with no parameters.
+ *
+ * @param method The string name of the ClientProtocol method.
+ */
+ public RemoteMethod(String method) {
+ this.params = null;
+ this.types = null;
+ this.methodName = method;
+ }
+
+ /**
+ * Creates a remote method generator.
+ *
+ * @param method The string name of the ClientProtocol method.
+ * @param pTypes A list of types to use to locate the specific method.
+ * @param pParams A list of parameters for the method. The order of the
+ * parameter list must match the order and number of the types.
+ * Parameters are grouped into 2 categories:
+ * <ul>
+ * <li>Static parameters that are immutable across locations.
+ * <li>Dynamic parameters that are determined for each location by a
+ * RemoteParam object. To specify a dynamic parameter, pass an
+ * instance of RemoteParam in place of the parameter value.
+ * </ul>
+ * @throws IOException If the types and parameter lists are not valid.
+ */
+ public RemoteMethod(String method, Class<?>[] pTypes, Object... pParams)
+ throws IOException {
+
+ if (pParams.length != pTypes.length) {
+ throw new IOException("Invalid parameters for method " + method);
+ }
+
+ this.params = pParams;
+ this.types = pTypes;
+ this.methodName = method;
+ }
+
+ /**
+ * Get the represented java method.
+ *
+ * @return Method
+ * @throws IOException If the method cannot be found.
+ */
+ public Method getMethod() throws IOException {
+ try {
+ if (types != null) {
+ return ClientProtocol.class.getDeclaredMethod(methodName, types);
+ } else {
+ return ClientProtocol.class.getDeclaredMethod(methodName);
+ }
+ } catch (NoSuchMethodException e) {
+ // Re-throw as an IOException
+ LOG.error("Cannot get method {} with types {}",
+ methodName, Arrays.toString(types), e);
+ throw new IOException(e);
+ } catch (SecurityException e) {
+ LOG.error("Cannot access method {} with types {}",
+ methodName, Arrays.toString(types), e);
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * Get the calling types for this method.
+ *
+ * @return An array of calling types.
+ */
+ public Class<?>[] getTypes() {
+ return this.types;
+ }
+
+ /**
+ * Generate a list of parameters for this specific location using no context.
+ *
+ * @return A list of parameters for the method customized for the location.
+ */
+ public Object[] getParams() {
+ return this.getParams(null);
+ }
+
+ /**
+ * Get the name of the method.
+ *
+ * @return Name of the method.
+ */
+ public String getMethodName() {
+ return this.methodName;
+ }
+
+ /**
+ * Generate a list of parameters for this specific location. Parameters are
+ * grouped into 2 categories:
+ * <ul>
+ * <li>Static parameters that are immutable across locations.
+ * <li>Dynamic parameters that are determined for each location by a
+ * RemoteParam object.
+ * </ul>
+ *
+ * @param context The context identifying the location.
+ * @return A list of parameters for the method customized for the location.
+ */
+ public Object[] getParams(RemoteLocationContext context) {
+ if (this.params == null) {
+ return new Object[] {};
+ }
+ Object[] objList = new Object[this.params.length];
+ for (int i = 0; i < this.params.length; i++) {
+ Object currentObj = this.params[i];
+ if (currentObj instanceof RemoteParam) {
+ // Map the parameter using the context
+ RemoteParam paramGetter = (RemoteParam) currentObj;
+ objList[i] = paramGetter.getParameterForContext(context);
+ } else {
+ objList[i] = currentObj;
+ }
+ }
+ return objList;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteParam.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteParam.java
new file mode 100644
index 0000000..8816ff6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteParam.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.util.Map;
+
+/**
+ * A dynamically assignable parameter that is location-specific.
+ * <p>
+ * There are 2 ways this mapping is determined:
+ * <ul>
+ * <li>Default: Uses the RemoteLocationContext's destination
+ * <li>Map: Uses the value of the RemoteLocationContext key provided in the
+ * parameter map.
+ * </ul>
+ */
+public class RemoteParam {
+
+ private final Map<? extends Object, ? extends Object> paramMap;
+
+ /**
+ * Constructs a default remote parameter. Always maps the value to the
+ * destination of the provided RemoveLocationContext.
+ */
+ public RemoteParam() {
+ this.paramMap = null;
+ }
+
+ /**
+ * Constructs a map based remote parameter. Determines the value using the
+ * provided RemoteLocationContext as a key into the map.
+ *
+ * @param map Map with RemoteLocationContext keys.
+ */
+ public RemoteParam(
+ Map<? extends RemoteLocationContext, ? extends Object> map) {
+ this.paramMap = map;
+ }
+
+ /**
+ * Determine the appropriate value for this parameter based on the location.
+ *
+ * @param context Context identifying the location.
+ * @return A parameter specific to this location.
+ */
+ public Object getParameterForContext(RemoteLocationContext context) {
+ if (context == null) {
+ return null;
+ } else if (this.paramMap != null) {
+ return this.paramMap.get(context);
+ } else {
+ // Default case
+ return context.getDest();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
new file mode 100644
index 0000000..38f5d4f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
@@ -0,0 +1,655 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.apache.hadoop.hdfs.server.federation.router.FederationUtil.newActiveNamenodeResolver;
+import static org.apache.hadoop.hdfs.server.federation.router.FederationUtil.newFileSubclusterResolver;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.HAUtil;
+import org.apache.hadoop.hdfs.server.federation.metrics.FederationMetrics;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.hdfs.server.federation.store.RouterStore;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.source.JvmMetrics;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.util.JvmPauseMonitor;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Router that provides a unified view of multiple federated HDFS clusters. It
+ * has two main roles: (1) federated interface and (2) NameNode heartbeat.
+ * <p>
+ * For the federated interface, the Router receives a client request, checks the
+ * State Store for the correct subcluster, and forwards the request to the
+ * active Namenode of that subcluster. The reply from the Namenode then flows in
+ * the opposite direction. The Routers are stateless and can be behind a load
+ * balancer. HDFS clients connect to the router using the same interfaces as are
+ * used to communicate with a namenode, namely the ClientProtocol RPC interface
+ * and the WebHdfs HTTP interface exposed by the router. {@link RouterRpcServer}
+ * {@link RouterHttpServer}
+ * <p>
+ * For NameNode heartbeat, the Router periodically checks the state of a
+ * NameNode (usually on the same server) and reports their high availability
+ * (HA) state and load/space status to the State Store. Note that this is an
+ * optional role as a Router can be independent of any subcluster.
+ * {@link StateStoreService} {@link NamenodeHeartbeatService}
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class Router extends CompositeService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Router.class);
+
+
+ /** Configuration for the Router. */
+ private Configuration conf;
+
+ /** Router address/identifier. */
+ private String routerId;
+
+ /** RPC interface to the client. */
+ private RouterRpcServer rpcServer;
+ private InetSocketAddress rpcAddress;
+
+ /** RPC interface for the admin. */
+ private RouterAdminServer adminServer;
+ private InetSocketAddress adminAddress;
+
+ /** HTTP interface and web application. */
+ private RouterHttpServer httpServer;
+
+ /** Interface with the State Store. */
+ private StateStoreService stateStore;
+
+ /** Interface to map global name space to HDFS subcluster name spaces. */
+ private FileSubclusterResolver subclusterResolver;
+
+ /** Interface to identify the active NN for a nameservice or blockpool ID. */
+ private ActiveNamenodeResolver namenodeResolver;
+ /** Updates the namenode status in the namenode resolver. */
+ private Collection<NamenodeHeartbeatService> namenodeHeartbeatServices;
+
+ /** Router metrics. */
+ private RouterMetricsService metrics;
+
+ /** JVM pauses (GC and others). */
+ private JvmPauseMonitor pauseMonitor;
+
+ /** Quota usage update service. */
+ private RouterQuotaUpdateService quotaUpdateService;
+ /** Quota cache manager. */
+ private RouterQuotaManager quotaManager;
+
+ /** Manages the current state of the router. */
+ private RouterStore routerStateManager;
+ /** Heartbeat our run status to the router state manager. */
+ private RouterHeartbeatService routerHeartbeatService;
+ /** Enter/exit safemode. */
+ private RouterSafemodeService safemodeService;
+
+ /** The start time of the namesystem. */
+ private final long startTime = Time.now();
+
+ /** State of the Router. */
+ private RouterServiceState state = RouterServiceState.UNINITIALIZED;
+
+
+ /////////////////////////////////////////////////////////
+ // Constructor
+ /////////////////////////////////////////////////////////
+
+ public Router() {
+ super(Router.class.getName());
+ }
+
+ /////////////////////////////////////////////////////////
+ // Service management
+ /////////////////////////////////////////////////////////
+
+ @Override
+ protected void serviceInit(Configuration configuration) throws Exception {
+ this.conf = configuration;
+ updateRouterState(RouterServiceState.INITIALIZING);
+
+ if (conf.getBoolean(
+ RBFConfigKeys.DFS_ROUTER_STORE_ENABLE,
+ RBFConfigKeys.DFS_ROUTER_STORE_ENABLE_DEFAULT)) {
+ // Service that maintains the State Store connection
+ this.stateStore = new StateStoreService();
+ addService(this.stateStore);
+ }
+
+ // Resolver to track active NNs
+ this.namenodeResolver = newActiveNamenodeResolver(
+ this.conf, this.stateStore);
+ if (this.namenodeResolver == null) {
+ throw new IOException("Cannot find namenode resolver.");
+ }
+
+ // Lookup interface to map between the global and subcluster name spaces
+ this.subclusterResolver = newFileSubclusterResolver(this.conf, this);
+ if (this.subclusterResolver == null) {
+ throw new IOException("Cannot find subcluster resolver");
+ }
+
+ if (conf.getBoolean(
+ RBFConfigKeys.DFS_ROUTER_RPC_ENABLE,
+ RBFConfigKeys.DFS_ROUTER_RPC_ENABLE_DEFAULT)) {
+ // Create RPC server
+ this.rpcServer = createRpcServer();
+ addService(this.rpcServer);
+ this.setRpcServerAddress(rpcServer.getRpcAddress());
+ }
+
+ if (conf.getBoolean(
+ RBFConfigKeys.DFS_ROUTER_ADMIN_ENABLE,
+ RBFConfigKeys.DFS_ROUTER_ADMIN_ENABLE_DEFAULT)) {
+ // Create admin server
+ this.adminServer = createAdminServer();
+ addService(this.adminServer);
+ }
+
+ if (conf.getBoolean(
+ RBFConfigKeys.DFS_ROUTER_HTTP_ENABLE,
+ RBFConfigKeys.DFS_ROUTER_HTTP_ENABLE_DEFAULT)) {
+ // Create HTTP server
+ this.httpServer = createHttpServer();
+ addService(this.httpServer);
+ }
+
+ if (conf.getBoolean(
+ RBFConfigKeys.DFS_ROUTER_HEARTBEAT_ENABLE,
+ RBFConfigKeys.DFS_ROUTER_HEARTBEAT_ENABLE_DEFAULT)) {
+
+ // Create status updater for each monitored Namenode
+ this.namenodeHeartbeatServices = createNamenodeHeartbeatServices();
+ for (NamenodeHeartbeatService hearbeatService :
+ this.namenodeHeartbeatServices) {
+ addService(hearbeatService);
+ }
+
+ if (this.namenodeHeartbeatServices.isEmpty()) {
+ LOG.error("Heartbeat is enabled but there are no namenodes to monitor");
+ }
+
+ // Periodically update the router state
+ this.routerHeartbeatService = new RouterHeartbeatService(this);
+ addService(this.routerHeartbeatService);
+ }
+
+ // Router metrics system
+ if (conf.getBoolean(
+ RBFConfigKeys.DFS_ROUTER_METRICS_ENABLE,
+ RBFConfigKeys.DFS_ROUTER_METRICS_ENABLE_DEFAULT)) {
+
+ DefaultMetricsSystem.initialize("Router");
+
+ this.metrics = new RouterMetricsService(this);
+ addService(this.metrics);
+
+ // JVM pause monitor
+ this.pauseMonitor = new JvmPauseMonitor();
+ this.pauseMonitor.init(conf);
+ }
+
+ // Initial quota relevant service
+ if (conf.getBoolean(RBFConfigKeys.DFS_ROUTER_QUOTA_ENABLE,
+ RBFConfigKeys.DFS_ROUTER_QUOTA_ENABLED_DEFAULT)) {
+ this.quotaManager = new RouterQuotaManager();
+ this.quotaUpdateService = new RouterQuotaUpdateService(this);
+ addService(this.quotaUpdateService);
+ }
+
+ // Safemode service to refuse RPC calls when the router is out of sync
+ if (conf.getBoolean(
+ RBFConfigKeys.DFS_ROUTER_SAFEMODE_ENABLE,
+ RBFConfigKeys.DFS_ROUTER_SAFEMODE_ENABLE_DEFAULT)) {
+ // Create safemode monitoring service
+ this.safemodeService = new RouterSafemodeService(this);
+ addService(this.safemodeService);
+ }
+
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+
+ if (this.safemodeService == null) {
+ // Router is running now
+ updateRouterState(RouterServiceState.RUNNING);
+ }
+
+ if (this.pauseMonitor != null) {
+ this.pauseMonitor.start();
+ JvmMetrics jvmMetrics = this.metrics.getJvmMetrics();
+ if (jvmMetrics != null) {
+ jvmMetrics.setPauseMonitor(pauseMonitor);
+ }
+ }
+
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+
+ // Update state
+ updateRouterState(RouterServiceState.SHUTDOWN);
+
+ // JVM pause monitor
+ if (this.pauseMonitor != null) {
+ this.pauseMonitor.stop();
+ }
+
+ super.serviceStop();
+ }
+
+ /**
+ * Shutdown the router.
+ */
+ public void shutDown() {
+ new Thread() {
+ @Override
+ public void run() {
+ Router.this.stop();
+ }
+ }.start();
+ }
+
+ /////////////////////////////////////////////////////////
+ // RPC Server
+ /////////////////////////////////////////////////////////
+
+ /**
+ * Create a new Router RPC server to proxy ClientProtocol requests.
+ *
+ * @return New Router RPC Server.
+ * @throws IOException If the router RPC server was not started.
+ */
+ protected RouterRpcServer createRpcServer() throws IOException {
+ return new RouterRpcServer(this.conf, this, this.getNamenodeResolver(),
+ this.getSubclusterResolver());
+ }
+
+ /**
+ * Get the Router RPC server.
+ *
+ * @return Router RPC server.
+ */
+ public RouterRpcServer getRpcServer() {
+ return this.rpcServer;
+ }
+
+ /**
+ * Set the current RPC socket for the router.
+ *
+ * @param address RPC address.
+ */
+ protected void setRpcServerAddress(InetSocketAddress address) {
+ this.rpcAddress = address;
+
+ // Use the RPC address as our unique router Id
+ if (this.rpcAddress != null) {
+ try {
+ String hostname = InetAddress.getLocalHost().getHostName();
+ setRouterId(hostname + ":" + this.rpcAddress.getPort());
+ } catch (UnknownHostException ex) {
+ LOG.error("Cannot set unique router ID, address not resolvable {}",
+ this.rpcAddress);
+ }
+ }
+ }
+
+ /**
+ * Get the current RPC socket address for the router.
+ *
+ * @return InetSocketAddress
+ */
+ public InetSocketAddress getRpcServerAddress() {
+ return this.rpcAddress;
+ }
+
+ /////////////////////////////////////////////////////////
+ // Admin server
+ /////////////////////////////////////////////////////////
+
+ /**
+ * Create a new router admin server to handle the router admin interface.
+ *
+ * @return RouterAdminServer
+ * @throws IOException If the admin server was not successfully started.
+ */
+ protected RouterAdminServer createAdminServer() throws IOException {
+ return new RouterAdminServer(this.conf, this);
+ }
+
+ /**
+ * Set the current Admin socket for the router.
+ *
+ * @param address Admin RPC address.
+ */
+ protected void setAdminServerAddress(InetSocketAddress address) {
+ this.adminAddress = address;
+ }
+
+ /**
+ * Get the current Admin socket address for the router.
+ *
+ * @return InetSocketAddress Admin address.
+ */
+ public InetSocketAddress getAdminServerAddress() {
+ return adminAddress;
+ }
+
+ /////////////////////////////////////////////////////////
+ // HTTP server
+ /////////////////////////////////////////////////////////
+
+ /**
+ * Create an HTTP server for this Router.
+ *
+ * @return HTTP server for this Router.
+ */
+ protected RouterHttpServer createHttpServer() {
+ return new RouterHttpServer(this);
+ }
+
+ /**
+ * Get the current HTTP socket address for the router.
+ *
+ * @return InetSocketAddress HTTP address.
+ */
+ public InetSocketAddress getHttpServerAddress() {
+ if (httpServer != null) {
+ return httpServer.getHttpAddress();
+ }
+ return null;
+ }
+
+ /////////////////////////////////////////////////////////
+ // Namenode heartbeat monitors
+ /////////////////////////////////////////////////////////
+
+ /**
+ * Create each of the services that will monitor a Namenode.
+ *
+ * @return List of heartbeat services.
+ */
+ protected Collection<NamenodeHeartbeatService>
+ createNamenodeHeartbeatServices() {
+
+ Map<String, NamenodeHeartbeatService> ret = new HashMap<>();
+
+ if (conf.getBoolean(
+ RBFConfigKeys.DFS_ROUTER_MONITOR_LOCAL_NAMENODE,
+ RBFConfigKeys.DFS_ROUTER_MONITOR_LOCAL_NAMENODE_DEFAULT)) {
+ // Create a local heartbet service
+ NamenodeHeartbeatService localHeartbeatService =
+ createLocalNamenodeHearbeatService();
+ if (localHeartbeatService != null) {
+ String nnDesc = localHeartbeatService.getNamenodeDesc();
+ ret.put(nnDesc, localHeartbeatService);
+ }
+ }
+
+ // Create heartbeat services for a list specified by the admin
+ String namenodes = this.conf.get(
+ RBFConfigKeys.DFS_ROUTER_MONITOR_NAMENODE);
+ if (namenodes != null) {
+ for (String namenode : namenodes.split(",")) {
+ String[] namenodeSplit = namenode.split("\\.");
+ String nsId = null;
+ String nnId = null;
+ if (namenodeSplit.length == 2) {
+ nsId = namenodeSplit[0];
+ nnId = namenodeSplit[1];
+ } else if (namenodeSplit.length == 1) {
+ nsId = namenode;
+ } else {
+ LOG.error("Wrong Namenode to monitor: {}", namenode);
+ }
+ if (nsId != null) {
+ NamenodeHeartbeatService heartbeatService =
+ createNamenodeHearbeatService(nsId, nnId);
+ if (heartbeatService != null) {
+ ret.put(heartbeatService.getNamenodeDesc(), heartbeatService);
+ }
+ }
+ }
+ }
+
+ return ret.values();
+ }
+
+ /**
+ * Create a new status updater for the local Namenode.
+ *
+ * @return Updater of the status for the local Namenode.
+ */
+ protected NamenodeHeartbeatService createLocalNamenodeHearbeatService() {
+ // Detect NN running in this machine
+ String nsId = DFSUtil.getNamenodeNameServiceId(conf);
+ String nnId = null;
+ if (HAUtil.isHAEnabled(conf, nsId)) {
+ nnId = HAUtil.getNameNodeId(conf, nsId);
+ if (nnId == null) {
+ LOG.error("Cannot find namenode id for local {}", nsId);
+ }
+ }
+
+ return createNamenodeHearbeatService(nsId, nnId);
+ }
+
+ /**
+ * Create a heartbeat monitor for a particular Namenode.
+ *
+ * @param nsId Identifier of the nameservice to monitor.
+ * @param nnId Identifier of the namenode (HA) to monitor.
+ * @return Updater of the status for the specified Namenode.
+ */
+ protected NamenodeHeartbeatService createNamenodeHearbeatService(
+ String nsId, String nnId) {
+
+ LOG.info("Creating heartbeat service for Namenode {} in {}", nnId, nsId);
+ NamenodeHeartbeatService ret = new NamenodeHeartbeatService(
+ namenodeResolver, nsId, nnId);
+ return ret;
+ }
+
+ /////////////////////////////////////////////////////////
+ // Router State Management
+ /////////////////////////////////////////////////////////
+
+ /**
+ * Update the router state and heartbeat to the state store.
+ *
+ * @param state The new router state.
+ */
+ public void updateRouterState(RouterServiceState newState) {
+ this.state = newState;
+ if (this.routerHeartbeatService != null) {
+ this.routerHeartbeatService.updateStateAsync();
+ }
+ }
+
+ /**
+ * Get the status of the router.
+ *
+ * @return Status of the router.
+ */
+ public RouterServiceState getRouterState() {
+ return this.state;
+ }
+
+ /////////////////////////////////////////////////////////
+ // Submodule getters
+ /////////////////////////////////////////////////////////
+
+ /**
+ * Get the State Store service.
+ *
+ * @return State Store service.
+ */
+ public StateStoreService getStateStore() {
+ return this.stateStore;
+ }
+
+ /**
+ * Get the metrics system for the Router.
+ *
+ * @return Router metrics.
+ */
+ public RouterMetrics getRouterMetrics() {
+ if (this.metrics != null) {
+ return this.metrics.getRouterMetrics();
+ }
+ return null;
+ }
+
+ /**
+ * Get the federation metrics.
+ *
+ * @return Federation metrics.
+ */
+ public FederationMetrics getMetrics() {
+ if (this.metrics != null) {
+ return this.metrics.getFederationMetrics();
+ }
+ return null;
+ }
+
+ /**
+ * Get the subcluster resolver for files.
+ *
+ * @return Subcluster resolver for files.
+ */
+ public FileSubclusterResolver getSubclusterResolver() {
+ return this.subclusterResolver;
+ }
+
+ /**
+ * Get the namenode resolver for a subcluster.
+ *
+ * @return The namenode resolver for a subcluster.
+ */
+ public ActiveNamenodeResolver getNamenodeResolver() {
+ return this.namenodeResolver;
+ }
+
+ /**
+ * Get the state store interface for the router heartbeats.
+ *
+ * @return FederationRouterStateStore state store API handle.
+ */
+ public RouterStore getRouterStateManager() {
+ if (this.routerStateManager == null && this.stateStore != null) {
+ this.routerStateManager = this.stateStore.getRegisteredRecordStore(
+ RouterStore.class);
+ }
+ return this.routerStateManager;
+ }
+
+ /////////////////////////////////////////////////////////
+ // Router info
+ /////////////////////////////////////////////////////////
+
+ /**
+ * Get the start date of the Router.
+ *
+ * @return Start date of the router.
+ */
+ public long getStartTime() {
+ return this.startTime;
+ }
+
+ /**
+ * Unique ID for the router, typically the hostname:port string for the
+ * router's RPC server. This ID may be null on router startup before the RPC
+ * server has bound to a port.
+ *
+ * @return Router identifier.
+ */
+ public String getRouterId() {
+ return this.routerId;
+ }
+
+ /**
+ * Sets a unique ID for this router.
+ *
+ * @param id Identifier of the Router.
+ */
+ public void setRouterId(String id) {
+ this.routerId = id;
+ if (this.stateStore != null) {
+ this.stateStore.setIdentifier(this.routerId);
+ }
+ if (this.namenodeResolver != null) {
+ this.namenodeResolver.setRouterId(this.routerId);
+ }
+ }
+
+ /**
+ * If the quota system is enabled in Router.
+ */
+ public boolean isQuotaEnabled() {
+ return this.quotaManager != null;
+ }
+
+ /**
+ * Get route quota manager.
+ * @return RouterQuotaManager Quota manager.
+ */
+ public RouterQuotaManager getQuotaManager() {
+ return this.quotaManager;
+ }
+
+ /**
+ * Get quota cache update service.
+ */
+ @VisibleForTesting
+ RouterQuotaUpdateService getQuotaCacheUpdateService() {
+ return this.quotaUpdateService;
+ }
+
+ /**
+ * Get the list of namenode heartbeat service.
+ */
+ @VisibleForTesting
+ Collection<NamenodeHeartbeatService> getNamenodeHearbeatServices() {
+ return this.namenodeHeartbeatServices;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
new file mode 100644
index 0000000..2c195c6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
@@ -0,0 +1,298 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_ENABLED_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.protocol.proto.RouterProtocolProtos.RouterAdminProtocolService;
+import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolPB;
+import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolServerSideTranslatorPB;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
+import org.apache.hadoop.hdfs.server.federation.store.MountTableStore;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.RPC.Server;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.AbstractService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.protobuf.BlockingService;
+
+/**
+ * This class is responsible for handling all of the Admin calls to the HDFS
+ * router. It is created, started, and stopped by {@link Router}.
+ */
+public class RouterAdminServer extends AbstractService
+ implements MountTableManager, RouterStateManager {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RouterAdminServer.class);
+
+ private Configuration conf;
+
+ private final Router router;
+
+ private MountTableStore mountTableStore;
+
+ /** The Admin server that listens to requests from clients. */
+ private final Server adminServer;
+ private final InetSocketAddress adminAddress;
+
+ /**
+ * Permission related info used for constructing new router permission
+ * checker instance.
+ */
+ private static String routerOwner;
+ private static String superGroup;
+ private static boolean isPermissionEnabled;
+
+ public RouterAdminServer(Configuration conf, Router router)
+ throws IOException {
+ super(RouterAdminServer.class.getName());
+
+ this.conf = conf;
+ this.router = router;
+
+ int handlerCount = this.conf.getInt(
+ RBFConfigKeys.DFS_ROUTER_ADMIN_HANDLER_COUNT_KEY,
+ RBFConfigKeys.DFS_ROUTER_ADMIN_HANDLER_COUNT_DEFAULT);
+
+ RPC.setProtocolEngine(this.conf, RouterAdminProtocolPB.class,
+ ProtobufRpcEngine.class);
+
+ RouterAdminProtocolServerSideTranslatorPB routerAdminProtocolTranslator =
+ new RouterAdminProtocolServerSideTranslatorPB(this);
+ BlockingService clientNNPbService = RouterAdminProtocolService.
+ newReflectiveBlockingService(routerAdminProtocolTranslator);
+
+ InetSocketAddress confRpcAddress = conf.getSocketAddr(
+ RBFConfigKeys.DFS_ROUTER_ADMIN_BIND_HOST_KEY,
+ RBFConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_KEY,
+ RBFConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_DEFAULT,
+ RBFConfigKeys.DFS_ROUTER_ADMIN_PORT_DEFAULT);
+
+ String bindHost = conf.get(
+ RBFConfigKeys.DFS_ROUTER_ADMIN_BIND_HOST_KEY,
+ confRpcAddress.getHostName());
+ LOG.info("Admin server binding to {}:{}",
+ bindHost, confRpcAddress.getPort());
+
+ initializePermissionSettings(this.conf);
+ this.adminServer = new RPC.Builder(this.conf)
+ .setProtocol(RouterAdminProtocolPB.class)
+ .setInstance(clientNNPbService)
+ .setBindAddress(bindHost)
+ .setPort(confRpcAddress.getPort())
+ .setNumHandlers(handlerCount)
+ .setVerbose(false)
+ .build();
+
+ // The RPC-server port can be ephemeral... ensure we have the correct info
+ InetSocketAddress listenAddress = this.adminServer.getListenerAddress();
+ this.adminAddress = new InetSocketAddress(
+ confRpcAddress.getHostName(), listenAddress.getPort());
+ router.setAdminServerAddress(this.adminAddress);
+ }
+
+ /**
+ * Initialize permission related settings.
+ *
+ * @param routerConf
+ * @throws IOException
+ */
+ private static void initializePermissionSettings(Configuration routerConf)
+ throws IOException {
+ routerOwner = UserGroupInformation.getCurrentUser().getShortUserName();
+ superGroup = routerConf.get(
+ DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_KEY,
+ DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT);
+ isPermissionEnabled = routerConf.getBoolean(DFS_PERMISSIONS_ENABLED_KEY,
+ DFS_PERMISSIONS_ENABLED_DEFAULT);
+ }
+
+ /** Allow access to the client RPC server for testing. */
+ @VisibleForTesting
+ Server getAdminServer() {
+ return this.adminServer;
+ }
+
+ private MountTableStore getMountTableStore() throws IOException {
+ if (this.mountTableStore == null) {
+ this.mountTableStore = router.getStateStore().getRegisteredRecordStore(
+ MountTableStore.class);
+ if (this.mountTableStore == null) {
+ throw new IOException("Mount table state store is not available.");
+ }
+ }
+ return this.mountTableStore;
+ }
+
+ /**
+ * Get the RPC address of the admin service.
+ * @return Administration service RPC address.
+ */
+ public InetSocketAddress getRpcAddress() {
+ return this.adminAddress;
+ }
+
+ @Override
+ protected void serviceInit(Configuration configuration) throws Exception {
+ this.conf = configuration;
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ this.adminServer.start();
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (this.adminServer != null) {
+ this.adminServer.stop();
+ }
+ super.serviceStop();
+ }
+
+ @Override
+ public AddMountTableEntryResponse addMountTableEntry(
+ AddMountTableEntryRequest request) throws IOException {
+ return getMountTableStore().addMountTableEntry(request);
+ }
+
+ @Override
+ public UpdateMountTableEntryResponse updateMountTableEntry(
+ UpdateMountTableEntryRequest request) throws IOException {
+ return getMountTableStore().updateMountTableEntry(request);
+ }
+
+ @Override
+ public RemoveMountTableEntryResponse removeMountTableEntry(
+ RemoveMountTableEntryRequest request) throws IOException {
+ return getMountTableStore().removeMountTableEntry(request);
+ }
+
+ @Override
+ public GetMountTableEntriesResponse getMountTableEntries(
+ GetMountTableEntriesRequest request) throws IOException {
+ return getMountTableStore().getMountTableEntries(request);
+ }
+
+ @Override
+ public EnterSafeModeResponse enterSafeMode(EnterSafeModeRequest request)
+ throws IOException {
+ this.router.updateRouterState(RouterServiceState.SAFEMODE);
+ this.router.getRpcServer().setSafeMode(true);
+ return EnterSafeModeResponse.newInstance(verifySafeMode(true));
+ }
+
+ @Override
+ public LeaveSafeModeResponse leaveSafeMode(LeaveSafeModeRequest request)
+ throws IOException {
+ this.router.updateRouterState(RouterServiceState.RUNNING);
+ this.router.getRpcServer().setSafeMode(false);
+ return LeaveSafeModeResponse.newInstance(verifySafeMode(false));
+ }
+
+ @Override
+ public GetSafeModeResponse getSafeMode(GetSafeModeRequest request)
+ throws IOException {
+ boolean isInSafeMode = this.router.getRpcServer().isInSafeMode();
+ return GetSafeModeResponse.newInstance(isInSafeMode);
+ }
+
+ /**
+ * Verify if Router set safe mode state correctly.
+ * @param isInSafeMode Expected state to be set.
+ * @return
+ */
+ private boolean verifySafeMode(boolean isInSafeMode) {
+ boolean serverInSafeMode = this.router.getRpcServer().isInSafeMode();
+ RouterServiceState currentState = this.router.getRouterState();
+
+ return (isInSafeMode && currentState == RouterServiceState.SAFEMODE
+ && serverInSafeMode)
+ || (!isInSafeMode && currentState != RouterServiceState.SAFEMODE
+ && !serverInSafeMode);
+ }
+
+ /**
+ * Get a new permission checker used for making mount table access
+ * control. This method will be invoked during each RPC call in router
+ * admin server.
+ *
+ * @return Router permission checker
+ * @throws AccessControlException
+ */
+ public static RouterPermissionChecker getPermissionChecker()
+ throws AccessControlException {
+ if (!isPermissionEnabled) {
+ return null;
+ }
+
+ try {
+ return new RouterPermissionChecker(routerOwner, superGroup,
+ NameNode.getRemoteUser());
+ } catch (IOException e) {
+ throw new AccessControlException(e);
+ }
+ }
+
+ /**
+ * Get super user name.
+ *
+ * @return String super user name.
+ */
+ public static String getSuperUser() {
+ return routerOwner;
+ }
+
+ /**
+ * Get super group name.
+ *
+ * @return String super group name.
+ */
+ public static String getSuperGroup(){
+ return superGroup;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FileSubclusterResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FileSubclusterResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FileSubclusterResolver.java
deleted file mode 100644
index af9f493..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FileSubclusterResolver.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs.server.federation.resolver;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * Interface to map a file path in the global name space to a specific
- * subcluster and path in an HDFS name space.
- * <p>
- * Each path in the global/federated namespace may map to 1-N different HDFS
- * locations. Each location specifies a single nameservice and a single HDFS
- * path. The behavior is similar to MergeFS and Nfly and allows the merger
- * of multiple HDFS locations into a single path. See HADOOP-8298 and
- * HADOOP-12077
- * <p>
- * For example, a directory listing will fetch listings for each destination
- * path and combine them into a single set of results.
- * <p>
- * When multiple destinations are available for a path, the destinations are
- * prioritized in a consistent manner. This allows the proxy server to
- * guess the best/most likely destination and attempt it first.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public interface FileSubclusterResolver {
-
- /**
- * Get the destinations for a global path. Results are from the mount table
- * cache. If multiple destinations are available, the first result is the
- * highest priority destination.
- *
- * @param path Global path.
- * @return Location in a destination namespace or null if it does not exist.
- * @throws IOException Throws exception if the data is not available.
- */
- PathLocation getDestinationForPath(String path) throws IOException;
-
- /**
- * Get a list of mount points for a path. Results are from the mount table
- * cache.
- *
- * @return List of mount points present at this path or zero-length list if
- * none are found.
- * @throws IOException Throws exception if the data is not available.
- */
- List<String> getMountPoints(String path) throws IOException;
-
- /**
- * Get the default namespace for the cluster.
- *
- * @return Default namespace identifier.
- */
- String getDefaultNamespace();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java
deleted file mode 100644
index b87eeec..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java
+++ /dev/null
@@ -1,317 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.resolver;
-
-import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.ACTIVE;
-import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.EXPIRED;
-import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.UNAVAILABLE;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.server.federation.store.MembershipStore;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreCache;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreUnavailableException;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateNamenodeRegistrationRequest;
-import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
-import org.apache.hadoop.hdfs.server.federation.store.records.MembershipStats;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Implements a cached lookup of the most recently active namenode for a
- * particular nameservice. Relies on the {@link StateStoreService} to
- * discover available nameservices and namenodes.
- */
-public class MembershipNamenodeResolver
- implements ActiveNamenodeResolver, StateStoreCache {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(MembershipNamenodeResolver.class);
-
- /** Reference to the State Store. */
- private final StateStoreService stateStore;
- /** Membership State Store interface. */
- private MembershipStore membershipInterface;
-
- /** Parent router ID. */
- private String routerId;
-
- /** Cached lookup of NN for nameservice. Invalidated on cache refresh. */
- private Map<String, List<? extends FederationNamenodeContext>> cacheNS;
- /** Cached lookup of NN for block pool. Invalidated on cache refresh. */
- private Map<String, List<? extends FederationNamenodeContext>> cacheBP;
-
-
- public MembershipNamenodeResolver(
- Configuration conf, StateStoreService store) throws IOException {
- this.stateStore = store;
-
- this.cacheNS = new ConcurrentHashMap<>();
- this.cacheBP = new ConcurrentHashMap<>();
-
- if (this.stateStore != null) {
- // Request cache updates from the state store
- this.stateStore.registerCacheExternal(this);
- }
- }
-
- private synchronized MembershipStore getMembershipStore() throws IOException {
- if (this.membershipInterface == null) {
- this.membershipInterface = this.stateStore.getRegisteredRecordStore(
- MembershipStore.class);
- if (this.membershipInterface == null) {
- throw new IOException("State Store does not have an interface for " +
- MembershipStore.class.getSimpleName());
- }
- }
- return this.membershipInterface;
- }
-
- @Override
- public boolean loadCache(boolean force) {
- // Our cache depends on the store, update it first
- try {
- MembershipStore membership = getMembershipStore();
- membership.loadCache(force);
- } catch (IOException e) {
- LOG.error("Cannot update membership from the State Store", e);
- }
-
- // Force refresh of active NN cache
- cacheBP.clear();
- cacheNS.clear();
- return true;
- }
-
- @Override
- public void updateActiveNamenode(
- final String nsId, final InetSocketAddress address) throws IOException {
-
- // Called when we have an RPC miss and successful hit on an alternate NN.
- // Temporarily update our cache, it will be overwritten on the next update.
- try {
- MembershipState partial = MembershipState.newInstance();
- String rpcAddress = address.getHostName() + ":" + address.getPort();
- partial.setRpcAddress(rpcAddress);
- partial.setNameserviceId(nsId);
-
- GetNamenodeRegistrationsRequest request =
- GetNamenodeRegistrationsRequest.newInstance(partial);
-
- MembershipStore membership = getMembershipStore();
- GetNamenodeRegistrationsResponse response =
- membership.getNamenodeRegistrations(request);
- List<MembershipState> records = response.getNamenodeMemberships();
-
- if (records != null && records.size() == 1) {
- MembershipState record = records.get(0);
- UpdateNamenodeRegistrationRequest updateRequest =
- UpdateNamenodeRegistrationRequest.newInstance(
- record.getNameserviceId(), record.getNamenodeId(), ACTIVE);
- membership.updateNamenodeRegistration(updateRequest);
- }
- } catch (StateStoreUnavailableException e) {
- LOG.error("Cannot update {} as active, State Store unavailable", address);
- }
- }
-
- @Override
- public List<? extends FederationNamenodeContext> getNamenodesForNameserviceId(
- final String nsId) throws IOException {
-
- List<? extends FederationNamenodeContext> ret = cacheNS.get(nsId);
- if (ret == null) {
- try {
- MembershipState partial = MembershipState.newInstance();
- partial.setNameserviceId(nsId);
- GetNamenodeRegistrationsRequest request =
- GetNamenodeRegistrationsRequest.newInstance(partial);
-
- final List<MembershipState> result =
- getRecentRegistrationForQuery(request, true, false);
- if (result == null || result.isEmpty()) {
- LOG.error("Cannot locate eligible NNs for {}", nsId);
- return null;
- } else {
- cacheNS.put(nsId, result);
- ret = result;
- }
- } catch (StateStoreUnavailableException e) {
- LOG.error("Cannot get active NN for {}, State Store unavailable", nsId);
- }
- }
- if (ret == null) {
- return null;
- }
- return Collections.unmodifiableList(ret);
- }
-
- @Override
- public List<? extends FederationNamenodeContext> getNamenodesForBlockPoolId(
- final String bpId) throws IOException {
-
- List<? extends FederationNamenodeContext> ret = cacheBP.get(bpId);
- if (ret == null) {
- try {
- MembershipState partial = MembershipState.newInstance();
- partial.setBlockPoolId(bpId);
- GetNamenodeRegistrationsRequest request =
- GetNamenodeRegistrationsRequest.newInstance(partial);
-
- final List<MembershipState> result =
- getRecentRegistrationForQuery(request, true, false);
- if (result == null || result.isEmpty()) {
- LOG.error("Cannot locate eligible NNs for {}", bpId);
- } else {
- cacheBP.put(bpId, result);
- ret = result;
- }
- } catch (StateStoreUnavailableException e) {
- LOG.error("Cannot get active NN for {}, State Store unavailable", bpId);
- return null;
- }
- }
- if (ret == null) {
- return null;
- }
- return Collections.unmodifiableList(ret);
- }
-
- @Override
- public boolean registerNamenode(NamenodeStatusReport report)
- throws IOException {
-
- if (this.routerId == null) {
- LOG.warn("Cannot register namenode, router ID is not known {}", report);
- return false;
- }
-
- MembershipState record = MembershipState.newInstance(
- routerId, report.getNameserviceId(), report.getNamenodeId(),
- report.getClusterId(), report.getBlockPoolId(), report.getRpcAddress(),
- report.getServiceAddress(), report.getLifelineAddress(),
- report.getWebAddress(), report.getState(), report.getSafemode());
-
- if (report.statsValid()) {
- MembershipStats stats = MembershipStats.newInstance();
- stats.setNumOfFiles(report.getNumFiles());
- stats.setNumOfBlocks(report.getNumBlocks());
- stats.setNumOfBlocksMissing(report.getNumBlocksMissing());
- stats.setNumOfBlocksPendingReplication(
- report.getNumOfBlocksPendingReplication());
- stats.setNumOfBlocksUnderReplicated(
- report.getNumOfBlocksUnderReplicated());
- stats.setNumOfBlocksPendingDeletion(
- report.getNumOfBlocksPendingDeletion());
- stats.setAvailableSpace(report.getAvailableSpace());
- stats.setTotalSpace(report.getTotalSpace());
- stats.setProvidedSpace(report.getProvidedSpace());
- stats.setNumOfDecommissioningDatanodes(
- report.getNumDecommissioningDatanodes());
- stats.setNumOfActiveDatanodes(report.getNumLiveDatanodes());
- stats.setNumOfDeadDatanodes(report.getNumDeadDatanodes());
- stats.setNumOfDecomActiveDatanodes(report.getNumDecomLiveDatanodes());
- stats.setNumOfDecomDeadDatanodes(report.getNumDecomDeadDatanodes());
- record.setStats(stats);
- }
-
- if (report.getState() != UNAVAILABLE) {
- // Set/update our last contact time
- record.setLastContact(Time.now());
- }
-
- NamenodeHeartbeatRequest request = NamenodeHeartbeatRequest.newInstance();
- request.setNamenodeMembership(record);
- return getMembershipStore().namenodeHeartbeat(request).getResult();
- }
-
- @Override
- public Set<FederationNamespaceInfo> getNamespaces() throws IOException {
- GetNamespaceInfoRequest request = GetNamespaceInfoRequest.newInstance();
- GetNamespaceInfoResponse response =
- getMembershipStore().getNamespaceInfo(request);
- return response.getNamespaceInfo();
- }
-
- /**
- * Picks the most relevant record registration that matches the query. Return
- * registrations matching the query in this preference: 1) Most recently
- * updated ACTIVE registration 2) Most recently updated STANDBY registration
- * (if showStandby) 3) Most recently updated UNAVAILABLE registration (if
- * showUnavailable). EXPIRED registrations are ignored.
- *
- * @param request The select query for NN registrations.
- * @param addUnavailable include UNAVAILABLE registrations.
- * @param addExpired include EXPIRED registrations.
- * @return List of memberships or null if no registrations that
- * both match the query AND the selected states.
- * @throws IOException
- */
- private List<MembershipState> getRecentRegistrationForQuery(
- GetNamenodeRegistrationsRequest request, boolean addUnavailable,
- boolean addExpired) throws IOException {
-
- // Retrieve a list of all registrations that match this query.
- // This may include all NN records for a namespace/blockpool, including
- // duplicate records for the same NN from different routers.
- MembershipStore membershipStore = getMembershipStore();
- GetNamenodeRegistrationsResponse response =
- membershipStore.getNamenodeRegistrations(request);
-
- List<MembershipState> memberships = response.getNamenodeMemberships();
- if (!addExpired || !addUnavailable) {
- Iterator<MembershipState> iterator = memberships.iterator();
- while (iterator.hasNext()) {
- MembershipState membership = iterator.next();
- if (membership.getState() == EXPIRED && !addExpired) {
- iterator.remove();
- } else if (membership.getState() == UNAVAILABLE && !addUnavailable) {
- iterator.remove();
- }
- }
- }
-
- List<MembershipState> priorityList = new ArrayList<>();
- priorityList.addAll(memberships);
- Collections.sort(priorityList, new NamenodePriorityComparator());
-
- LOG.debug("Selected most recent NN {} for query", priorityList);
- return priorityList;
- }
-
- @Override
- public void setRouterId(String router) {
- this.routerId = router;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableManager.java
deleted file mode 100644
index c2e4a5b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableManager.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.resolver;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse;
-
-/**
- * Manage a mount table.
- */
-public interface MountTableManager {
-
- /**
- * Add an entry to the mount table.
- *
- * @param request Fully populated request object.
- * @return True if the mount table entry was successfully committed to the
- * data store.
- * @throws IOException Throws exception if the data store is not initialized.
- */
- AddMountTableEntryResponse addMountTableEntry(
- AddMountTableEntryRequest request) throws IOException;
-
- /**
- * Updates an existing entry in the mount table.
- *
- * @param request Fully populated request object.
- * @return True if the mount table entry was successfully committed to the
- * data store.
- * @throws IOException Throws exception if the data store is not initialized.
- */
- UpdateMountTableEntryResponse updateMountTableEntry(
- UpdateMountTableEntryRequest request) throws IOException;
-
- /**
- * Remove an entry from the mount table.
- *
- * @param request Fully populated request object.
- * @return True the mount table entry was removed from the data store.
- * @throws IOException Throws exception if the data store is not initialized.
- */
- RemoveMountTableEntryResponse removeMountTableEntry(
- RemoveMountTableEntryRequest request) throws IOException;
-
- /**
- * List all mount table entries present at or below the path. Fetches from the
- * state store.
- *
- * @param request Fully populated request object.
- *
- * @return List of all mount table entries under the path. Zero-length list if
- * none are found.
- * @throws IOException Throws exception if the data store cannot be queried.
- */
- GetMountTableEntriesResponse getMountTableEntries(
- GetMountTableEntriesRequest request) throws IOException;
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
deleted file mode 100644
index 3c45faf..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
+++ /dev/null
@@ -1,597 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.resolver;
-
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE_DEFAULT;
-import static org.apache.hadoop.hdfs.server.federation.router.FederationUtil.isParentEntry;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import org.apache.hadoop.HadoopIllegalArgumentException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.DFSUtilClient;
-import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
-import org.apache.hadoop.hdfs.server.federation.router.Router;
-import org.apache.hadoop.hdfs.server.federation.store.MountTableStore;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreCache;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreUnavailableException;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
-
-/**
- * Mount table to map between global paths and remote locations. This allows the
- * {@link org.apache.hadoop.hdfs.server.federation.router.Router Router} to map
- * the global HDFS view to the remote namespaces. This is similar to
- * {@link org.apache.hadoop.fs.viewfs.ViewFs ViewFs}.
- * This is implemented as a tree.
- */
-public class MountTableResolver
- implements FileSubclusterResolver, StateStoreCache {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(MountTableResolver.class);
-
- /** Reference to Router. */
- private final Router router;
- /** Reference to the State Store. */
- private final StateStoreService stateStore;
- /** Interface to the mount table store. */
- private MountTableStore mountTableStore;
-
- /** If the tree has been initialized. */
- private boolean init = false;
- /** Path -> Remote HDFS location. */
- private final TreeMap<String, MountTable> tree = new TreeMap<>();
- /** Path -> Remote location. */
- private final Cache<String, PathLocation> locationCache;
-
- /** Default nameservice when no mount matches the math. */
- private String defaultNameService = "";
-
- /** Synchronization for both the tree and the cache. */
- private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
- private final Lock readLock = readWriteLock.readLock();
- private final Lock writeLock = readWriteLock.writeLock();
-
-
- @VisibleForTesting
- public MountTableResolver(Configuration conf) {
- this(conf, (StateStoreService)null);
- }
-
- public MountTableResolver(Configuration conf, Router routerService) {
- this(conf, routerService, null);
- }
-
- public MountTableResolver(Configuration conf, StateStoreService store) {
- this(conf, null, store);
- }
-
- public MountTableResolver(Configuration conf, Router routerService,
- StateStoreService store) {
- this.router = routerService;
- if (store != null) {
- this.stateStore = store;
- } else if (this.router != null) {
- this.stateStore = this.router.getStateStore();
- } else {
- this.stateStore = null;
- }
-
- int maxCacheSize = conf.getInt(
- FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE,
- FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE_DEFAULT);
- this.locationCache = CacheBuilder.newBuilder()
- .maximumSize(maxCacheSize)
- .build();
-
- registerCacheExternal();
- initDefaultNameService(conf);
- }
-
- /**
- * Request cache updates from the State Store for this resolver.
- */
- private void registerCacheExternal() {
- if (this.stateStore != null) {
- this.stateStore.registerCacheExternal(this);
- }
- }
-
- /**
- * Nameservice for APIs that cannot be resolved to a specific one.
- *
- * @param conf Configuration for this resolver.
- */
- private void initDefaultNameService(Configuration conf) {
- try {
- this.defaultNameService = conf.get(
- DFS_ROUTER_DEFAULT_NAMESERVICE,
- DFSUtil.getNamenodeNameServiceId(conf));
- } catch (HadoopIllegalArgumentException e) {
- LOG.error("Cannot find default name service, setting it to the first");
- Collection<String> nsIds = DFSUtilClient.getNameServiceIds(conf);
- this.defaultNameService = nsIds.iterator().next();
- LOG.info("Default name service: {}", this.defaultNameService);
- }
- }
-
- /**
- * Get a reference for the Router for this resolver.
- *
- * @return Router for this resolver.
- */
- protected Router getRouter() {
- return this.router;
- }
-
- /**
- * Get the mount table store for this resolver.
- *
- * @return Mount table store.
- * @throws IOException If it cannot connect to the State Store.
- */
- protected MountTableStore getMountTableStore() throws IOException {
- if (this.mountTableStore == null) {
- this.mountTableStore = this.stateStore.getRegisteredRecordStore(
- MountTableStore.class);
- if (this.mountTableStore == null) {
- throw new IOException("State Store does not have an interface for " +
- MountTableStore.class);
- }
- }
- return this.mountTableStore;
- }
-
- /**
- * Add a mount entry to the table.
- *
- * @param entry The mount table record to add from the state store.
- */
- public void addEntry(final MountTable entry) {
- writeLock.lock();
- try {
- String srcPath = entry.getSourcePath();
- this.tree.put(srcPath, entry);
- invalidateLocationCache(srcPath);
- } finally {
- writeLock.unlock();
- }
- this.init = true;
- }
-
- /**
- * Remove a mount table entry.
- *
- * @param srcPath Source path for the entry to remove.
- */
- public void removeEntry(final String srcPath) {
- writeLock.lock();
- try {
- this.tree.remove(srcPath);
- invalidateLocationCache(srcPath);
- } finally {
- writeLock.unlock();
- }
- }
-
- /**
- * Invalidates all cache entries below this path. It requires the write lock.
- *
- * @param path Source path.
- */
- private void invalidateLocationCache(final String path) {
- LOG.debug("Invalidating {} from {}", path, locationCache);
- if (locationCache.size() == 0) {
- return;
- }
-
- // Go through the entries and remove the ones from the path to invalidate
- ConcurrentMap<String, PathLocation> map = locationCache.asMap();
- Set<Entry<String, PathLocation>> entries = map.entrySet();
- Iterator<Entry<String, PathLocation>> it = entries.iterator();
- while (it.hasNext()) {
- Entry<String, PathLocation> entry = it.next();
- PathLocation loc = entry.getValue();
- String src = loc.getSourcePath();
- if (src != null) {
- if (isParentEntry(src, path)) {
- LOG.debug("Removing {}", src);
- it.remove();
- }
- } else {
- String dest = loc.getDefaultLocation().getDest();
- if (dest.startsWith(path)) {
- LOG.debug("Removing default cache {}", dest);
- it.remove();
- }
- }
- }
-
- LOG.debug("Location cache after invalidation: {}", locationCache);
- }
-
- /**
- * Updates the mount path tree with a new set of mount table entries. It also
- * updates the needed caches.
- *
- * @param entries Full set of mount table entries to update.
- */
- @VisibleForTesting
- public void refreshEntries(final Collection<MountTable> entries) {
- // The tree read/write must be atomic
- writeLock.lock();
- try {
- // New entries
- Map<String, MountTable> newEntries = new ConcurrentHashMap<>();
- for (MountTable entry : entries) {
- String srcPath = entry.getSourcePath();
- newEntries.put(srcPath, entry);
- }
-
- // Old entries (reversed to sort from the leaves to the root)
- Set<String> oldEntries = new TreeSet<>(Collections.reverseOrder());
- for (MountTable entry : getTreeValues("/")) {
- String srcPath = entry.getSourcePath();
- oldEntries.add(srcPath);
- }
-
- // Entries that need to be removed
- for (String srcPath : oldEntries) {
- if (!newEntries.containsKey(srcPath)) {
- this.tree.remove(srcPath);
- invalidateLocationCache(srcPath);
- LOG.info("Removed stale mount point {} from resolver", srcPath);
- }
- }
-
- // Entries that need to be added
- for (MountTable entry : entries) {
- String srcPath = entry.getSourcePath();
- if (!oldEntries.contains(srcPath)) {
- // Add node, it does not exist
- this.tree.put(srcPath, entry);
- invalidateLocationCache(srcPath);
- LOG.info("Added new mount point {} to resolver", srcPath);
- } else {
- // Node exists, check for updates
- MountTable existingEntry = this.tree.get(srcPath);
- if (existingEntry != null && !existingEntry.equals(entry)) {
- LOG.info("Entry has changed from \"{}\" to \"{}\"",
- existingEntry, entry);
- this.tree.put(srcPath, entry);
- invalidateLocationCache(srcPath);
- LOG.info("Updated mount point {} in resolver", srcPath);
- }
- }
- }
- } finally {
- writeLock.unlock();
- }
- this.init = true;
- }
-
- /**
- * Replaces the current in-memory cached of the mount table with a new
- * version fetched from the data store.
- */
- @Override
- public boolean loadCache(boolean force) {
- try {
- // Our cache depends on the store, update it first
- MountTableStore mountTable = this.getMountTableStore();
- mountTable.loadCache(force);
-
- GetMountTableEntriesRequest request =
- GetMountTableEntriesRequest.newInstance("/");
- GetMountTableEntriesResponse response =
- mountTable.getMountTableEntries(request);
- List<MountTable> records = response.getEntries();
- refreshEntries(records);
- } catch (IOException e) {
- LOG.error("Cannot fetch mount table entries from State Store", e);
- return false;
- }
- return true;
- }
-
- /**
- * Clears all data.
- */
- public void clear() {
- LOG.info("Clearing all mount location caches");
- writeLock.lock();
- try {
- this.locationCache.invalidateAll();
- this.tree.clear();
- } finally {
- writeLock.unlock();
- }
- }
-
- @Override
- public PathLocation getDestinationForPath(final String path)
- throws IOException {
- verifyMountTable();
- readLock.lock();
- try {
- Callable<? extends PathLocation> meh = new Callable<PathLocation>() {
- @Override
- public PathLocation call() throws Exception {
- return lookupLocation(path);
- }
- };
- return this.locationCache.get(path, meh);
- } catch (ExecutionException e) {
- throw new IOException(e);
- } finally {
- readLock.unlock();
- }
- }
-
- /**
- * Build the path location to insert into the cache atomically. It must hold
- * the read lock.
- * @param path Path to check/insert.
- * @return New remote location.
- */
- public PathLocation lookupLocation(final String path) {
- PathLocation ret = null;
- MountTable entry = findDeepest(path);
- if (entry != null) {
- ret = buildLocation(path, entry);
- } else {
- // Not found, use default location
- RemoteLocation remoteLocation =
- new RemoteLocation(defaultNameService, path);
- List<RemoteLocation> locations =
- Collections.singletonList(remoteLocation);
- ret = new PathLocation(null, locations);
- }
- return ret;
- }
-
- /**
- * Get the mount table entry for a path.
- *
- * @param path Path to look for.
- * @return Mount table entry the path belongs.
- * @throws IOException If the State Store could not be reached.
- */
- public MountTable getMountPoint(final String path) throws IOException {
- verifyMountTable();
- return findDeepest(path);
- }
-
- @Override
- public List<String> getMountPoints(final String path) throws IOException {
- verifyMountTable();
-
- Set<String> children = new TreeSet<>();
- readLock.lock();
- try {
- String from = path;
- String to = path + Character.MAX_VALUE;
- SortedMap<String, MountTable> subMap = this.tree.subMap(from, to);
-
- boolean exists = false;
- for (String subPath : subMap.keySet()) {
- String child = subPath;
-
- // Special case for /
- if (!path.equals(Path.SEPARATOR)) {
- // Get the children
- int ini = path.length();
- child = subPath.substring(ini);
- }
-
- if (child.isEmpty()) {
- // This is a mount point but without children
- exists = true;
- } else if (child.startsWith(Path.SEPARATOR)) {
- // This is a mount point with children
- exists = true;
- child = child.substring(1);
-
- // We only return immediate children
- int fin = child.indexOf(Path.SEPARATOR);
- if (fin > -1) {
- child = child.substring(0, fin);
- }
- if (!child.isEmpty()) {
- children.add(child);
- }
- }
- }
- if (!exists) {
- return null;
- }
- return new LinkedList<>(children);
- } finally {
- readLock.unlock();
- }
- }
-
- /**
- * Get all the mount records at or beneath a given path.
- * @param path Path to get the mount points from.
- * @return List of mount table records under the path or null if the path is
- * not found.
- * @throws IOException If it's not connected to the State Store.
- */
- public List<MountTable> getMounts(final String path) throws IOException {
- verifyMountTable();
-
- return getTreeValues(path, false);
- }
-
- /**
- * Check if the Mount Table is ready to be used.
- * @throws StateStoreUnavailableException If it cannot connect to the store.
- */
- private void verifyMountTable() throws StateStoreUnavailableException {
- if (!this.init) {
- throw new StateStoreUnavailableException("Mount Table not initialized");
- }
- }
-
- @Override
- public String toString() {
- readLock.lock();
- try {
- return this.tree.toString();
- } finally {
- readLock.unlock();
- }
- }
-
- /**
- * Build a location for this result beneath the discovered mount point.
- *
- * @param path Path to build for.
- * @param entry Mount table entry.
- * @return PathLocation containing the namespace, local path.
- */
- private static PathLocation buildLocation(
- final String path, final MountTable entry) {
-
- String srcPath = entry.getSourcePath();
- if (!path.startsWith(srcPath)) {
- LOG.error("Cannot build location, {} not a child of {}", path, srcPath);
- return null;
- }
- String remainingPath = path.substring(srcPath.length());
- if (remainingPath.startsWith(Path.SEPARATOR)) {
- remainingPath = remainingPath.substring(1);
- }
-
- List<RemoteLocation> locations = new LinkedList<>();
- for (RemoteLocation oneDst : entry.getDestinations()) {
- String nsId = oneDst.getNameserviceId();
- String dest = oneDst.getDest();
- String newPath = dest;
- if (!newPath.endsWith(Path.SEPARATOR) && !remainingPath.isEmpty()) {
- newPath += Path.SEPARATOR;
- }
- newPath += remainingPath;
- RemoteLocation remoteLocation = new RemoteLocation(nsId, newPath);
- locations.add(remoteLocation);
- }
- DestinationOrder order = entry.getDestOrder();
- return new PathLocation(srcPath, locations, order);
- }
-
- @Override
- public String getDefaultNamespace() {
- return this.defaultNameService;
- }
-
- /**
- * Find the deepest mount point for a path.
- * @param path Path to look for.
- * @return Mount table entry.
- */
- private MountTable findDeepest(final String path) {
- readLock.lock();
- try {
- Entry<String, MountTable> entry = this.tree.floorEntry(path);
- while (entry != null && !isParentEntry(path, entry.getKey())) {
- entry = this.tree.lowerEntry(entry.getKey());
- }
- if (entry == null) {
- return null;
- }
- return entry.getValue();
- } finally {
- readLock.unlock();
- }
- }
-
- /**
- * Get the mount table entries under a path.
- * @param path Path to search from.
- * @return Mount Table entries.
- */
- private List<MountTable> getTreeValues(final String path) {
- return getTreeValues(path, false);
- }
-
- /**
- * Get the mount table entries under a path.
- * @param path Path to search from.
- * @param reverse If the order should be reversed.
- * @return Mount Table entries.
- */
- private List<MountTable> getTreeValues(final String path, boolean reverse) {
- LinkedList<MountTable> ret = new LinkedList<>();
- readLock.lock();
- try {
- String from = path;
- String to = path + Character.MAX_VALUE;
- SortedMap<String, MountTable> subMap = this.tree.subMap(from, to);
- for (MountTable entry : subMap.values()) {
- if (!reverse) {
- ret.add(entry);
- } else {
- ret.addFirst(entry);
- }
- }
- } finally {
- readLock.unlock();
- }
- return ret;
- }
-
- /**
- * Get the size of the cache.
- * @return Size of the cache.
- */
- protected long getCacheSize() {
- return this.locationCache.size();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MultipleDestinationMountTableResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MultipleDestinationMountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MultipleDestinationMountTableResolver.java
deleted file mode 100644
index 8dd73ec..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MultipleDestinationMountTableResolver.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.resolver;
-
-import java.io.IOException;
-import java.util.EnumMap;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
-import org.apache.hadoop.hdfs.server.federation.resolver.order.HashFirstResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.order.HashResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.order.LocalResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.order.OrderedResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.order.RandomResolver;
-import org.apache.hadoop.hdfs.server.federation.router.Router;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.annotations.VisibleForTesting;
-
-/**
- * Mount table resolver that supports multiple locations for each mount entry.
- * The returned location contains prioritized remote paths from highest priority
- * to the lowest priority. Multiple locations for a mount point are optional.
- * When multiple locations are specified, both will be checked for the presence
- * of a file and the nameservice for a new file/dir is chosen based on the
- * results of a consistent hashing algorithm.
- * <p>
- * Does the Mount table entry for this path have multiple destinations?
- * <ul>
- * <li>No -> Return the location
- * <li>Yes -> Return all locations, prioritizing the best guess from the
- * consistent hashing algorithm.
- * </ul>
- * <p>
- * It has multiple options to order the locations: HASH (default), LOCAL,
- * RANDOM, and HASH_ALL.
- * <p>
- * The consistent hashing result is dependent on the number and combination of
- * nameservices that are registered for particular mount point. The order of
- * nameservices/locations in the mount table is not prioritized. Each consistent
- * hash calculation considers only the set of unique nameservices present for
- * the mount table location.
- */
-public class MultipleDestinationMountTableResolver extends MountTableResolver {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(MultipleDestinationMountTableResolver.class);
-
-
- /** Resolvers that use a particular order for multiple destinations. */
- private EnumMap<DestinationOrder, OrderedResolver> orderedResolvers =
- new EnumMap<>(DestinationOrder.class);
-
-
- public MultipleDestinationMountTableResolver(
- Configuration conf, Router router) {
- super(conf, router);
-
- // Initialize the ordered resolvers
- addResolver(DestinationOrder.HASH, new HashFirstResolver());
- addResolver(DestinationOrder.LOCAL, new LocalResolver(conf, router));
- addResolver(DestinationOrder.RANDOM, new RandomResolver());
- addResolver(DestinationOrder.HASH_ALL, new HashResolver());
- }
-
- @Override
- public PathLocation getDestinationForPath(String path) throws IOException {
- PathLocation mountTableResult = super.getDestinationForPath(path);
- if (mountTableResult == null) {
- LOG.error("The {} cannot find a location for {}",
- super.getClass().getSimpleName(), path);
- } else if (mountTableResult.hasMultipleDestinations()) {
- DestinationOrder order = mountTableResult.getDestinationOrder();
- OrderedResolver orderedResolver = orderedResolvers.get(order);
- if (orderedResolver == null) {
- LOG.error("Cannot find resolver for order {}", order);
- } else {
- String firstNamespace =
- orderedResolver.getFirstNamespace(path, mountTableResult);
-
- // Change the order of the name spaces according to the policy
- if (firstNamespace != null) {
- // This is the entity in the tree, we need to create our own copy
- mountTableResult = new PathLocation(mountTableResult, firstNamespace);
- LOG.debug("Ordered locations following {} are {}",
- order, mountTableResult);
- } else {
- LOG.error("Cannot get main namespace for path {} with order {}",
- path, order);
- }
- }
- }
- return mountTableResult;
- }
-
- @VisibleForTesting
- public void addResolver(DestinationOrder order, OrderedResolver resolver) {
- orderedResolvers.put(order, resolver);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodePriorityComparator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodePriorityComparator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodePriorityComparator.java
deleted file mode 100644
index fe82f29..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodePriorityComparator.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.resolver;
-
-import java.util.Comparator;
-
-/**
- * Compares NNs in the same namespace and prioritizes by their status. The
- * priorities are:
- * <ul>
- * <li>ACTIVE
- * <li>STANDBY
- * <li>UNAVAILABLE
- * </ul>
- * When two NNs have the same state, the last modification date is the tie
- * breaker, newest has priority. Expired NNs are excluded.
- */
-public class NamenodePriorityComparator
- implements Comparator<FederationNamenodeContext> {
-
- @Override
- public int compare(FederationNamenodeContext o1,
- FederationNamenodeContext o2) {
- FederationNamenodeServiceState state1 = o1.getState();
- FederationNamenodeServiceState state2 = o2.getState();
-
- if (state1 == state2) {
- // Both have the same state, use mode dates
- return compareModDates(o1, o2);
- } else {
- // Enum is ordered by priority
- return state1.compareTo(state2);
- }
- }
-
- /**
- * Compare the modification dates.
- *
- * @param o1 Context 1.
- * @param o2 Context 2.
- * @return Comparison between dates.
- */
- private int compareModDates(FederationNamenodeContext o1,
- FederationNamenodeContext o2) {
- // Reverse sort, lowest position is highest priority.
- return (int) (o2.getDateModified() - o1.getDateModified());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java
deleted file mode 100644
index c3c6fa8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java
+++ /dev/null
@@ -1,406 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.resolver;
-
-import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
-import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
-
-/**
- * Status of the namenode.
- */
-public class NamenodeStatusReport {
-
- /** Namenode information. */
- private String nameserviceId = "";
- private String namenodeId = "";
- private String clusterId = "";
- private String blockPoolId = "";
- private String rpcAddress = "";
- private String serviceAddress = "";
- private String lifelineAddress = "";
- private String webAddress = "";
-
- /** Namenode state. */
- private HAServiceState status = HAServiceState.STANDBY;
- private boolean safeMode = false;
-
- /** Datanodes stats. */
- private int liveDatanodes = -1;
- private int deadDatanodes = -1;
- /** Decommissioning datanodes. */
- private int decomDatanodes = -1;
- /** Live decommissioned datanodes. */
- private int liveDecomDatanodes = -1;
- /** Dead decommissioned datanodes. */
- private int deadDecomDatanodes = -1;
-
- /** Space stats. */
- private long availableSpace = -1;
- private long numOfFiles = -1;
- private long numOfBlocks = -1;
- private long numOfBlocksMissing = -1;
- private long numOfBlocksPendingReplication = -1;
- private long numOfBlocksUnderReplicated = -1;
- private long numOfBlocksPendingDeletion = -1;
- private long totalSpace = -1;
- private long providedSpace = -1;
-
- /** If the fields are valid. */
- private boolean registrationValid = false;
- private boolean statsValid = false;
- private boolean haStateValid = false;
-
- public NamenodeStatusReport(String ns, String nn, String rpc, String service,
- String lifeline, String web) {
- this.nameserviceId = ns;
- this.namenodeId = nn;
- this.rpcAddress = rpc;
- this.serviceAddress = service;
- this.lifelineAddress = lifeline;
- this.webAddress = web;
- }
-
- /**
- * If the statistics are valid.
- *
- * @return If the statistics are valid.
- */
- public boolean statsValid() {
- return this.statsValid;
- }
-
- /**
- * If the registration is valid.
- *
- * @return If the registration is valid.
- */
- public boolean registrationValid() {
- return this.registrationValid;
- }
-
- /**
- * If the HA state is valid.
- *
- * @return If the HA state is valid.
- */
- public boolean haStateValid() {
- return this.haStateValid;
- }
-
- /**
- * Get the state of the Namenode being monitored.
- *
- * @return State of the Namenode.
- */
- public FederationNamenodeServiceState getState() {
- if (!registrationValid) {
- return FederationNamenodeServiceState.UNAVAILABLE;
- } else if (haStateValid) {
- return FederationNamenodeServiceState.getState(status);
- } else {
- return FederationNamenodeServiceState.ACTIVE;
- }
- }
-
- /**
- * Get the name service identifier.
- *
- * @return The name service identifier.
- */
- public String getNameserviceId() {
- return this.nameserviceId;
- }
-
- /**
- * Get the namenode identifier.
- *
- * @return The namenode identifier.
- */
- public String getNamenodeId() {
- return this.namenodeId;
- }
-
- /**
- * Get the cluster identifier.
- *
- * @return The cluster identifier.
- */
- public String getClusterId() {
- return this.clusterId;
- }
-
- /**
- * Get the block pool identifier.
- *
- * @return The block pool identifier.
- */
- public String getBlockPoolId() {
- return this.blockPoolId;
- }
-
- /**
- * Get the RPC address.
- *
- * @return The RPC address.
- */
- public String getRpcAddress() {
- return this.rpcAddress;
- }
-
- /**
- * Get the Service RPC address.
- *
- * @return The Service RPC address.
- */
- public String getServiceAddress() {
- return this.serviceAddress;
- }
-
- /**
- * Get the Lifeline RPC address.
- *
- * @return The Lifeline RPC address.
- */
- public String getLifelineAddress() {
- return this.lifelineAddress;
- }
-
- /**
- * Get the web address.
- *
- * @return The web address.
- */
- public String getWebAddress() {
- return this.webAddress;
- }
-
- /**
- * Get the HA service state.
- *
- * @return The HA service state.
- */
- public void setHAServiceState(HAServiceState state) {
- this.status = state;
- this.haStateValid = true;
- }
-
- /**
- * Set the namespace information.
- *
- * @param info Namespace information.
- */
- public void setNamespaceInfo(NamespaceInfo info) {
- this.clusterId = info.getClusterID();
- this.blockPoolId = info.getBlockPoolID();
- this.registrationValid = true;
- }
-
- public void setSafeMode(boolean safemode) {
- this.safeMode = safemode;
- }
-
- public boolean getSafemode() {
- return this.safeMode;
- }
-
- /**
- * Set the datanode information.
- *
- * @param numLive Number of live nodes.
- * @param numDead Number of dead nodes.
- * @param numDecom Number of decommissioning nodes.
- * @param numLiveDecom Number of decommissioned live nodes.
- * @param numDeadDecom Number of decommissioned dead nodes.
- */
- public void setDatanodeInfo(int numLive, int numDead, int numDecom,
- int numLiveDecom, int numDeadDecom) {
- this.liveDatanodes = numLive;
- this.deadDatanodes = numDead;
- this.decomDatanodes = numDecom;
- this.liveDecomDatanodes = numLiveDecom;
- this.deadDecomDatanodes = numDeadDecom;
- this.statsValid = true;
- }
-
- /**
- * Get the number of live blocks.
- *
- * @return The number of dead nodes.
- */
- public int getNumLiveDatanodes() {
- return this.liveDatanodes;
- }
-
- /**
- * Get the number of dead blocks.
- *
- * @return The number of dead nodes.
- */
- public int getNumDeadDatanodes() {
- return this.deadDatanodes;
- }
-
- /**
- * Get the number of decommissionining nodes.
- *
- * @return The number of decommissionining nodes.
- */
- public int getNumDecommissioningDatanodes() {
- return this.decomDatanodes;
- }
-
- /**
- * Get the number of live decommissioned nodes.
- *
- * @return The number of live decommissioned nodes.
- */
- public int getNumDecomLiveDatanodes() {
- return this.liveDecomDatanodes;
- }
-
- /**
- * Get the number of dead decommissioned nodes.
- *
- * @return The number of dead decommissioned nodes.
- */
- public int getNumDecomDeadDatanodes() {
- return this.deadDecomDatanodes;
- }
-
- /**
- * Set the filesystem information.
- *
- * @param available Available capacity.
- * @param total Total capacity.
- * @param numFiles Number of files.
- * @param numBlocks Total number of blocks.
- * @param numBlocksMissing Number of missing blocks.
- * @param numBlocksPendingReplication Number of blocks pending replication.
- * @param numBlocksUnderReplicated Number of blocks under replication.
- * @param numBlocksPendingDeletion Number of blocks pending deletion.
- */
- public void setNamesystemInfo(long available, long total,
- long numFiles, long numBlocks, long numBlocksMissing,
- long numBlocksPendingReplication, long numBlocksUnderReplicated,
- long numBlocksPendingDeletion, long providedSpace) {
- this.totalSpace = total;
- this.availableSpace = available;
- this.numOfBlocks = numBlocks;
- this.numOfBlocksMissing = numBlocksMissing;
- this.numOfBlocksPendingReplication = numBlocksPendingReplication;
- this.numOfBlocksUnderReplicated = numBlocksUnderReplicated;
- this.numOfBlocksPendingDeletion = numBlocksPendingDeletion;
- this.numOfFiles = numFiles;
- this.statsValid = true;
- this.providedSpace = providedSpace;
- }
-
- /**
- * Get the number of blocks.
- *
- * @return The number of blocks.
- */
- public long getNumBlocks() {
- return this.numOfBlocks;
- }
-
- /**
- * Get the number of files.
- *
- * @return The number of files.
- */
- public long getNumFiles() {
- return this.numOfFiles;
- }
-
- /**
- * Get the total space.
- *
- * @return The total space.
- */
- public long getTotalSpace() {
- return this.totalSpace;
- }
-
- /**
- * Get the available space.
- *
- * @return The available space.
- */
- public long getAvailableSpace() {
- return this.availableSpace;
- }
-
- /**
- * Get the space occupied by provided storage.
- *
- * @return the provided capacity.
- */
- public long getProvidedSpace() {
- return this.providedSpace;
- }
- /**
- * Get the number of missing blocks.
- *
- * @return Number of missing blocks.
- */
- public long getNumBlocksMissing() {
- return this.numOfBlocksMissing;
- }
-
- /**
- * Get the number of pending replication blocks.
- *
- * @return Number of pending replication blocks.
- */
- public long getNumOfBlocksPendingReplication() {
- return this.numOfBlocksPendingReplication;
- }
-
- /**
- * Get the number of under replicated blocks.
- *
- * @return Number of under replicated blocks.
- */
- public long getNumOfBlocksUnderReplicated() {
- return this.numOfBlocksUnderReplicated;
- }
-
- /**
- * Get the number of pending deletion blocks.
- *
- * @return Number of pending deletion blocks.
- */
- public long getNumOfBlocksPendingDeletion() {
- return this.numOfBlocksPendingDeletion;
- }
-
- /**
- * Set the validity of registration.
- * @param isValid The desired value to be set.
- */
- public void setRegistrationValid(boolean isValid) {
- this.registrationValid = isValid;
- }
-
- @Override
- public String toString() {
- return String.format("%s-%s:%s",
- nameserviceId, namenodeId, serviceAddress);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/PathLocation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/PathLocation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/PathLocation.java
deleted file mode 100644
index 945d81d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/PathLocation.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.resolver;
-
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A map of the properties and target destinations (name space + path) for
- * a path in the global/federated name space.
- * This data is generated from the @see MountTable records.
- */
-public class PathLocation {
-
- private static final Logger LOG = LoggerFactory.getLogger(PathLocation.class);
-
-
- /** Source path in global namespace. */
- private final String sourcePath;
-
- /** Remote paths in the target name spaces. */
- private final List<RemoteLocation> destinations;
- /** Order for the destinations. */
- private final DestinationOrder destOrder;
-
-
- /**
- * Create a new PathLocation.
- *
- * @param source Source path in the global name space.
- * @param dest Destinations of the mount table entry.
- * @param order Order of the locations.
- */
- public PathLocation(
- String source, List<RemoteLocation> dest, DestinationOrder order) {
- this.sourcePath = source;
- this.destinations = Collections.unmodifiableList(dest);
- this.destOrder = order;
- }
-
- /**
- * Create a new PathLocation with default HASH order.
- *
- * @param source Source path in the global name space.
- * @param dest Destinations of the mount table entry.
- */
- public PathLocation(String source, List<RemoteLocation> dest) {
- this(source, dest, DestinationOrder.HASH);
- }
-
- /**
- * Create a path location from another path.
- *
- * @param other Other path location to copy from.
- */
- public PathLocation(final PathLocation other) {
- this.sourcePath = other.sourcePath;
- this.destinations = Collections.unmodifiableList(other.destinations);
- this.destOrder = other.destOrder;
- }
-
- /**
- * Create a path location from another path with the destinations sorted.
- *
- * @param other Other path location to copy from.
- * @param firstNsId Identifier of the namespace to place first.
- */
- public PathLocation(PathLocation other, String firstNsId) {
- this.sourcePath = other.sourcePath;
- this.destOrder = other.destOrder;
- this.destinations = orderedNamespaces(other.destinations, firstNsId);
- }
-
- /**
- * Prioritize a location/destination by its name space/nameserviceId.
- * This destination might be used by other threads, so the source is not
- * modifiable.
- *
- * @param original List of destinations to order.
- * @param nsId The name space/nameserviceID to prioritize.
- * @return Prioritized list of detinations that cannot be modified.
- */
- private static List<RemoteLocation> orderedNamespaces(
- final List<RemoteLocation> original, final String nsId) {
- if (original.size() <= 1) {
- return original;
- }
-
- LinkedList<RemoteLocation> newDestinations = new LinkedList<>();
- boolean found = false;
- for (RemoteLocation dest : original) {
- if (dest.getNameserviceId().equals(nsId)) {
- found = true;
- newDestinations.addFirst(dest);
- } else {
- newDestinations.add(dest);
- }
- }
-
- if (!found) {
- LOG.debug("Cannot find location with namespace {} in {}",
- nsId, original);
- }
- return Collections.unmodifiableList(newDestinations);
- }
-
- /**
- * Get the source path in the global namespace for this path location.
- *
- * @return The path in the global namespace.
- */
- public String getSourcePath() {
- return this.sourcePath;
- }
-
- /**
- * Get the subclusters defined for the destinations.
- *
- * @return Set containing the subclusters.
- */
- public Set<String> getNamespaces() {
- Set<String> namespaces = new HashSet<>();
- List<RemoteLocation> locations = this.getDestinations();
- for (RemoteLocation location : locations) {
- String nsId = location.getNameserviceId();
- namespaces.add(nsId);
- }
- return namespaces;
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- for (RemoteLocation destination : this.destinations) {
- String nsId = destination.getNameserviceId();
- String path = destination.getDest();
- if (sb.length() > 0) {
- sb.append(",");
- }
- sb.append(nsId + "->" + path);
- }
- if (this.destinations.size() > 1) {
- sb.append(" [");
- sb.append(this.destOrder.toString());
- sb.append("]");
- }
- return sb.toString();
- }
-
- /**
- * Check if this location supports multiple clusters/paths.
- *
- * @return If it has multiple destinations.
- */
- public boolean hasMultipleDestinations() {
- return this.destinations.size() > 1;
- }
-
- /**
- * Get the list of locations found in the mount table.
- * The first result is the highest priority path.
- *
- * @return List of remote locations.
- */
- public List<RemoteLocation> getDestinations() {
- return Collections.unmodifiableList(this.destinations);
- }
-
- /**
- * Get the order for the destinations.
- *
- * @return Order for the destinations.
- */
- public DestinationOrder getDestinationOrder() {
- return this.destOrder;
- }
-
- /**
- * Get the default or highest priority location.
- *
- * @return The default location.
- */
- public RemoteLocation getDefaultLocation() {
- if (destinations.isEmpty() || destinations.get(0).getDest() == null) {
- throw new UnsupportedOperationException(
- "Unsupported path " + sourcePath + " please check mount table");
- }
- return destinations.get(0);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java
deleted file mode 100644
index 6aa12ce..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.resolver;
-
-import org.apache.hadoop.hdfs.server.federation.router.RemoteLocationContext;
-
-/**
- * A single in a remote namespace consisting of a nameservice ID
- * and a HDFS path.
- */
-public class RemoteLocation extends RemoteLocationContext {
-
- /** Identifier of the remote namespace for this location. */
- private final String nameserviceId;
- /** Identifier of the namenode in the namespace for this location. */
- private final String namenodeId;
- /** Path in the remote location. */
- private final String path;
-
- /**
- * Create a new remote location.
- *
- * @param nsId
- * @param pPath
- */
- public RemoteLocation(String nsId, String pPath) {
- this(nsId, null, pPath);
- }
-
- /**
- * Create a new remote location pointing to a particular namenode in the
- * namespace.
- *
- * @param nsId Destination namespace.
- * @param pPath Path in the destination namespace.
- */
- public RemoteLocation(String nsId, String nnId, String pPath) {
- this.nameserviceId = nsId;
- this.namenodeId = nnId;
- this.path = pPath;
- }
-
- @Override
- public String getNameserviceId() {
- String ret = this.nameserviceId;
- if (this.namenodeId != null) {
- ret += "-" + this.namenodeId;
- }
- return ret;
- }
-
- @Override
- public String getDest() {
- return this.path;
- }
-
- @Override
- public String toString() {
- return getNameserviceId() + "->" + this.path;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/DestinationOrder.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/DestinationOrder.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/DestinationOrder.java
deleted file mode 100644
index 03e68e5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/DestinationOrder.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.resolver.order;
-
-/**
- * Order of the destinations when we have multiple of them. When the resolver
- * of files to subclusters (FileSubclusterResolver) has multiple destinations,
- * this determines which location should be checked first.
- */
-public enum DestinationOrder {
- HASH, // Follow consistent hashing in the first folder level
- LOCAL, // Local first
- RANDOM, // Random order
- HASH_ALL // Follow consistent hashing
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/HashFirstResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/HashFirstResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/HashFirstResolver.java
deleted file mode 100644
index 831b082..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/HashFirstResolver.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.resolver.order;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
-
-/**
- * Variation of HashResolver that only uses the first level of the path.
- */
-public class HashFirstResolver extends HashResolver {
-
- @Override
- public String getFirstNamespace(final String path, final PathLocation loc) {
- String srcPath = loc.getSourcePath();
- String trimmedPath = trimPathToChild(path, srcPath);
- LOG.debug("Only using the first part of the path: {} -> {}",
- path, trimmedPath);
- return super.getFirstNamespace(trimmedPath, loc);
- }
-
- /**
- * Hash only up to the immediate child of the mount point. This prevents the
- * need to create/maintain subtrees under each multi-destination mount point.
- * Each child of a multi-destination mount is mapped to only one hash
- * location.
- * <p>
- * Trims a path to at most the immediate child of a parent path. For example:
- * <ul>
- * <li>path = /a/b/c, parent = /a will be trimmed to /a/b.
- * <li>path = /a/b, parent = /a/b will be trimmed to /a/b
- * </ul>
- *
- * @param path The path to trim.
- * @param parent The parent used to find the immediate child.
- * @return Trimmed path.
- */
- private static String trimPathToChild(String path, String parent) {
- // Path is invalid or equal to the parent
- if (path.length() <= parent.length()) {
- return parent;
- }
- String remainder = path.substring(parent.length());
- String[] components =
- remainder.replaceFirst("^/", "").split(Path.SEPARATOR);
- if (components.length > 0 && components[0].length() > 0) {
- if (parent.endsWith(Path.SEPARATOR)) {
- return parent + components[0];
- } else {
- return parent + Path.SEPARATOR + components[0];
- }
- } else {
- return parent;
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/HashResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/HashResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/HashResolver.java
deleted file mode 100644
index 4034a46..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/HashResolver.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.resolver.order;
-
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
-import org.apache.hadoop.hdfs.server.federation.utils.ConsistentHashRing;
-import org.apache.hadoop.util.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.annotations.VisibleForTesting;
-
-/**
- * Order the destinations based on consistent hashing.
- */
-public class HashResolver implements OrderedResolver {
-
- protected static final Logger LOG =
- LoggerFactory.getLogger(HashResolver.class);
-
-
- /** Namespace set hash -> Locator. */
- private final Map<Integer, ConsistentHashRing> hashResolverMap;
-
- /** Patterns for temporary files. */
- private static final String HEX_PATTERN = "\\p{XDigit}";
- private static final String UUID_PATTERN = HEX_PATTERN + "{8}-" +
- HEX_PATTERN + "{4}-" + HEX_PATTERN + "{4}-" + HEX_PATTERN + "{4}-" +
- HEX_PATTERN + "{12}";
- private static final String ATTEMPT_PATTERN =
- "attempt_\\d+_\\d{4}_._\\d{6}_\\d{2}";
- private static final String[] TEMP_FILE_PATTERNS = {
- "(.+)\\.COPYING$",
- "(.+)\\._COPYING_.*$",
- "(.+)\\.tmp$",
- "_temp/(.+)$",
- "_temporary/(.+)\\." + UUID_PATTERN + "$",
- "(.*)_temporary/\\d/_temporary/" + ATTEMPT_PATTERN + "/(.+)$" };
- /** Pattern for temporary files (or of the individual patterns). */
- private static final Pattern TEMP_FILE_PATTERN =
- Pattern.compile(StringUtils.join("|", TEMP_FILE_PATTERNS));
-
-
- public HashResolver() {
- this.hashResolverMap = new ConcurrentHashMap<>();
- }
-
- /**
- * Use the result from consistent hashing locator to prioritize the locations
- * for a path.
- *
- * @param path Path to check.
- * @param loc Federated location with multiple destinations.
- * @return First namespace based on hash.
- */
- @Override
- public String getFirstNamespace(final String path, final PathLocation loc) {
- String finalPath = extractTempFileName(path);
- Set<String> namespaces = loc.getNamespaces();
- ConsistentHashRing locator = getHashResolver(namespaces);
- String hashedSubcluster = locator.getLocation(finalPath);
- if (hashedSubcluster == null) {
- String srcPath = loc.getSourcePath();
- LOG.error("Cannot find subcluster for {} ({} -> {})",
- srcPath, path, finalPath);
- }
- LOG.debug("Namespace for {} ({}) is {}", path, finalPath, hashedSubcluster);
- return hashedSubcluster;
- }
-
- /**
- * Get the cached (if available) or generate a new hash resolver for this
- * particular set of unique namespace identifiers.
- *
- * @param namespaces A set of unique namespace identifiers.
- * @return A hash resolver configured to consistently resolve paths to
- * namespaces using the provided set of namespace identifiers.
- */
- private ConsistentHashRing getHashResolver(final Set<String> namespaces) {
- int hash = namespaces.hashCode();
- ConsistentHashRing resolver = this.hashResolverMap.get(hash);
- if (resolver == null) {
- resolver = new ConsistentHashRing(namespaces);
- this.hashResolverMap.put(hash, resolver);
- }
- return resolver;
- }
-
- /**
- * Some files use a temporary naming pattern. Extract the final name from the
- * temporary name. For example, files *._COPYING_ will be renamed, so we
- * remove that chunk.
- *
- * @param input Input string.
- * @return Final file name.
- */
- @VisibleForTesting
- public static String extractTempFileName(final String input) {
- StringBuilder sb = new StringBuilder();
- Matcher matcher = TEMP_FILE_PATTERN.matcher(input);
- if (matcher.find()) {
- for (int i=1; i <= matcher.groupCount(); i++) {
- String match = matcher.group(i);
- if (match != null) {
- sb.append(match);
- }
- }
- }
- if (sb.length() > 0) {
- String ret = sb.toString();
- LOG.debug("Extracted {} from {}", ret, input);
- return ret;
- }
- return input;
- }
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html
new file mode 100644
index 0000000..71e2132
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html
@@ -0,0 +1,424 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<meta http-equiv="X-UA-Compatible" content="IE=9" />
+<link rel="stylesheet" type="text/css" href="/static/bootstrap-3.0.2/css/bootstrap.min.css" />
+<link rel="stylesheet" type="text/css" href="/static/dataTables.bootstrap.css" />
+<link rel="stylesheet" type="text/css" href="/static/hadoop.css" />
+<title>Router Information</title>
+</head>
+<body>
+
+<header class="navbar navbar-inverse bs-docs-nav" role="banner">
+<div class="container">
+ <div class="navbar-header">
+ <div class="navbar-brand">Hadoop</div>
+ </div>
+
+ <ul class="nav navbar-nav" id="ui-tabs">
+ <li><a href="#tab-overview">Overview</a></li>
+ <li><a href="#tab-namenode">Subclusters</a></li>
+ <li><a href="#tab-router">Routers</a></li>
+ <li><a href="#tab-datanode">Datanodes</a></li>
+ <li><a href="#tab-mounttable">Mount table</a></li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">Utilities <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+ <li><a href="jmx">Metrics</a></li>
+ <li><a href="conf">Configuration</a></li>
+ <li><a href="logs">Logs</a></li>
+ <li><a href="fsck">FSCK</a></li>
+ </ul>
+ </li>
+ </ul>
+</div>
+</header>
+
+<div class="container">
+
+<div id="alert-panel">
+ <div class="alert alert-danger">
+ <button type="button" class="close" onclick="$('#alert-panel').hide();">×</button>
+ <div class="alert-body" id="alert-panel-body"></div>
+ </div>
+</div>
+
+<div class="tab-content">
+ <div class="tab-pane" id="tab-overview"></div>
+ <div class="tab-pane" id="tab-namenode"></div>
+ <div class="tab-pane" id="tab-router"></div>
+ <div class="tab-pane" id="tab-datanode"></div>
+ <div class="tab-pane" id="tab-mounttable"></div>
+</div>
+
+<div class="row">
+ <hr />
+ <div class="col-xs-2"><p>Hadoop, {release-year-token}.</p></div>
+</div>
+</div>
+
+<!-- Overview -->
+<script type="text/x-dust-template" id="tmpl-federationhealth">
+<div class="page-header"><h1>Router {#federation}<small>'{HostAndPort}'</small>{/federation}</h1></div>
+{#federation}
+<table class="table table-bordered table-striped">
+ <tr><th>Started:</th><td>{RouterStarted}</td></tr>
+ <tr><th>Version:</th><td>{Version}</td></tr>
+ <tr><th>Compiled:</th><td>{CompileInfo}</td></tr>
+ <tr><th>Cluster ID:</th><td>{ClusterId}</td></tr>
+ <tr><th>Block Pool ID:</th><td>{BlockPoolId}</td></tr>
+ <tr><th>Status:</th><td>{RouterStatus}</td></tr>
+</table>
+{/federation}
+
+<div class="page-header"><h1>Summary</h1></div>
+{#federation}
+<table class="table table-bordered table-striped">
+ <tr><th>Total capacity</th><td>{TotalCapacity|fmt_bytes}</td></tr>
+ <tr><th>Used capacity</th><td>{UsedCapacity|fmt_bytes}</td></tr>
+ <tr><th>Remaining capacity</th><td>{RemainingCapacity|fmt_bytes}</td></tr>
+ <tr><th>Nameservices</th><td>{NumNameservices}</td></tr>
+ <tr><th>Namenodes</th><td>{NumNamenodes}</td></tr>
+ <tr>
+ <th>DataNodes usages% (Min/Median/Max/stdDev)</th>
+ <td>{#NodeUsage.nodeUsage}{min} / {median} / {max} / {stdDev}{/NodeUsage.nodeUsage}</td>
+ </tr>
+ <tr><th><a href="#tab-datanode">Live Nodes</a></th><td>{NumLiveNodes} (Decommissioned: {NumDecomLiveNodes})</td></tr>
+ <tr><th><a href="#tab-datanode">Dead Nodes</a></th><td>{NumDeadNodes} (Decommissioned: {NumDecomDeadNodes})</td></tr>
+ <tr><th><a href="#tab-datanode">Decommissioning Nodes</a></th><td>{NumDecommissioningNodes}</td></tr>
+ <tr><th>Files</th><td>{NumFiles}</td></tr>
+ <tr><th>Blocks</th><td>{NumBlocks}</td></tr>
+ <tr><th title="Excludes missing blocks.">Number of Under-Replicated Blocks</th><td>{NumOfBlocksUnderReplicated}</td></tr>
+ <tr><th>Number of Blocks Pending Deletion</th><td>{NumOfBlocksPendingDeletion}</td></tr>
+</table>
+{/federation}
+</script>
+
+<!-- Subclusters info: list of nameservices and namenodes -->
+<script type="text/x-dust-template" id="tmpl-namenode">
+<div class="page-header"><h1>Nameservice Information</h1></div>
+<div>
+ <ul class="dfshealth-node-legend">
+ <li class="dfshealth-node-icon dfshealth-node-alive">Active</li>
+ <li class="dfshealth-node-icon dfshealth-node-down-decommissioned">Standby</li>
+ <li class="dfshealth-node-icon dfshealth-node-down-maintenance">Safe mode</li>
+ <li class="dfshealth-node-icon dfshealth-node-down">Unavailable</li>
+ </ul>
+</div>
+<small>
+<table class="table">
+ <thead>
+ <tr>
+ <th colspan="6"></th>
+ <th colspan="3">Blocks</th>
+ <th colspan="2">Nodes</th>
+ <th colspan="3">Decom</th>
+ </tr>
+ <tr>
+ <th></th>
+ <th>Nameservice</th>
+ <th>Namenode</th>
+ <th>Last Contact</th>
+ <th>Capacity</th>
+ <th>Files</th>
+ <th>Total</th>
+ <th>Missing</th>
+ <th>Under-Replicated</th>
+ <th>Live</th>
+ <th>Dead</th>
+ <th>Progress</th>
+ <th>Live</th>
+ <th>Dead</th>
+ </tr>
+ </thead>
+ <tbody>
+ {#Nameservices}
+ <tr>
+ <td class="dfshealth-node-icon dfshealth-node-{iconState}" title="{title}"></td>
+ <td><a href="http://{webAddress}">{nameserviceId}</a></td>
+ <td><a href="http://{webAddress}">{namenodeId}</a></td>
+ <td>{lastHeartbeat}</td>
+ <td ng-value="{usedPercentage}" style="width:210px">
+ <div>
+ <div style="display:inline-block; float: left; padding-right: 10px; width:60px;">{totalSpace|fmt_bytes}</div>
+ <div class="clearfix progress dfshealth-node-capacity-bar" title="Used: {used|fmt_bytes}">
+ <div class="progress-bar {#helper_usage_bar value="{usedPercentage}"/}" style="width: {usedPercentage}%">
+ </div>
+ </div>
+ </div>
+ </td>
+ <td>{numOfFiles}</td>
+ <td>{numOfBlocks}</td>
+ <td>{numOfBlocksMissing}</td>
+ <td>{numOfBlocksUnderReplicated}</td>
+ <td>{numOfActiveDatanodes}</td>
+ <td>{numOfDeadDatanodes}</td>
+ <td>{numOfDecommissioningDatanodes}</td>
+ <td>{numOfDecomActiveDatanodes}</td>
+ <td>{numOfDecomDeadDatanodes}</td>
+ </tr>
+ {/Nameservices}
+ </tbody>
+</table>
+</small>
+
+<div class="page-header"><h1>Namenode Information</h1></div>
+<div>
+ <ul class="dfshealth-node-legend">
+ <li class="dfshealth-node-icon dfshealth-node-alive">Active</li>
+ <li class="dfshealth-node-icon dfshealth-node-down-decommissioned">Standby</li>
+ <li class="dfshealth-node-icon dfshealth-node-down-maintenance">Safe mode</li>
+ <li class="dfshealth-node-icon dfshealth-node-down">Unavailable</li>
+ </ul>
+</div>
+<small>
+<table class="table">
+ <thead>
+ <tr>
+ <th colspan="7"></th>
+ <th colspan="3">Blocks</th>
+ <th colspan="2">Nodes</th>
+ <th colspan="3">Decom</th>
+ </tr>
+ <tr>
+ <th></th>
+ <th colspan="2">Namenode</th>
+ <th>Web address</th>
+ <th>Last Contact</th>
+ <th>Capacity</th>
+ <th>Files</th>
+ <th>Total</th>
+ <th>Missing</th>
+ <th>Under-Replicated</th>
+ <th>Live</th>
+ <th>Dead</th>
+ <th>Progress</th>
+ <th>Live</th>
+ <th>Dead</th>
+ </tr>
+ </thead>
+ <tbody>
+ {#Namenodes}
+ <tr>
+ <td class="dfshealth-node-icon dfshealth-node-{iconState}" title="{title}"></td>
+ <td>{nameserviceId}</td>
+ <td>{namenodeId}</td>
+ <td><a href="http://{webAddress}">{webAddress}</a></td>
+ <td>{lastHeartbeat}</td>
+ <td ng-value="{usedPercentage}" style="width:210px">
+ <div>
+ <div style="display:inline-block; float: left; padding-right: 10px; width:60px;">{totalSpace|fmt_bytes}</div>
+ <div class="clearfix progress dfshealth-node-capacity-bar" title="Used: {used|fmt_bytes}">
+ <div class="progress-bar {#helper_usage_bar value="{usedPercentage}"/}" style="width: {usedPercentage}%">
+ </div>
+ </div>
+ </div>
+ </td>
+ <td>{numOfFiles}</td>
+ <td>{numOfBlocks}</td>
+ <td>{numOfBlocksMissing}</td>
+ <td>{numOfBlocksUnderReplicated}</td>
+ <td>{numOfActiveDatanodes}</td>
+ <td>{numOfDeadDatanodes}</td>
+ <td>{numOfDecommissioningDatanodes}</td>
+ <td>{numOfDecomActiveDatanodes}</td>
+ <td>{numOfDecomDeadDatanodes}</td>
+ </tr>
+ {/Namenodes}
+ </tbody>
+</table>
+</small>
+</script>
+
+<!-- Routers -->
+<script type="text/x-dust-template" id="tmpl-router">
+<div class="page-header"><h1>Routers Information</h1></div>
+<div>
+ <ul class="dfshealth-node-legend">
+ <li class="dfshealth-node-icon dfshealth-node-alive">Active</li>
+ <li class="dfshealth-node-icon dfshealth-node-decommissioned">Safe mode</li>
+ <li class="dfshealth-node-icon dfshealth-node-down">Unavailable</li>
+ </ul>
+</div>
+<table class="table">
+ <thead>
+ <tr>
+ <th colspan="4"></th>
+ <th colspan="2">Last update</th>
+ <th></th>
+ </tr>
+ <tr>
+ <th></th>
+ <th>Address</th>
+ <th>Status</th>
+ <th>Heartbeat</th>
+ <th>Membership</th>
+ <th>Mount table</th>
+ <th>Version</th>
+ </tr>
+ </thead>
+ <tbody>
+ {#Routers}
+ <tr>
+ <td class="dfshealth-node-icon dfshealth-node-{iconState}" title="{title}"></td>
+ <td>{address}</td>
+ <td>{status}</td>
+ <td>{lastHeartbeat} sec ago</td>
+ <td>{lastMembershipUpdate}</td>
+ <td>{lastMountTableUpdate}</td>
+ <td>{version}</td>
+ </tr>
+ {/Routers}
+ </tbody>
+</table>
+</script>
+
+<!-- Datanodes -->
+<script type="text/x-dust-template" id="tmpl-datanode">
+<div class="page-header"><h1>Datanode Information</h1></div>
+<div>
+ <ul class="dfshealth-node-legend">
+ <li class="dfshealth-node-icon dfshealth-node-alive">In service</li>
+ <li class="dfshealth-node-icon dfshealth-node-down">Down</li>
+ <li class="dfshealth-node-icon dfshealth-node-decommissioned">Decommissioned</li>
+ <li class="dfshealth-node-icon dfshealth-node-down-decommissioned">Decommissioned & dead</li>
+ </ul>
+</div>
+<div class="page-header"><h1><small>In operation</small></h1></div>
+<small>
+<table class="table" id="table-datanodes">
+ <thead>
+ <tr>
+ <th>Node</th>
+ <th>Last contact</th>
+ <th style="width:180px; text-align:center">Capacity</th>
+ <!--th>Blocks</th-->
+ <th>Block pool used</th>
+ <!--th>Version</th-->
+ </tr>
+ </thead>
+ {#LiveNodes}
+ <tr>
+ <td ng-value="{state}-{name}" class="dfshealth-node-icon dfshealth-node-{state}">{location}/{name} ({xferaddr})</td>
+ <td ng-value="{lastContact}">{#helper_relative_time value="{lastContact}"/}</td>
+ <td ng-value="{usedPercentage}" style="width:210px">
+ <div>
+ <div style="display:inline-block; float: left; padding-right: 10px; width:60px;">{capacity|fmt_bytes}</div>
+ <div class="clearfix progress dfshealth-node-capacity-bar" title="Non DFS: {nonDfsUsedSpace|fmt_bytes}, Used: {used|fmt_bytes}">
+ <div class="progress-bar {#helper_usage_bar value="{usedPercentage}"/}" style="width: {usedPercentage}%">
+ </div>
+ </div>
+ </div>
+ </td>
+ <!--td>{numBlocks}</td-->
+ <td ng-value="{blockPoolUsedPercent}">{blockPoolUsed|fmt_bytes} ({blockPoolUsedPercent|fmt_percentage})</td>
+ <!--td>{version}</td-->
+ </tr>
+ {/LiveNodes}
+ {#DeadNodes}
+ <tr class="danger">
+ <td ng-value="{state}-{name}" class="dfshealth-node-icon dfshealth-node-{state}">{location}/{name} ({xferaddr})</td>
+ <td>{#helper_relative_time value="{lastContact}"/}</td>
+ <td></td>
+ <!--td></td-->
+ <td></td>
+ <!--td></td-->
+ </tr>
+ {/DeadNodes}
+</table>
+</small>
+
+<div class="page-header"><h1><small>Decommissioning</small></h1></div>
+<small>
+<table class="table">
+ <thead>
+ <tr>
+ <th>Node</th>
+ <th>Under replicated blocks</th>
+ <th>Blocks with no live replicas</th>
+ <th>Under Replicated Blocks <br/>In files under construction</th>
+ </tr>
+ </thead>
+ {#DecomNodes}
+ <tr>
+ <td>{location}/{name} ({xferaddr})</td>
+ <td>{underReplicatedBlocks}</td>
+ <td>{decommissionOnlyReplicas}</td>
+ <td>{underReplicateInOpenFiles}</td>
+ </tr>
+ {/DecomNodes}
+</table>
+</small>
+</script>
+
+<!-- Mount table -->
+<script type="text/x-dust-template" id="tmpl-mounttable">
+<div class="page-header"><h1>Mount Table</h1></div>
+<small>
+<table class="table">
+ <thead>
+ <tr>
+ <th>Global path</th>
+ <th>Target nameservice</th>
+ <th>Target path</th>
+ <th>Order</th>
+ <th>Read only</th>
+ <th>Owner</th>
+ <th>Group</th>
+ <th>Permission</th>
+ <th>Quota/Usage</th>
+ <th>Date Modified</th>
+ <th>Date Created</th>
+ </tr>
+ </thead>
+ <tbody>
+ {#MountTable}
+ <tr>
+ <td>{sourcePath}</td>
+ <td>{nameserviceId}</td>
+ <td>{path}</td>
+ <td>{order}</td>
+ <td class="dfshealth-node-icon dfshealth-mount-read-only-{readonly}"/>
+ <td>{ownerName}</td>
+ <td>{groupName}</td>
+ <td>{mode}</td>
+ <td>{quota}</td>
+ <td>{dateModified}</td>
+ <td>{dateCreated}</td>
+ </tr>
+ {/MountTable}
+ </tbody>
+</table>
+</small>
+</script>
+
+
+
+<script type="text/javascript" src="/static/jquery-1.10.2.min.js"></script>
+<script type="text/javascript" src="/static/jquery.dataTables.min.js"></script>
+<script type="text/javascript" src="/static/bootstrap-3.0.2/js/bootstrap.min.js"></script>
+<script type="text/javascript" src="/static/dataTables.bootstrap.js"></script>
+<script type="text/javascript" src="/static/moment.min.js"></script>
+<script type="text/javascript" src="/static/dust-full-2.0.0.min.js"></script>
+<script type="text/javascript" src="/static/dust-helpers-1.1.1.min.js"></script>
+<script type="text/javascript" src="/static/dfs-dust.js"></script>
+<script type="text/javascript" src="federationhealth.js"></script>
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.js b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.js
new file mode 100644
index 0000000..b3ed868
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.js
@@ -0,0 +1,381 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+(function () {
+ "use strict";
+
+ dust.loadSource(dust.compile($('#tmpl-federationhealth').html(), 'federationhealth'));
+ dust.loadSource(dust.compile($('#tmpl-namenode').html(), 'namenode-info'));
+ dust.loadSource(dust.compile($('#tmpl-router').html(), 'router-info'));
+ dust.loadSource(dust.compile($('#tmpl-datanode').html(), 'datanode-info'));
+ dust.loadSource(dust.compile($('#tmpl-mounttable').html(), 'mounttable'));
+
+ $.fn.dataTable.ext.order['ng-value'] = function (settings, col)
+ {
+ return this.api().column(col, {order:'index'} ).nodes().map(function (td, i) {
+ return $(td).attr('ng-value');
+ });
+ };
+
+ function load_overview() {
+ var BEANS = [
+ {"name": "federation", "url": "/jmx?qry=Hadoop:service=Router,name=FederationState"}
+ ];
+
+ var HELPERS = {
+ 'helper_fs_max_objects': function (chunk, ctx, bodies, params) {
+ var o = ctx.current();
+ if (o.MaxObjects > 0) {
+ chunk.write('(' + Math.round((o.FilesTotal + o.BlockTotal) / o.MaxObjects * 100) * 100 + ')%');
+ }
+ },
+
+ 'helper_dir_status': function (chunk, ctx, bodies, params) {
+ var j = ctx.current();
+ for (var i in j) {
+ chunk.write('<tr><td>' + i + '</td><td>' + j[i] + '</td><td>' + params.type + '</td></tr>');
+ }
+ },
+
+ 'helper_date_tostring' : function (chunk, ctx, bodies, params) {
+ var value = dust.helpers.tap(params.value, chunk, ctx);
+ return chunk.write('' + new Date(Number(value)).toLocaleString());
+ }
+ };
+
+ var data = {};
+
+ // Workarounds for the fact that JMXJsonServlet returns non-standard JSON strings
+ function workaround(nn) {
+ nn.NodeUsage = JSON.parse(nn.NodeUsage);
+ return nn;
+ }
+
+ load_json(
+ BEANS,
+ guard_with_startup_progress(function(d) {
+ for (var k in d) {
+ data[k] = k === 'federation' ? workaround(d[k].beans[0]) : d[k].beans[0];
+ }
+ render();
+ }),
+ function (url, jqxhr, text, err) {
+ show_err_msg('<p>Failed to retrieve data from ' + url + ', cause: ' + err + '</p>');
+ });
+
+ function render() {
+ var base = dust.makeBase(HELPERS);
+ dust.render('federationhealth', base.push(data), function(err, out) {
+ $('#tab-overview').html(out);
+ $('#ui-tabs a[href="#tab-overview"]').tab('show');
+ });
+ }
+ }
+
+ function load_namenode_info() {
+ var HELPERS = {
+ 'helper_lastcontact_tostring' : function (chunk, ctx, bodies, params) {
+ var value = dust.helpers.tap(params.value, chunk, ctx);
+ return chunk.write('' + new Date(Date.now()-1000*Number(value)));
+ }
+ };
+
+ function workaround(r) {
+ function node_map_to_array(nodes) {
+ var res = [];
+ for (var n in nodes) {
+ var p = nodes[n];
+ p.name = n;
+ res.push(p);
+ }
+ return res;
+ }
+
+ function capitalise(string) {
+ return string.charAt(0).toUpperCase() + string.slice(1).toLowerCase();
+ }
+
+ function augment_namenodes(nodes) {
+ for (var i = 0, e = nodes.length; i < e; ++i) {
+ var n = nodes[i];
+ n.usedPercentage = Math.round(n.used * 1.0 / n.totalSpace * 100);
+ n.title = "Unavailable";
+ n.iconState = "down";
+ if (n.isSafeMode === true) {
+ n.title = capitalise(n.state) + " (safe mode)"
+ n.iconState = "decommissioned";
+ } else if (n.state === "ACTIVE") {
+ n.title = capitalise(n.state);
+ n.iconState = "alive";
+ } else if (nodes[i].state === "STANDBY") {
+ n.title = capitalise(n.state);
+ n.iconState = "down-decommissioned";
+ } else if (nodes[i].state === "UNAVAILABLE") {
+ n.title = capitalise(n.state);
+ n.iconState = "down";
+ }
+ if (n.namenodeId === "null") {
+ n.namenodeId = "";
+ }
+ }
+ }
+
+ r.Nameservices = node_map_to_array(JSON.parse(r.Nameservices));
+ augment_namenodes(r.Nameservices);
+ r.Namenodes = node_map_to_array(JSON.parse(r.Namenodes));
+ augment_namenodes(r.Namenodes);
+ return r;
+ }
+
+ $.get(
+ '/jmx?qry=Hadoop:service=Router,name=FederationState',
+ guard_with_startup_progress(function (resp) {
+ var data = workaround(resp.beans[0]);
+ var base = dust.makeBase(HELPERS);
+ dust.render('namenode-info', base.push(data), function(err, out) {
+ $('#tab-namenode').html(out);
+ $('#ui-tabs a[href="#tab-namenode"]').tab('show');
+ });
+ })).error(ajax_error_handler);
+ }
+
+ function load_router_info() {
+ var HELPERS = {
+ 'helper_lastcontact_tostring' : function (chunk, ctx, bodies, params) {
+ var value = dust.helpers.tap(params.value, chunk, ctx);
+ return chunk.write('' + new Date(Date.now()-1000*Number(value)));
+ }
+ };
+
+ function workaround(r) {
+ function node_map_to_array(nodes) {
+ var res = [];
+ for (var n in nodes) {
+ var p = nodes[n];
+ p.name = n;
+ res.push(p);
+ }
+ return res;
+ }
+
+ function capitalise(string) {
+ return string.charAt(0).toUpperCase() + string.slice(1).toLowerCase();
+ }
+
+ function augment_routers(nodes) {
+ for (var i = 0, e = nodes.length; i < e; ++i) {
+ var n = nodes[i];
+ n.title = "Unavailable"
+ n.iconState = "down";
+ if (n.status === "INITIALIZING") {
+ n.title = capitalise(n.status);
+ n.iconState = "alive";
+ } else if (n.status === "RUNNING") {
+ n.title = capitalise(n.status);
+ n.iconState = "alive";
+ } else if (n.status === "SAFEMODE") {
+ n.title = capitalise(n.status);
+ n.iconState = "down-decommissioned";
+ } else if (n.status === "STOPPING") {
+ n.title = capitalise(n.status);
+ n.iconState = "decommissioned";
+ } else if (n.status === "SHUTDOWN") {
+ n.title = capitalise(n.status);
+ n.iconState = "down";
+ }
+ }
+ }
+
+ r.Routers = node_map_to_array(JSON.parse(r.Routers));
+ augment_routers(r.Routers);
+ return r;
+ }
+
+ $.get(
+ '/jmx?qry=Hadoop:service=Router,name=FederationState',
+ guard_with_startup_progress(function (resp) {
+ var data = workaround(resp.beans[0]);
+ var base = dust.makeBase(HELPERS);
+ dust.render('router-info', base.push(data), function(err, out) {
+ $('#tab-router').html(out);
+ $('#ui-tabs a[href="#tab-router"]').tab('show');
+ });
+ })).error(ajax_error_handler);
+ }
+
+ // TODO Copied directly from dfshealth.js; is there a way to import this function?
+ function load_datanode_info() {
+
+ var HELPERS = {
+ 'helper_relative_time' : function (chunk, ctx, bodies, params) {
+ var value = dust.helpers.tap(params.value, chunk, ctx);
+ return chunk.write(moment().subtract(Number(value), 'seconds').format('YYYY-MM-DD HH:mm:ss'));
+ },
+ 'helper_usage_bar' : function (chunk, ctx, bodies, params) {
+ var value = dust.helpers.tap(params.value, chunk, ctx);
+ var v = Number(value);
+ var r = null;
+ if (v < 70) {
+ r = 'progress-bar-success';
+ } else if (v < 85) {
+ r = 'progress-bar-warning';
+ } else {
+ r = "progress-bar-danger";
+ }
+ return chunk.write(r);
+ },
+ };
+
+ function workaround(r) {
+ function node_map_to_array(nodes) {
+ var res = [];
+ for (var n in nodes) {
+ var p = nodes[n];
+ p.name = n;
+ res.push(p);
+ }
+ return res;
+ }
+
+ function augment_live_nodes(nodes) {
+ for (var i = 0, e = nodes.length; i < e; ++i) {
+ var n = nodes[i];
+ n.usedPercentage = Math.round((n.used + n.nonDfsUsedSpace) * 1.0 / n.capacity * 100);
+ if (n.adminState === "In Service") {
+ n.state = "alive";
+ } else if (nodes[i].adminState === "Decommission In Progress") {
+ n.state = "decommissioning";
+ } else if (nodes[i].adminState === "Decommissioned") {
+ n.state = "decommissioned";
+ }
+ }
+ }
+
+ function augment_dead_nodes(nodes) {
+ for (var i = 0, e = nodes.length; i < e; ++i) {
+ if (nodes[i].decommissioned) {
+ nodes[i].state = "down-decommissioned";
+ } else {
+ nodes[i].state = "down";
+ }
+ }
+ }
+
+ r.LiveNodes = node_map_to_array(JSON.parse(r.LiveNodes));
+ augment_live_nodes(r.LiveNodes);
+ r.DeadNodes = node_map_to_array(JSON.parse(r.DeadNodes));
+ augment_dead_nodes(r.DeadNodes);
+ r.DecomNodes = node_map_to_array(JSON.parse(r.DecomNodes));
+ return r;
+ }
+
+ $.get(
+ '/jmx?qry=Hadoop:service=NameNode,name=NameNodeInfo',
+ guard_with_startup_progress(function (resp) {
+ var data = workaround(resp.beans[0]);
+ var base = dust.makeBase(HELPERS);
+ dust.render('datanode-info', base.push(data), function(err, out) {
+ $('#tab-datanode').html(out);
+ $('#table-datanodes').dataTable( {
+ 'lengthMenu': [ [25, 50, 100, -1], [25, 50, 100, "All"] ],
+ 'columns': [
+ { 'orderDataType': 'ng-value', 'searchable': true },
+ { 'orderDataType': 'ng-value', 'type': 'numeric' },
+ { 'orderDataType': 'ng-value', 'type': 'numeric' },
+ { 'orderDataType': 'ng-value', 'type': 'numeric'}
+ ]});
+ $('#ui-tabs a[href="#tab-datanode"]').tab('show');
+ });
+ })).error(ajax_error_handler);
+ }
+
+ function load_mount_table() {
+ var HELPERS = {}
+
+ function workaround(resource) {
+ resource.MountTable = JSON.parse(resource.MountTable)
+ return resource;
+ }
+
+ $.get(
+ '/jmx?qry=Hadoop:service=Router,name=FederationState',
+ guard_with_startup_progress(function (resp) {
+ var data = workaround(resp.beans[0]);
+ var base = dust.makeBase(HELPERS);
+ dust.render('mounttable', base.push(data), function(err, out) {
+ $('#tab-mounttable').html(out);
+ $('#ui-tabs a[href="#tab-mounttable"]').tab('show');
+ });
+ })).error(ajax_error_handler);
+ }
+
+ function toTitleCase(str) {
+ return str.replace(/\w\S*/g, function(txt){
+ return txt.charAt(0).toUpperCase() + txt.substr(1).toLowerCase();
+ });
+ }
+
+ function show_err_msg(msg) {
+ $('#alert-panel-body').html(msg);
+ $('#alert-panel').show();
+ }
+
+ function ajax_error_handler(url, jqxhr, text, err) {
+ show_err_msg('<p>Failed to retrieve data from ' + url + ', cause: ' + err + '</p>');
+ }
+
+ function guard_with_startup_progress(fn) {
+ return function() {
+ try {
+ fn.apply(this, arguments);
+ } catch (err) {
+ if (err instanceof TypeError) {
+ show_err_msg('Router error: ' + err);
+ }
+ }
+ };
+ }
+
+ function load_page() {
+ var hash = window.location.hash;
+ switch(hash) {
+ case "#tab-overview":
+ load_overview();
+ break;
+ case "#tab-namenode":
+ load_namenode_info();
+ break;
+ case "#tab-router":
+ load_router_info();
+ break;
+ case "#tab-datanode":
+ load_datanode_info();
+ break;
+ case "#tab-mounttable":
+ load_mount_table();
+ break;
+ default:
+ window.location.hash = "tab-overview";
+ break;
+ }
+ }
+ load_page();
+
+ $(window).bind('hashchange', function () {
+ load_page();
+ });
+})();
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/index.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/index.html b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/index.html
new file mode 100644
index 0000000..4ee04ac
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/index.html
@@ -0,0 +1,24 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<meta http-equiv="REFRESH" content="0;url=federationhealth.html" />
+<title>Hadoop Administration</title>
+</head>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/robots.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/robots.txt b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/robots.txt
new file mode 100644
index 0000000..77470cb
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/robots.txt
@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md
new file mode 100644
index 0000000..fdaaa11
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md
@@ -0,0 +1,373 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+HDFS Router-based Federation
+============================
+
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
+
+Introduction
+------------
+
+NameNodes have scalability limits because of the metadata overhead comprised of inodes (files and directories) and file blocks, the number of Datanode heartbeats, and the number of HDFS RPC client requests.
+The common solution is to split the filesystem into smaller subclusters [HDFS Federation](./Federation.html) and provide a federated view [ViewFs](./ViewFs.html).
+The problem is how to maintain the split of the subclusters (e.g., namespace partition), which forces users to connect to multiple subclusters and manage the allocation of folders/files to them.
+
+
+Architecture
+------------
+
+A natural extension to this partitioned federation is to add a layer of software responsible for federating the namespaces.
+This extra layer allows users to access any subcluster transparently, lets subclusters manage their own block pools independently, and will support rebalancing of data across subclusters later
+(see more info in [HDFS-13123](https://issues.apache.org/jira/browse/HDFS-13123)). The subclusters in RBF are not required to be the independent HDFS clusters, a normal federation cluster
+(with multiple block pools) or a mixed cluster with federation and independent cluster is also allowed.
+To accomplish these goals, the federation layer directs block accesses to the proper subcluster, maintains the state of the namespaces, and provides mechanisms for data rebalancing.
+This layer must be scalable, highly available, and fault tolerant.
+
+This federation layer comprises multiple components.
+The _Router_ component that has the same interface as a NameNode, and forwards the client requests to the correct subcluster, based on ground-truth information from a State Store.
+The _State Store_ combines a remote _Mount Table_ (in the flavor of [ViewFs](./ViewFs.html), but shared between clients) and utilization (load/capacity) information about the subclusters.
+This approach has the same architecture as [YARN federation](../hadoop-yarn/Federation.html).
+
+
+
+
+### Example flow
+The simplest configuration deploys a Router on each NameNode machine.
+The Router monitors the local NameNode and heartbeats the state to the State Store.
+When a regular DFS client contacts any of the Routers to access a file in the federated filesystem, the Router checks the Mount Table in the State Store (i.e., the local cache) to find out which subcluster contains the file.
+Then it checks the Membership table in the State Store (i.e., the local cache) for the NameNode responsible for the subcluster.
+After it has identified the correct NameNode, the Router proxies the request.
+The client accesses Datanodes directly.
+
+
+### Router
+There can be multiple Routers in the system with soft state.
+Each Router has two roles:
+
+* Federated interface: expose a single, global NameNode interface to the clients and forward the requests to the active NameNode in the correct subcluster
+* NameNode heartbeat: maintain the information about a NameNode in the State Store
+
+#### Federated interface
+The Router receives a client request, checks the State Store for the correct subcluster, and forwards the request to the active NameNode of that subcluster.
+The reply from the NameNode then flows in the opposite direction.
+The Routers are stateless and can be behind a load balancer.
+For performance, the Router also caches remote mount table entries and the state of the subclusters.
+To make sure that changes have been propagated to all Routers, each Router heartbeats its state to the State Store.
+
+The communications between the Routers and the State Store are cached (with timed expiration for freshness).
+This improves the performance of the system.
+
+#### NameNode heartbeat
+For this role, the Router periodically checks the state of a NameNode (usually on the same server) and reports their high availability (HA) state and load/space status to the State Store.
+Note that this is an optional role, as a Router can be independent of any subcluster.
+For performance with NameNode HA, the Router uses the high availability state information in the State Store to forward the request to the NameNode that is most likely to be active.
+Note that this service can be embedded into the NameNode itself to simplify the operation.
+
+#### Availability and fault tolerance
+The Router operates with failures at multiple levels.
+
+* **Federated interface HA:**
+The Routers are stateless and metadata operations are atomic at the NameNodes.
+If a Router becomes unavailable, any Router can take over for it.
+The clients configure their DFS HA client (e.g., ConfiguredFailoverProvider or RequestHedgingProxyProvider) with all the Routers in the federation as endpoints.
+
+* **Unavailable State Store:**
+If a Router cannot contact the State Store, it will enter into a Safe Mode state which disallows it from serving requests.
+Clients will treat Routers in Safe Mode as it was an Standby NameNode and try another Router. There is a manual way to manage the Safe Mode for the Router.
+
+The Safe Mode state can be managed by using the following command:
+
+ [hdfs]$ $HADOOP_HOME/bin/hdfs dfsrouteradmin -safemode enter | leave | get
+
+* **NameNode heartbeat HA:**
+For high availability and flexibility, multiple Routers can monitor the same NameNode and heartbeat the information to the State Store.
+This increases clients' resiliency to stale information, should a Router fail.
+Conflicting NameNode information in the State Store is resolved by each Router via a quorum.
+
+* **Unavailable NameNodes:**
+If a Router cannot contact the active NameNode, then it will try the other NameNodes in the subcluster.
+It will first try those reported as standby and then the unavailable ones.
+If the Router cannot reach any NameNode, then it throws an exception.
+
+* **Expired NameNodes:**
+If a NameNode heartbeat has not been recorded in the State Store for a multiple of the heartbeat interval, the monitoring Router will record that the NameNode has expired and no Routers will attempt to access it.
+If an updated heartbeat is subsequently recorded for the NameNode, the monitoring Router will restore the NameNode from the expired state.
+
+#### Interfaces
+To interact with the users and the administrators, the Router exposes multiple interfaces.
+
+* **RPC:**
+The Router RPC implements the most common interfaces clients use to interact with HDFS.
+The current implementation has been tested using analytics workloads written in plain MapReduce, Spark, and Hive (on Tez, Spark, and MapReduce).
+Advanced functions like snapshot, encryption and tiered storage are left for future versions.
+All unimplemented functions will throw exceptions.
+
+* **Admin:**
+Administrators can query information from clusters and add/remove entries from the mount table over RPC.
+This interface is also exposed through the command line to get and modify information from the federation.
+
+* **Web UI:**
+The Router exposes a Web UI visualizing the state of the federation, mimicking the current NameNode UI.
+It displays information about the mount table, membership information about each subcluster, and the status of the Routers.
+
+* **WebHDFS:**
+The Router provides the HDFS REST interface (WebHDFS) in addition to the RPC one.
+
+* **JMX:**
+It exposes metrics through JMX mimicking the NameNode.
+This is used by the Web UI to get the cluster status.
+
+Some operations are not available in Router-based federation.
+The Router throws exceptions for those.
+Examples users may encounter include the following.
+
+* Rename file/folder in two different nameservices.
+* Copy file/folder in two different nameservices.
+* Write into a file/folder being rebalanced.
+
+### Quota management
+Federation supports and controls global quota at mount table level.
+For performance reasons, the Router caches the quota usage and updates it periodically. These quota usage values
+will be used for quota-verification during each WRITE RPC call invoked in RouterRPCSever. See [HDFS Quotas Guide](./HdfsQuotaAdminGuide.html)
+for the quota detail.
+
+### State Store
+The (logically centralized, but physically distributed) State Store maintains:
+
+* The state of the subclusters in terms of their block access load, available disk space, HA state, etc.
+* The mapping between folder/files and subclusters, i.e. the remote mount table.
+
+The backend of the State Store is pluggable.
+We leverage the fault tolerance of the backend implementations.
+The main information stored in the State Store and its implementation:
+
+* **Membership**:
+The membership information encodes the state of the NameNodes in the federation.
+This includes information about the subcluster, such as storage capacity and the number of nodes.
+The Router periodically heartbeats this information about one or more NameNodes.
+Given that multiple Routers can monitor a single NameNode, the heartbeat from every Router is stored.
+The Routers apply a quorum of the data when querying this information from the State Store.
+The Routers discard the entries older than a certain threshold (e.g., ten Router heartbeat periods).
+
+* **Mount Table**:
+This table hosts the mapping between folders and subclusters.
+It is similar to the mount table in [ViewFs](.ViewFs.html) where it specifies the federated folder, the destination subcluster and the path in that folder.
+
+
+### Security
+Secure authentication and authorization are not supported yet, so the Router will not proxy to Hadoop clusters with security enabled.
+
+
+Deployment
+----------
+
+By default, the Router is ready to take requests and monitor the NameNode in the local machine.
+It needs to know the State Store endpoint by setting `dfs.federation.router.store.driver.class`.
+The rest of the options are documented in [hdfs-default.xml](./hdfs-default.xml).
+
+Once the Router is configured, it can be started:
+
+ [hdfs]$ $HADOOP_PREFIX/sbin/hadoop-daemon.sh --script $HADOOP_PREFIX/bin/hdfs start dfsrouter
+
+And to stop it:
+
+ [hdfs]$ $HADOOP_PREFIX/sbin/hadoop-daemon.sh --script $HADOOP_PREFIX/bin/hdfs stop dfsrouter
+
+### Mount table management
+
+The mount table entries are pretty much the same as in [ViewFs](./ViewFs.html).
+A good practice for simplifying the management is to name the federated namespace with the same names as the destination namespaces.
+For example, if we to mount `/data/app1` in the federated namespace, it is recommended to have that same name as in the destination namespace.
+
+The federation admin tool supports managing the mount table.
+For example, to create three mount points and list them:
+
+ [hdfs]$ $HADOOP_HOME/bin/hdfs dfsrouteradmin -add /tmp ns1 /tmp
+ [hdfs]$ $HADOOP_HOME/bin/hdfs dfsrouteradmin -add /data/app1 ns2 /data/app1
+ [hdfs]$ $HADOOP_HOME/bin/hdfs dfsrouteradmin -add /data/app2 ns3 /data/app2
+ [hdfs]$ $HADOOP_HOME/bin/hdfs dfsrouteradmin -ls
+
+It also supports mount points that disallow writes:
+
+ [hdfs]$ $HADOOP_HOME/bin/hdfs dfsrouteradmin -add /readonly ns1 / -readonly
+
+If a mount point is not set, the Router will map it to the default namespace `dfs.federation.router.default.nameserviceId`.
+
+Mount table have UNIX-like *permissions*, which restrict which users and groups have access to the mount point. Write permissions allow users to add
+, update or remove mount point. Read permissions allow users to list mount point. Execute permissions are unused.
+
+Mount table permission can be set by following command:
+
+ [hdfs]$ $HADOOP_HOME/bin/hdfs dfsrouteradmin -add /tmp ns1 /tmp -owner root -group supergroup -mode 0755
+
+The option mode is UNIX-style permissions for the mount table. Permissions are specified in octal, e.g. 0755. By default, this is set to 0755.
+
+Router-based federation supports global quota at mount table level. Mount table entries may spread multiple subclusters and the global quota will be
+accounted across these subclusters.
+
+The federation admin tool supports setting quotas for specified mount table entries:
+
+ [hdfs]$ $HADOOP_HOME/bin/hdfs dfsrouteradmin -setQuota /path -nsQuota 100 -ssQuota 1024
+
+The above command means that we allow the path to have a maximum of 100 file/directories and use at most 1024 bytes storage space. The parameter for *ssQuota*
+supports multiple size-unit suffix (e.g. 1k is 1KB, 5m is 5MB). If no suffix is specified then bytes is assumed.
+
+Ls command will show below information for each mount table entry:
+
+ Source Destinations Owner Group Mode Quota/Usage
+ /path ns0->/path root supergroup rwxr-xr-x [NsQuota: 50/0, SsQuota: 100 B/0 B]
+
+Client configuration
+--------------------
+
+For clients to use the federated namespace, they need to create a new one that points to the routers.
+For example, a cluster with 4 namespaces **ns0, ns1, ns2, ns3**, can add a new one to **hdfs-site.xml** called **ns-fed** which points to two of the routers:
+
+```xml
+<configuration>
+ <property>
+ <name>dfs.nameservices</name>
+ <value>ns0,ns1,ns2,ns3,ns-fed</value>
+ </property>
+ <property>
+ <name>dfs.ha.namenodes.ns-fed</name>
+ <value>r1,r2</value>
+ </property>
+ <property>
+ <name>dfs.namenode.rpc-address.ns-fed.r1</name>
+ <value>router1:rpc-port</value>
+ </property>
+ <property>
+ <name>dfs.namenode.rpc-address.ns-fed.r2</name>
+ <value>router2:rpc-port</value>
+ </property>
+ <property>
+ <name>dfs.client.failover.proxy.provider.ns-fed</name>
+ <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
+ </property>
+ <property>
+ <name>dfs.client.failover.random.order</name>
+ <value>true</value>
+ </property>
+</configuration>
+```
+
+The `dfs.client.failover.random.order` set to `true` allows distributing the load evenly across the routers.
+
+With this setting a user can interact with `ns-fed` as a regular namespace:
+
+ $ $HADOOP_HOME/bin/hdfs dfs -ls hdfs://ns-fed/
+ /tmp
+ /data
+
+This federated namespace can also be set as the default one at **core-site.xml** using `fs.defaultFS`.
+
+
+Router configuration
+--------------------
+
+One can add the configurations for Router-based federation to **hdfs-site.xml**.
+The main options are documented in [hdfs-default.xml](./hdfs-default.xml).
+The configuration values are described in this section.
+
+### RPC server
+
+The RPC server to receive connections from the clients.
+
+| Property | Default | Description|
+|:---- |:---- |:---- |
+| dfs.federation.router.default.nameserviceId | | Nameservice identifier of the default subcluster to monitor. |
+| dfs.federation.router.rpc.enable | `true` | If `true`, the RPC service to handle client requests in the router is enabled. |
+| dfs.federation.router.rpc-address | 0.0.0.0:8888 | RPC address that handles all clients requests. |
+| dfs.federation.router.rpc-bind-host | 0.0.0.0 | The actual address the RPC server will bind to. |
+| dfs.federation.router.handler.count | 10 | The number of server threads for the router to handle RPC requests from clients. |
+| dfs.federation.router.handler.queue.size | 100 | The size of the queue for the number of handlers to handle RPC client requests. |
+| dfs.federation.router.reader.count | 1 | The number of readers for the router to handle RPC client requests. |
+| dfs.federation.router.reader.queue.size | 100 | The size of the queue for the number of readers for the router to handle RPC client requests. |
+
+#### Connection to the Namenodes
+
+The Router forwards the client requests to the NameNodes.
+It uses a pool of connections to reduce the latency of creating them.
+
+| Property | Default | Description|
+|:---- |:---- |:---- |
+| dfs.federation.router.connection.pool-size | 1 | Size of the pool of connections from the router to namenodes. |
+| dfs.federation.router.connection.clean.ms | 10000 | Time interval, in milliseconds, to check if the connection pool should remove unused connections. |
+| dfs.federation.router.connection.pool.clean.ms | 60000 | Time interval, in milliseconds, to check if the connection manager should remove unused connection pools. |
+
+### Admin server
+
+The administration server to manage the Mount Table.
+
+| Property | Default | Description|
+|:---- |:---- |:---- |
+| dfs.federation.router.admin.enable | `true` | If `true`, the RPC admin service to handle client requests in the router is enabled. |
+| dfs.federation.router.admin-address | 0.0.0.0:8111 | RPC address that handles the admin requests. |
+| dfs.federation.router.admin-bind-host | 0.0.0.0 | The actual address the RPC admin server will bind to. |
+| dfs.federation.router.admin.handler.count | 1 | The number of server threads for the router to handle RPC requests from admin. |
+
+### State Store
+
+The connection to the State Store and the internal caching at the Router.
+
+| Property | Default | Description|
+|:---- |:---- |:---- |
+| dfs.federation.router.store.enable | `true` | If `true`, the Router connects to the State Store. |
+| dfs.federation.router.store.serializer | `org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreSerializerPBImpl` | Class to serialize State Store records. |
+| dfs.federation.router.store.driver.class | `org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl` | Class to implement the State Store. |
+| dfs.federation.router.store.connection.test | 60000 | How often to check for the connection to the State Store in milliseconds. |
+| dfs.federation.router.cache.ttl | 60000 | How often to refresh the State Store caches in milliseconds. |
+| dfs.federation.router.store.membership.expiration | 300000 | Expiration time in milliseconds for a membership record. |
+
+### Routing
+
+Forwarding client requests to the right subcluster.
+
+| Property | Default | Description|
+|:---- |:---- |:---- |
+| dfs.federation.router.file.resolver.client.class | `org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver` | Class to resolve files to subclusters. |
+| dfs.federation.router.namenode.resolver.client.class | `org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver` | Class to resolve the namenode for a subcluster. |
+
+### Namenode monitoring
+
+Monitor the namenodes in the subclusters for forwarding the client requests.
+
+| Property | Default | Description|
+|:---- |:---- |:---- |
+| dfs.federation.router.heartbeat.enable | `true` | If `true`, the Router heartbeats into the State Store. |
+| dfs.federation.router.heartbeat.interval | 5000 | How often the Router should heartbeat into the State Store in milliseconds. |
+| dfs.federation.router.monitor.namenode | | The identifier of the namenodes to monitor and heartbeat. |
+| dfs.federation.router.monitor.localnamenode.enable | `true` | If `true`, the Router should monitor the namenode in the local machine. |
+
+Note: The config *dfs.nameservice.id* is recommended to configure if *dfs.federation.router.monitor.localnamenode.enable* is enabled.
+This will allow the Router finding the local node directly. Otherwise, it will find the nameservice Id by matching namenode RPC address with the
+local node address. If multiple addresses are matched, the Router will fail to start. In addition, if the local node is in a HA mode, it is recommend
+to configure *dfs.ha.namenode.id*.
+
+### Quota
+
+Global quota supported in federation.
+
+| Property | Default | Description|
+|:---- |:---- |:---- |
+| dfs.federation.router.quota.enable | `false` | If `true`, the quota system enabled in the Router. |
+| dfs.federation.router.quota-cache.update.interval | 60s | How often the Router updates quota cache. This setting supports multiple time unit suffixes. If no suffix is specified then milliseconds is assumed. |
+
+Metrics
+-------
+
+The Router and State Store statistics are exposed in metrics/JMX. These info will be very useful for monitoring.
+More metrics info can see [Router RPC Metrics](../../hadoop-project-dist/hadoop-common/Metrics.html#RouterRPCMetrics) and [State Store Metrics](../../hadoop-project-dist/hadoop-common/Metrics.html#StateStoreMetrics).
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/resources/css/site.css
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/resources/css/site.css b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/resources/css/site.css
new file mode 100644
index 0000000..f830baa
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/resources/css/site.css
@@ -0,0 +1,30 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+#banner {
+ height: 93px;
+ background: none;
+}
+
+#bannerLeft img {
+ margin-left: 30px;
+ margin-top: 10px;
+}
+
+#bannerRight img {
+ margin: 17px;
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/resources/images/routerfederation.png
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/resources/images/routerfederation.png b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/resources/images/routerfederation.png
new file mode 100644
index 0000000..2b158bb
Binary files /dev/null and b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/resources/images/routerfederation.png differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/site.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/site.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/site.xml
new file mode 100644
index 0000000..5feae9a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/site.xml
@@ -0,0 +1,28 @@
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project name="Apache Hadoop ${project.version}">
+
+ <skin>
+ <groupId>org.apache.maven.skins</groupId>
+ <artifactId>maven-stylus-skin</artifactId>
+ <version>${maven-stylus-skin.version}</version>
+ </skin>
+
+ <body>
+ <links>
+ <item name="Apache Hadoop" href="http://hadoop.apache.org/"/>
+ </links>
+ </body>
+
+</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
new file mode 100644
index 0000000..0ead93e
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
@@ -0,0 +1,246 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+import java.io.BufferedReader;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.lang.management.ManagementFactory;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.Date;
+import java.util.List;
+import java.util.Random;
+
+import javax.management.JMX;
+import javax.management.MBeanServer;
+import javax.management.MalformedObjectNameException;
+import javax.management.ObjectName;
+
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.test.GenericTestUtils;
+
+import com.google.common.base.Supplier;
+
+/**
+ * Helper utilities for testing HDFS Federation.
+ */
+public final class FederationTestUtils {
+
+ public final static String[] NAMESERVICES = {"ns0", "ns1"};
+ public final static String[] NAMENODES = {"nn0", "nn1", "nn2", "nn3"};
+ public final static String[] ROUTERS =
+ {"router0", "router1", "router2", "router3"};
+
+
+ private FederationTestUtils() {
+ // Utility class
+ }
+
+ public static void verifyException(Object obj, String methodName,
+ Class<? extends Exception> exceptionClass, Class<?>[] parameterTypes,
+ Object[] arguments) {
+
+ Throwable triggeredException = null;
+ try {
+ Method m = obj.getClass().getMethod(methodName, parameterTypes);
+ m.invoke(obj, arguments);
+ } catch (InvocationTargetException ex) {
+ triggeredException = ex.getTargetException();
+ } catch (Exception e) {
+ triggeredException = e;
+ }
+ if (exceptionClass != null) {
+ assertNotNull("No exception was triggered, expected exception"
+ + exceptionClass.getName(), triggeredException);
+ assertEquals(exceptionClass, triggeredException.getClass());
+ } else {
+ assertNull("Exception was triggered but no exception was expected",
+ triggeredException);
+ }
+ }
+
+ public static NamenodeStatusReport createNamenodeReport(String ns, String nn,
+ HAServiceState state) {
+ Random rand = new Random();
+ NamenodeStatusReport report = new NamenodeStatusReport(ns, nn,
+ "localhost:" + rand.nextInt(10000), "localhost:" + rand.nextInt(10000),
+ "localhost:" + rand.nextInt(10000), "testwebaddress-" + ns + nn);
+ if (state == null) {
+ // Unavailable, no additional info
+ return report;
+ }
+ report.setHAServiceState(state);
+ NamespaceInfo nsInfo = new NamespaceInfo(
+ 1, "tesclusterid", ns, 0, "testbuildvesion", "testsoftwareversion");
+ report.setNamespaceInfo(nsInfo);
+ return report;
+ }
+
+ /**
+ * Wait for a namenode to be registered with a particular state.
+ * @param resolver Active namenode resolver.
+ * @param nsId Nameservice identifier.
+ * @param nnId Namenode identifier.
+ * @param finalState State to check for.
+ * @throws Exception Failed to verify State Store registration of namenode
+ * nsId:nnId for state.
+ */
+ public static void waitNamenodeRegistered(
+ final ActiveNamenodeResolver resolver,
+ final String nsId, final String nnId,
+ final FederationNamenodeServiceState state) throws Exception {
+
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ try {
+ List<? extends FederationNamenodeContext> namenodes =
+ resolver.getNamenodesForNameserviceId(nsId);
+ if (namenodes != null) {
+ for (FederationNamenodeContext namenode : namenodes) {
+ // Check if this is the Namenode we are checking
+ if (namenode.getNamenodeId() == nnId ||
+ namenode.getNamenodeId().equals(nnId)) {
+ return state == null || namenode.getState().equals(state);
+ }
+ }
+ }
+ } catch (IOException e) {
+ // Ignore
+ }
+ return false;
+ }
+ }, 1000, 20 * 1000);
+ }
+
+ public static boolean verifyDate(Date d1, Date d2, long precision) {
+ return Math.abs(d1.getTime() - d2.getTime()) < precision;
+ }
+
+ public static <T> T getBean(String name, Class<T> obj)
+ throws MalformedObjectNameException {
+ MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer();
+ ObjectName poolName = new ObjectName(name);
+ return JMX.newMXBeanProxy(mBeanServer, poolName, obj);
+ }
+
+ public static boolean addDirectory(FileSystem context, String path)
+ throws IOException {
+ context.mkdirs(new Path(path), new FsPermission("777"));
+ return verifyFileExists(context, path);
+ }
+
+ public static FileStatus getFileStatus(FileSystem context, String path)
+ throws IOException {
+ return context.getFileStatus(new Path(path));
+ }
+
+ public static boolean verifyFileExists(FileSystem context, String path) {
+ try {
+ FileStatus status = getFileStatus(context, path);
+ if (status != null) {
+ return true;
+ }
+ } catch (Exception e) {
+ return false;
+ }
+ return false;
+ }
+
+ public static boolean checkForFileInDirectory(
+ FileSystem context, String testPath, String targetFile)
+ throws IOException, AccessControlException, FileNotFoundException,
+ UnsupportedFileSystemException, IllegalArgumentException {
+
+ FileStatus[] fileStatus = context.listStatus(new Path(testPath));
+ String file = null;
+ String verifyPath = testPath + "/" + targetFile;
+ if (testPath.equals("/")) {
+ verifyPath = testPath + targetFile;
+ }
+
+ Boolean found = false;
+ for (int i = 0; i < fileStatus.length; i++) {
+ FileStatus f = fileStatus[i];
+ file = Path.getPathWithoutSchemeAndAuthority(f.getPath()).toString();
+ if (file.equals(verifyPath)) {
+ found = true;
+ }
+ }
+ return found;
+ }
+
+ public static int countContents(FileSystem context, String testPath)
+ throws IOException {
+ Path path = new Path(testPath);
+ FileStatus[] fileStatus = context.listStatus(path);
+ return fileStatus.length;
+ }
+
+ public static void createFile(FileSystem fs, String path, long length)
+ throws IOException {
+ FsPermission permissions = new FsPermission("700");
+ FSDataOutputStream writeStream = fs.create(new Path(path), permissions,
+ true, 1000, (short) 1, DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT, null);
+ for (int i = 0; i < length; i++) {
+ writeStream.write(i);
+ }
+ writeStream.close();
+ }
+
+ public static String readFile(FileSystem fs, String path) throws IOException {
+ // Read the file from the filesystem via the active namenode
+ Path fileName = new Path(path);
+ InputStreamReader reader = new InputStreamReader(fs.open(fileName));
+ BufferedReader bufferedReader = new BufferedReader(reader);
+ StringBuilder data = new StringBuilder();
+ String line;
+
+ while ((line = bufferedReader.readLine()) != null) {
+ data.append(line);
+ }
+
+ bufferedReader.close();
+ reader.close();
+ return data.toString();
+ }
+
+ public static boolean deleteFile(FileSystem fs, String path)
+ throws IOException {
+ return fs.delete(new Path(path), true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
new file mode 100644
index 0000000..151d731
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
@@ -0,0 +1,313 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.NamenodePriorityComparator;
+import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport;
+import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.federation.router.Router;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.util.Time;
+
+/**
+ * In-memory cache/mock of a namenode and file resolver. Stores the most
+ * recently updated NN information for each nameservice and block pool. It also
+ * stores a virtual mount table for resolving global namespace paths to local NN
+ * paths.
+ */
+public class MockResolver
+ implements ActiveNamenodeResolver, FileSubclusterResolver {
+
+ private Map<String, List<? extends FederationNamenodeContext>> resolver =
+ new HashMap<>();
+ private Map<String, List<RemoteLocation>> locations = new HashMap<>();
+ private Set<FederationNamespaceInfo> namespaces = new HashSet<>();
+ private String defaultNamespace = null;
+
+ public MockResolver() {
+ this.cleanRegistrations();
+ }
+
+ public MockResolver(Configuration conf) {
+ this();
+ }
+
+ public MockResolver(Configuration conf, StateStoreService store) {
+ this();
+ }
+
+ public MockResolver(Configuration conf, Router router) {
+ this();
+ }
+
+ public void addLocation(String mount, String nsId, String location) {
+ List<RemoteLocation> locationsList = this.locations.get(mount);
+ if (locationsList == null) {
+ locationsList = new LinkedList<>();
+ this.locations.put(mount, locationsList);
+ }
+
+ final RemoteLocation remoteLocation = new RemoteLocation(nsId, location);
+ if (!locationsList.contains(remoteLocation)) {
+ locationsList.add(remoteLocation);
+ }
+
+ if (this.defaultNamespace == null) {
+ this.defaultNamespace = nsId;
+ }
+ }
+
+ public synchronized void cleanRegistrations() {
+ this.resolver = new HashMap<>();
+ this.namespaces = new HashSet<>();
+ }
+
+ @Override
+ public void updateActiveNamenode(
+ String nsId, InetSocketAddress successfulAddress) {
+
+ String address = successfulAddress.getHostName() + ":" +
+ successfulAddress.getPort();
+ String key = nsId;
+ if (key != null) {
+ // Update the active entry
+ @SuppressWarnings("unchecked")
+ List<FederationNamenodeContext> namenodes =
+ (List<FederationNamenodeContext>) this.resolver.get(key);
+ for (FederationNamenodeContext namenode : namenodes) {
+ if (namenode.getRpcAddress().equals(address)) {
+ MockNamenodeContext nn = (MockNamenodeContext) namenode;
+ nn.setState(FederationNamenodeServiceState.ACTIVE);
+ break;
+ }
+ }
+ // This operation modifies the list so we need to be careful
+ synchronized(namenodes) {
+ Collections.sort(namenodes, new NamenodePriorityComparator());
+ }
+ }
+ }
+
+ @Override
+ public List<? extends FederationNamenodeContext>
+ getNamenodesForNameserviceId(String nameserviceId) {
+ // Return a copy of the list because it is updated periodically
+ List<? extends FederationNamenodeContext> namenodes =
+ this.resolver.get(nameserviceId);
+ return Collections.unmodifiableList(new ArrayList<>(namenodes));
+ }
+
+ @Override
+ public List<? extends FederationNamenodeContext> getNamenodesForBlockPoolId(
+ String blockPoolId) {
+ // Return a copy of the list because it is updated periodically
+ List<? extends FederationNamenodeContext> namenodes =
+ this.resolver.get(blockPoolId);
+ return Collections.unmodifiableList(new ArrayList<>(namenodes));
+ }
+
+ private static class MockNamenodeContext
+ implements FederationNamenodeContext {
+
+ private String namenodeId;
+ private String nameserviceId;
+
+ private String webAddress;
+ private String rpcAddress;
+ private String serviceAddress;
+ private String lifelineAddress;
+
+ private FederationNamenodeServiceState state;
+ private long dateModified;
+
+ MockNamenodeContext(
+ String rpc, String service, String lifeline, String web,
+ String ns, String nn, FederationNamenodeServiceState state) {
+ this.rpcAddress = rpc;
+ this.serviceAddress = service;
+ this.lifelineAddress = lifeline;
+ this.webAddress = web;
+ this.namenodeId = nn;
+ this.nameserviceId = ns;
+ this.state = state;
+ this.dateModified = Time.now();
+ }
+
+ public void setState(FederationNamenodeServiceState newState) {
+ this.state = newState;
+ this.dateModified = Time.now();
+ }
+
+ @Override
+ public String getRpcAddress() {
+ return rpcAddress;
+ }
+
+ @Override
+ public String getServiceAddress() {
+ return serviceAddress;
+ }
+
+ @Override
+ public String getLifelineAddress() {
+ return lifelineAddress;
+ }
+
+ @Override
+ public String getWebAddress() {
+ return webAddress;
+ }
+
+ @Override
+ public String getNamenodeKey() {
+ return nameserviceId + " " + namenodeId + " " + rpcAddress;
+ }
+
+ @Override
+ public String getNameserviceId() {
+ return nameserviceId;
+ }
+
+ @Override
+ public String getNamenodeId() {
+ return namenodeId;
+ }
+
+ @Override
+ public FederationNamenodeServiceState getState() {
+ return state;
+ }
+
+ @Override
+ public long getDateModified() {
+ return dateModified;
+ }
+ }
+
+ @Override
+ public synchronized boolean registerNamenode(NamenodeStatusReport report)
+ throws IOException {
+
+ MockNamenodeContext context = new MockNamenodeContext(
+ report.getRpcAddress(), report.getServiceAddress(),
+ report.getLifelineAddress(), report.getWebAddress(),
+ report.getNameserviceId(), report.getNamenodeId(), report.getState());
+
+ String nsId = report.getNameserviceId();
+ String bpId = report.getBlockPoolId();
+ String cId = report.getClusterId();
+
+ @SuppressWarnings("unchecked")
+ List<MockNamenodeContext> existingItems =
+ (List<MockNamenodeContext>) this.resolver.get(nsId);
+ if (existingItems == null) {
+ existingItems = new ArrayList<>();
+ this.resolver.put(bpId, existingItems);
+ this.resolver.put(nsId, existingItems);
+ }
+ boolean added = false;
+ for (int i=0; i<existingItems.size() && !added; i++) {
+ MockNamenodeContext existing = existingItems.get(i);
+ if (existing.getNamenodeKey().equals(context.getNamenodeKey())) {
+ existingItems.set(i, context);
+ added = true;
+ }
+ }
+ if (!added) {
+ existingItems.add(context);
+ }
+ Collections.sort(existingItems, new NamenodePriorityComparator());
+
+ FederationNamespaceInfo info = new FederationNamespaceInfo(bpId, cId, nsId);
+ this.namespaces.add(info);
+ return true;
+ }
+
+ @Override
+ public Set<FederationNamespaceInfo> getNamespaces() throws IOException {
+ return this.namespaces;
+ }
+
+ @Override
+ public PathLocation getDestinationForPath(String path) throws IOException {
+ List<RemoteLocation> remoteLocations = new LinkedList<>();
+ // We go from the leaves to the root
+ List<String> keys = new ArrayList<>(this.locations.keySet());
+ Collections.sort(keys, Collections.reverseOrder());
+ for (String key : keys) {
+ if (path.startsWith(key)) {
+ for (RemoteLocation location : this.locations.get(key)) {
+ String finalPath = location.getDest() + path.substring(key.length());
+ String nameservice = location.getNameserviceId();
+ RemoteLocation remoteLocation =
+ new RemoteLocation(nameservice, finalPath);
+ remoteLocations.add(remoteLocation);
+ }
+ break;
+ }
+ }
+ if (remoteLocations.isEmpty()) {
+ // Path isn't supported, mimic resolver behavior.
+ return null;
+ }
+ return new PathLocation(path, remoteLocations);
+ }
+
+ @Override
+ public List<String> getMountPoints(String path) throws IOException {
+ List<String> mounts = new ArrayList<>();
+ if (path.equals("/")) {
+ // Mounts only supported under root level
+ for (String mount : this.locations.keySet()) {
+ if (mount.length() > 1) {
+ // Remove leading slash, this is the behavior of the mount tree,
+ // return only names.
+ mounts.add(mount.replace("/", ""));
+ }
+ }
+ }
+ return mounts;
+ }
+
+ @Override
+ public void setRouterId(String router) {
+ }
+
+ @Override
+ public String getDefaultNamespace() {
+ return defaultNamespace;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
new file mode 100644
index 0000000..be0de52
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
@@ -0,0 +1,163 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+
+/**
+ * Constructs a router configuration with individual features enabled/disabled.
+ */
+public class RouterConfigBuilder {
+
+ private Configuration conf;
+
+ private boolean enableRpcServer = false;
+ private boolean enableAdminServer = false;
+ private boolean enableHttpServer = false;
+ private boolean enableHeartbeat = false;
+ private boolean enableLocalHeartbeat = false;
+ private boolean enableStateStore = false;
+ private boolean enableMetrics = false;
+ private boolean enableQuota = false;
+ private boolean enableSafemode = false;
+
+ public RouterConfigBuilder(Configuration configuration) {
+ this.conf = configuration;
+ }
+
+ public RouterConfigBuilder() {
+ this.conf = new Configuration(false);
+ }
+
+ public RouterConfigBuilder all() {
+ this.enableRpcServer = true;
+ this.enableAdminServer = true;
+ this.enableHttpServer = true;
+ this.enableHeartbeat = true;
+ this.enableLocalHeartbeat = true;
+ this.enableStateStore = true;
+ this.enableMetrics = true;
+ this.enableSafemode = true;
+ return this;
+ }
+
+ public RouterConfigBuilder enableLocalHeartbeat(boolean enable) {
+ this.enableLocalHeartbeat = enable;
+ return this;
+ }
+
+ public RouterConfigBuilder rpc(boolean enable) {
+ this.enableRpcServer = enable;
+ return this;
+ }
+
+ public RouterConfigBuilder admin(boolean enable) {
+ this.enableAdminServer = enable;
+ return this;
+ }
+
+ public RouterConfigBuilder http(boolean enable) {
+ this.enableHttpServer = enable;
+ return this;
+ }
+
+ public RouterConfigBuilder heartbeat(boolean enable) {
+ this.enableHeartbeat = enable;
+ return this;
+ }
+
+ public RouterConfigBuilder stateStore(boolean enable) {
+ this.enableStateStore = enable;
+ return this;
+ }
+
+ public RouterConfigBuilder metrics(boolean enable) {
+ this.enableMetrics = enable;
+ return this;
+ }
+
+ public RouterConfigBuilder quota(boolean enable) {
+ this.enableQuota = enable;
+ return this;
+ }
+
+ public RouterConfigBuilder safemode(boolean enable) {
+ this.enableSafemode = enable;
+ return this;
+ }
+
+ public RouterConfigBuilder rpc() {
+ return this.rpc(true);
+ }
+
+ public RouterConfigBuilder admin() {
+ return this.admin(true);
+ }
+
+ public RouterConfigBuilder http() {
+ return this.http(true);
+ }
+
+ public RouterConfigBuilder heartbeat() {
+ return this.heartbeat(true);
+ }
+
+ public RouterConfigBuilder stateStore() {
+ // reset the State Store driver implementation class for testing
+ conf.setClass(RBFConfigKeys.FEDERATION_STORE_DRIVER_CLASS,
+ FederationStateStoreTestUtils.getTestDriverClass(),
+ StateStoreDriver.class);
+ return this.stateStore(true);
+ }
+
+ public RouterConfigBuilder metrics() {
+ return this.metrics(true);
+ }
+
+ public RouterConfigBuilder quota() {
+ return this.quota(true);
+ }
+
+ public RouterConfigBuilder safemode() {
+ return this.safemode(true);
+ }
+
+ public Configuration build() {
+ conf.setBoolean(RBFConfigKeys.DFS_ROUTER_STORE_ENABLE,
+ this.enableStateStore);
+ conf.setBoolean(RBFConfigKeys.DFS_ROUTER_RPC_ENABLE, this.enableRpcServer);
+ conf.setBoolean(RBFConfigKeys.DFS_ROUTER_ADMIN_ENABLE,
+ this.enableAdminServer);
+ conf.setBoolean(RBFConfigKeys.DFS_ROUTER_HTTP_ENABLE,
+ this.enableHttpServer);
+ conf.setBoolean(RBFConfigKeys.DFS_ROUTER_HEARTBEAT_ENABLE,
+ this.enableHeartbeat);
+ conf.setBoolean(RBFConfigKeys.DFS_ROUTER_MONITOR_LOCAL_NAMENODE,
+ this.enableLocalHeartbeat);
+ conf.setBoolean(RBFConfigKeys.DFS_ROUTER_METRICS_ENABLE,
+ this.enableMetrics);
+ conf.setBoolean(RBFConfigKeys.DFS_ROUTER_QUOTA_ENABLE,
+ this.enableQuota);
+ conf.setBoolean(RBFConfigKeys.DFS_ROUTER_SAFEMODE_ENABLE,
+ this.enableSafemode);
+ return conf;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/federationhealth.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/federationhealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/federationhealth.html
deleted file mode 100644
index 71e2132..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/federationhealth.html
+++ /dev/null
@@ -1,424 +0,0 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<html xmlns="http://www.w3.org/1999/xhtml">
-<head>
-<meta http-equiv="X-UA-Compatible" content="IE=9" />
-<link rel="stylesheet" type="text/css" href="/static/bootstrap-3.0.2/css/bootstrap.min.css" />
-<link rel="stylesheet" type="text/css" href="/static/dataTables.bootstrap.css" />
-<link rel="stylesheet" type="text/css" href="/static/hadoop.css" />
-<title>Router Information</title>
-</head>
-<body>
-
-<header class="navbar navbar-inverse bs-docs-nav" role="banner">
-<div class="container">
- <div class="navbar-header">
- <div class="navbar-brand">Hadoop</div>
- </div>
-
- <ul class="nav navbar-nav" id="ui-tabs">
- <li><a href="#tab-overview">Overview</a></li>
- <li><a href="#tab-namenode">Subclusters</a></li>
- <li><a href="#tab-router">Routers</a></li>
- <li><a href="#tab-datanode">Datanodes</a></li>
- <li><a href="#tab-mounttable">Mount table</a></li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown">Utilities <b class="caret"></b></a>
- <ul class="dropdown-menu">
- <li><a href="jmx">Metrics</a></li>
- <li><a href="conf">Configuration</a></li>
- <li><a href="logs">Logs</a></li>
- <li><a href="fsck">FSCK</a></li>
- </ul>
- </li>
- </ul>
-</div>
-</header>
-
-<div class="container">
-
-<div id="alert-panel">
- <div class="alert alert-danger">
- <button type="button" class="close" onclick="$('#alert-panel').hide();">×</button>
- <div class="alert-body" id="alert-panel-body"></div>
- </div>
-</div>
-
-<div class="tab-content">
- <div class="tab-pane" id="tab-overview"></div>
- <div class="tab-pane" id="tab-namenode"></div>
- <div class="tab-pane" id="tab-router"></div>
- <div class="tab-pane" id="tab-datanode"></div>
- <div class="tab-pane" id="tab-mounttable"></div>
-</div>
-
-<div class="row">
- <hr />
- <div class="col-xs-2"><p>Hadoop, {release-year-token}.</p></div>
-</div>
-</div>
-
-<!-- Overview -->
-<script type="text/x-dust-template" id="tmpl-federationhealth">
-<div class="page-header"><h1>Router {#federation}<small>'{HostAndPort}'</small>{/federation}</h1></div>
-{#federation}
-<table class="table table-bordered table-striped">
- <tr><th>Started:</th><td>{RouterStarted}</td></tr>
- <tr><th>Version:</th><td>{Version}</td></tr>
- <tr><th>Compiled:</th><td>{CompileInfo}</td></tr>
- <tr><th>Cluster ID:</th><td>{ClusterId}</td></tr>
- <tr><th>Block Pool ID:</th><td>{BlockPoolId}</td></tr>
- <tr><th>Status:</th><td>{RouterStatus}</td></tr>
-</table>
-{/federation}
-
-<div class="page-header"><h1>Summary</h1></div>
-{#federation}
-<table class="table table-bordered table-striped">
- <tr><th>Total capacity</th><td>{TotalCapacity|fmt_bytes}</td></tr>
- <tr><th>Used capacity</th><td>{UsedCapacity|fmt_bytes}</td></tr>
- <tr><th>Remaining capacity</th><td>{RemainingCapacity|fmt_bytes}</td></tr>
- <tr><th>Nameservices</th><td>{NumNameservices}</td></tr>
- <tr><th>Namenodes</th><td>{NumNamenodes}</td></tr>
- <tr>
- <th>DataNodes usages% (Min/Median/Max/stdDev)</th>
- <td>{#NodeUsage.nodeUsage}{min} / {median} / {max} / {stdDev}{/NodeUsage.nodeUsage}</td>
- </tr>
- <tr><th><a href="#tab-datanode">Live Nodes</a></th><td>{NumLiveNodes} (Decommissioned: {NumDecomLiveNodes})</td></tr>
- <tr><th><a href="#tab-datanode">Dead Nodes</a></th><td>{NumDeadNodes} (Decommissioned: {NumDecomDeadNodes})</td></tr>
- <tr><th><a href="#tab-datanode">Decommissioning Nodes</a></th><td>{NumDecommissioningNodes}</td></tr>
- <tr><th>Files</th><td>{NumFiles}</td></tr>
- <tr><th>Blocks</th><td>{NumBlocks}</td></tr>
- <tr><th title="Excludes missing blocks.">Number of Under-Replicated Blocks</th><td>{NumOfBlocksUnderReplicated}</td></tr>
- <tr><th>Number of Blocks Pending Deletion</th><td>{NumOfBlocksPendingDeletion}</td></tr>
-</table>
-{/federation}
-</script>
-
-<!-- Subclusters info: list of nameservices and namenodes -->
-<script type="text/x-dust-template" id="tmpl-namenode">
-<div class="page-header"><h1>Nameservice Information</h1></div>
-<div>
- <ul class="dfshealth-node-legend">
- <li class="dfshealth-node-icon dfshealth-node-alive">Active</li>
- <li class="dfshealth-node-icon dfshealth-node-down-decommissioned">Standby</li>
- <li class="dfshealth-node-icon dfshealth-node-down-maintenance">Safe mode</li>
- <li class="dfshealth-node-icon dfshealth-node-down">Unavailable</li>
- </ul>
-</div>
-<small>
-<table class="table">
- <thead>
- <tr>
- <th colspan="6"></th>
- <th colspan="3">Blocks</th>
- <th colspan="2">Nodes</th>
- <th colspan="3">Decom</th>
- </tr>
- <tr>
- <th></th>
- <th>Nameservice</th>
- <th>Namenode</th>
- <th>Last Contact</th>
- <th>Capacity</th>
- <th>Files</th>
- <th>Total</th>
- <th>Missing</th>
- <th>Under-Replicated</th>
- <th>Live</th>
- <th>Dead</th>
- <th>Progress</th>
- <th>Live</th>
- <th>Dead</th>
- </tr>
- </thead>
- <tbody>
- {#Nameservices}
- <tr>
- <td class="dfshealth-node-icon dfshealth-node-{iconState}" title="{title}"></td>
- <td><a href="http://{webAddress}">{nameserviceId}</a></td>
- <td><a href="http://{webAddress}">{namenodeId}</a></td>
- <td>{lastHeartbeat}</td>
- <td ng-value="{usedPercentage}" style="width:210px">
- <div>
- <div style="display:inline-block; float: left; padding-right: 10px; width:60px;">{totalSpace|fmt_bytes}</div>
- <div class="clearfix progress dfshealth-node-capacity-bar" title="Used: {used|fmt_bytes}">
- <div class="progress-bar {#helper_usage_bar value="{usedPercentage}"/}" style="width: {usedPercentage}%">
- </div>
- </div>
- </div>
- </td>
- <td>{numOfFiles}</td>
- <td>{numOfBlocks}</td>
- <td>{numOfBlocksMissing}</td>
- <td>{numOfBlocksUnderReplicated}</td>
- <td>{numOfActiveDatanodes}</td>
- <td>{numOfDeadDatanodes}</td>
- <td>{numOfDecommissioningDatanodes}</td>
- <td>{numOfDecomActiveDatanodes}</td>
- <td>{numOfDecomDeadDatanodes}</td>
- </tr>
- {/Nameservices}
- </tbody>
-</table>
-</small>
-
-<div class="page-header"><h1>Namenode Information</h1></div>
-<div>
- <ul class="dfshealth-node-legend">
- <li class="dfshealth-node-icon dfshealth-node-alive">Active</li>
- <li class="dfshealth-node-icon dfshealth-node-down-decommissioned">Standby</li>
- <li class="dfshealth-node-icon dfshealth-node-down-maintenance">Safe mode</li>
- <li class="dfshealth-node-icon dfshealth-node-down">Unavailable</li>
- </ul>
-</div>
-<small>
-<table class="table">
- <thead>
- <tr>
- <th colspan="7"></th>
- <th colspan="3">Blocks</th>
- <th colspan="2">Nodes</th>
- <th colspan="3">Decom</th>
- </tr>
- <tr>
- <th></th>
- <th colspan="2">Namenode</th>
- <th>Web address</th>
- <th>Last Contact</th>
- <th>Capacity</th>
- <th>Files</th>
- <th>Total</th>
- <th>Missing</th>
- <th>Under-Replicated</th>
- <th>Live</th>
- <th>Dead</th>
- <th>Progress</th>
- <th>Live</th>
- <th>Dead</th>
- </tr>
- </thead>
- <tbody>
- {#Namenodes}
- <tr>
- <td class="dfshealth-node-icon dfshealth-node-{iconState}" title="{title}"></td>
- <td>{nameserviceId}</td>
- <td>{namenodeId}</td>
- <td><a href="http://{webAddress}">{webAddress}</a></td>
- <td>{lastHeartbeat}</td>
- <td ng-value="{usedPercentage}" style="width:210px">
- <div>
- <div style="display:inline-block; float: left; padding-right: 10px; width:60px;">{totalSpace|fmt_bytes}</div>
- <div class="clearfix progress dfshealth-node-capacity-bar" title="Used: {used|fmt_bytes}">
- <div class="progress-bar {#helper_usage_bar value="{usedPercentage}"/}" style="width: {usedPercentage}%">
- </div>
- </div>
- </div>
- </td>
- <td>{numOfFiles}</td>
- <td>{numOfBlocks}</td>
- <td>{numOfBlocksMissing}</td>
- <td>{numOfBlocksUnderReplicated}</td>
- <td>{numOfActiveDatanodes}</td>
- <td>{numOfDeadDatanodes}</td>
- <td>{numOfDecommissioningDatanodes}</td>
- <td>{numOfDecomActiveDatanodes}</td>
- <td>{numOfDecomDeadDatanodes}</td>
- </tr>
- {/Namenodes}
- </tbody>
-</table>
-</small>
-</script>
-
-<!-- Routers -->
-<script type="text/x-dust-template" id="tmpl-router">
-<div class="page-header"><h1>Routers Information</h1></div>
-<div>
- <ul class="dfshealth-node-legend">
- <li class="dfshealth-node-icon dfshealth-node-alive">Active</li>
- <li class="dfshealth-node-icon dfshealth-node-decommissioned">Safe mode</li>
- <li class="dfshealth-node-icon dfshealth-node-down">Unavailable</li>
- </ul>
-</div>
-<table class="table">
- <thead>
- <tr>
- <th colspan="4"></th>
- <th colspan="2">Last update</th>
- <th></th>
- </tr>
- <tr>
- <th></th>
- <th>Address</th>
- <th>Status</th>
- <th>Heartbeat</th>
- <th>Membership</th>
- <th>Mount table</th>
- <th>Version</th>
- </tr>
- </thead>
- <tbody>
- {#Routers}
- <tr>
- <td class="dfshealth-node-icon dfshealth-node-{iconState}" title="{title}"></td>
- <td>{address}</td>
- <td>{status}</td>
- <td>{lastHeartbeat} sec ago</td>
- <td>{lastMembershipUpdate}</td>
- <td>{lastMountTableUpdate}</td>
- <td>{version}</td>
- </tr>
- {/Routers}
- </tbody>
-</table>
-</script>
-
-<!-- Datanodes -->
-<script type="text/x-dust-template" id="tmpl-datanode">
-<div class="page-header"><h1>Datanode Information</h1></div>
-<div>
- <ul class="dfshealth-node-legend">
- <li class="dfshealth-node-icon dfshealth-node-alive">In service</li>
- <li class="dfshealth-node-icon dfshealth-node-down">Down</li>
- <li class="dfshealth-node-icon dfshealth-node-decommissioned">Decommissioned</li>
- <li class="dfshealth-node-icon dfshealth-node-down-decommissioned">Decommissioned & dead</li>
- </ul>
-</div>
-<div class="page-header"><h1><small>In operation</small></h1></div>
-<small>
-<table class="table" id="table-datanodes">
- <thead>
- <tr>
- <th>Node</th>
- <th>Last contact</th>
- <th style="width:180px; text-align:center">Capacity</th>
- <!--th>Blocks</th-->
- <th>Block pool used</th>
- <!--th>Version</th-->
- </tr>
- </thead>
- {#LiveNodes}
- <tr>
- <td ng-value="{state}-{name}" class="dfshealth-node-icon dfshealth-node-{state}">{location}/{name} ({xferaddr})</td>
- <td ng-value="{lastContact}">{#helper_relative_time value="{lastContact}"/}</td>
- <td ng-value="{usedPercentage}" style="width:210px">
- <div>
- <div style="display:inline-block; float: left; padding-right: 10px; width:60px;">{capacity|fmt_bytes}</div>
- <div class="clearfix progress dfshealth-node-capacity-bar" title="Non DFS: {nonDfsUsedSpace|fmt_bytes}, Used: {used|fmt_bytes}">
- <div class="progress-bar {#helper_usage_bar value="{usedPercentage}"/}" style="width: {usedPercentage}%">
- </div>
- </div>
- </div>
- </td>
- <!--td>{numBlocks}</td-->
- <td ng-value="{blockPoolUsedPercent}">{blockPoolUsed|fmt_bytes} ({blockPoolUsedPercent|fmt_percentage})</td>
- <!--td>{version}</td-->
- </tr>
- {/LiveNodes}
- {#DeadNodes}
- <tr class="danger">
- <td ng-value="{state}-{name}" class="dfshealth-node-icon dfshealth-node-{state}">{location}/{name} ({xferaddr})</td>
- <td>{#helper_relative_time value="{lastContact}"/}</td>
- <td></td>
- <!--td></td-->
- <td></td>
- <!--td></td-->
- </tr>
- {/DeadNodes}
-</table>
-</small>
-
-<div class="page-header"><h1><small>Decommissioning</small></h1></div>
-<small>
-<table class="table">
- <thead>
- <tr>
- <th>Node</th>
- <th>Under replicated blocks</th>
- <th>Blocks with no live replicas</th>
- <th>Under Replicated Blocks <br/>In files under construction</th>
- </tr>
- </thead>
- {#DecomNodes}
- <tr>
- <td>{location}/{name} ({xferaddr})</td>
- <td>{underReplicatedBlocks}</td>
- <td>{decommissionOnlyReplicas}</td>
- <td>{underReplicateInOpenFiles}</td>
- </tr>
- {/DecomNodes}
-</table>
-</small>
-</script>
-
-<!-- Mount table -->
-<script type="text/x-dust-template" id="tmpl-mounttable">
-<div class="page-header"><h1>Mount Table</h1></div>
-<small>
-<table class="table">
- <thead>
- <tr>
- <th>Global path</th>
- <th>Target nameservice</th>
- <th>Target path</th>
- <th>Order</th>
- <th>Read only</th>
- <th>Owner</th>
- <th>Group</th>
- <th>Permission</th>
- <th>Quota/Usage</th>
- <th>Date Modified</th>
- <th>Date Created</th>
- </tr>
- </thead>
- <tbody>
- {#MountTable}
- <tr>
- <td>{sourcePath}</td>
- <td>{nameserviceId}</td>
- <td>{path}</td>
- <td>{order}</td>
- <td class="dfshealth-node-icon dfshealth-mount-read-only-{readonly}"/>
- <td>{ownerName}</td>
- <td>{groupName}</td>
- <td>{mode}</td>
- <td>{quota}</td>
- <td>{dateModified}</td>
- <td>{dateCreated}</td>
- </tr>
- {/MountTable}
- </tbody>
-</table>
-</small>
-</script>
-
-
-
-<script type="text/javascript" src="/static/jquery-1.10.2.min.js"></script>
-<script type="text/javascript" src="/static/jquery.dataTables.min.js"></script>
-<script type="text/javascript" src="/static/bootstrap-3.0.2/js/bootstrap.min.js"></script>
-<script type="text/javascript" src="/static/dataTables.bootstrap.js"></script>
-<script type="text/javascript" src="/static/moment.min.js"></script>
-<script type="text/javascript" src="/static/dust-full-2.0.0.min.js"></script>
-<script type="text/javascript" src="/static/dust-helpers-1.1.1.min.js"></script>
-<script type="text/javascript" src="/static/dfs-dust.js"></script>
-<script type="text/javascript" src="federationhealth.js"></script>
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/federationhealth.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/federationhealth.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/federationhealth.js
deleted file mode 100644
index b3ed868..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/federationhealth.js
+++ /dev/null
@@ -1,381 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-(function () {
- "use strict";
-
- dust.loadSource(dust.compile($('#tmpl-federationhealth').html(), 'federationhealth'));
- dust.loadSource(dust.compile($('#tmpl-namenode').html(), 'namenode-info'));
- dust.loadSource(dust.compile($('#tmpl-router').html(), 'router-info'));
- dust.loadSource(dust.compile($('#tmpl-datanode').html(), 'datanode-info'));
- dust.loadSource(dust.compile($('#tmpl-mounttable').html(), 'mounttable'));
-
- $.fn.dataTable.ext.order['ng-value'] = function (settings, col)
- {
- return this.api().column(col, {order:'index'} ).nodes().map(function (td, i) {
- return $(td).attr('ng-value');
- });
- };
-
- function load_overview() {
- var BEANS = [
- {"name": "federation", "url": "/jmx?qry=Hadoop:service=Router,name=FederationState"}
- ];
-
- var HELPERS = {
- 'helper_fs_max_objects': function (chunk, ctx, bodies, params) {
- var o = ctx.current();
- if (o.MaxObjects > 0) {
- chunk.write('(' + Math.round((o.FilesTotal + o.BlockTotal) / o.MaxObjects * 100) * 100 + ')%');
- }
- },
-
- 'helper_dir_status': function (chunk, ctx, bodies, params) {
- var j = ctx.current();
- for (var i in j) {
- chunk.write('<tr><td>' + i + '</td><td>' + j[i] + '</td><td>' + params.type + '</td></tr>');
- }
- },
-
- 'helper_date_tostring' : function (chunk, ctx, bodies, params) {
- var value = dust.helpers.tap(params.value, chunk, ctx);
- return chunk.write('' + new Date(Number(value)).toLocaleString());
- }
- };
-
- var data = {};
-
- // Workarounds for the fact that JMXJsonServlet returns non-standard JSON strings
- function workaround(nn) {
- nn.NodeUsage = JSON.parse(nn.NodeUsage);
- return nn;
- }
-
- load_json(
- BEANS,
- guard_with_startup_progress(function(d) {
- for (var k in d) {
- data[k] = k === 'federation' ? workaround(d[k].beans[0]) : d[k].beans[0];
- }
- render();
- }),
- function (url, jqxhr, text, err) {
- show_err_msg('<p>Failed to retrieve data from ' + url + ', cause: ' + err + '</p>');
- });
-
- function render() {
- var base = dust.makeBase(HELPERS);
- dust.render('federationhealth', base.push(data), function(err, out) {
- $('#tab-overview').html(out);
- $('#ui-tabs a[href="#tab-overview"]').tab('show');
- });
- }
- }
-
- function load_namenode_info() {
- var HELPERS = {
- 'helper_lastcontact_tostring' : function (chunk, ctx, bodies, params) {
- var value = dust.helpers.tap(params.value, chunk, ctx);
- return chunk.write('' + new Date(Date.now()-1000*Number(value)));
- }
- };
-
- function workaround(r) {
- function node_map_to_array(nodes) {
- var res = [];
- for (var n in nodes) {
- var p = nodes[n];
- p.name = n;
- res.push(p);
- }
- return res;
- }
-
- function capitalise(string) {
- return string.charAt(0).toUpperCase() + string.slice(1).toLowerCase();
- }
-
- function augment_namenodes(nodes) {
- for (var i = 0, e = nodes.length; i < e; ++i) {
- var n = nodes[i];
- n.usedPercentage = Math.round(n.used * 1.0 / n.totalSpace * 100);
- n.title = "Unavailable";
- n.iconState = "down";
- if (n.isSafeMode === true) {
- n.title = capitalise(n.state) + " (safe mode)"
- n.iconState = "decommissioned";
- } else if (n.state === "ACTIVE") {
- n.title = capitalise(n.state);
- n.iconState = "alive";
- } else if (nodes[i].state === "STANDBY") {
- n.title = capitalise(n.state);
- n.iconState = "down-decommissioned";
- } else if (nodes[i].state === "UNAVAILABLE") {
- n.title = capitalise(n.state);
- n.iconState = "down";
- }
- if (n.namenodeId === "null") {
- n.namenodeId = "";
- }
- }
- }
-
- r.Nameservices = node_map_to_array(JSON.parse(r.Nameservices));
- augment_namenodes(r.Nameservices);
- r.Namenodes = node_map_to_array(JSON.parse(r.Namenodes));
- augment_namenodes(r.Namenodes);
- return r;
- }
-
- $.get(
- '/jmx?qry=Hadoop:service=Router,name=FederationState',
- guard_with_startup_progress(function (resp) {
- var data = workaround(resp.beans[0]);
- var base = dust.makeBase(HELPERS);
- dust.render('namenode-info', base.push(data), function(err, out) {
- $('#tab-namenode').html(out);
- $('#ui-tabs a[href="#tab-namenode"]').tab('show');
- });
- })).error(ajax_error_handler);
- }
-
- function load_router_info() {
- var HELPERS = {
- 'helper_lastcontact_tostring' : function (chunk, ctx, bodies, params) {
- var value = dust.helpers.tap(params.value, chunk, ctx);
- return chunk.write('' + new Date(Date.now()-1000*Number(value)));
- }
- };
-
- function workaround(r) {
- function node_map_to_array(nodes) {
- var res = [];
- for (var n in nodes) {
- var p = nodes[n];
- p.name = n;
- res.push(p);
- }
- return res;
- }
-
- function capitalise(string) {
- return string.charAt(0).toUpperCase() + string.slice(1).toLowerCase();
- }
-
- function augment_routers(nodes) {
- for (var i = 0, e = nodes.length; i < e; ++i) {
- var n = nodes[i];
- n.title = "Unavailable"
- n.iconState = "down";
- if (n.status === "INITIALIZING") {
- n.title = capitalise(n.status);
- n.iconState = "alive";
- } else if (n.status === "RUNNING") {
- n.title = capitalise(n.status);
- n.iconState = "alive";
- } else if (n.status === "SAFEMODE") {
- n.title = capitalise(n.status);
- n.iconState = "down-decommissioned";
- } else if (n.status === "STOPPING") {
- n.title = capitalise(n.status);
- n.iconState = "decommissioned";
- } else if (n.status === "SHUTDOWN") {
- n.title = capitalise(n.status);
- n.iconState = "down";
- }
- }
- }
-
- r.Routers = node_map_to_array(JSON.parse(r.Routers));
- augment_routers(r.Routers);
- return r;
- }
-
- $.get(
- '/jmx?qry=Hadoop:service=Router,name=FederationState',
- guard_with_startup_progress(function (resp) {
- var data = workaround(resp.beans[0]);
- var base = dust.makeBase(HELPERS);
- dust.render('router-info', base.push(data), function(err, out) {
- $('#tab-router').html(out);
- $('#ui-tabs a[href="#tab-router"]').tab('show');
- });
- })).error(ajax_error_handler);
- }
-
- // TODO Copied directly from dfshealth.js; is there a way to import this function?
- function load_datanode_info() {
-
- var HELPERS = {
- 'helper_relative_time' : function (chunk, ctx, bodies, params) {
- var value = dust.helpers.tap(params.value, chunk, ctx);
- return chunk.write(moment().subtract(Number(value), 'seconds').format('YYYY-MM-DD HH:mm:ss'));
- },
- 'helper_usage_bar' : function (chunk, ctx, bodies, params) {
- var value = dust.helpers.tap(params.value, chunk, ctx);
- var v = Number(value);
- var r = null;
- if (v < 70) {
- r = 'progress-bar-success';
- } else if (v < 85) {
- r = 'progress-bar-warning';
- } else {
- r = "progress-bar-danger";
- }
- return chunk.write(r);
- },
- };
-
- function workaround(r) {
- function node_map_to_array(nodes) {
- var res = [];
- for (var n in nodes) {
- var p = nodes[n];
- p.name = n;
- res.push(p);
- }
- return res;
- }
-
- function augment_live_nodes(nodes) {
- for (var i = 0, e = nodes.length; i < e; ++i) {
- var n = nodes[i];
- n.usedPercentage = Math.round((n.used + n.nonDfsUsedSpace) * 1.0 / n.capacity * 100);
- if (n.adminState === "In Service") {
- n.state = "alive";
- } else if (nodes[i].adminState === "Decommission In Progress") {
- n.state = "decommissioning";
- } else if (nodes[i].adminState === "Decommissioned") {
- n.state = "decommissioned";
- }
- }
- }
-
- function augment_dead_nodes(nodes) {
- for (var i = 0, e = nodes.length; i < e; ++i) {
- if (nodes[i].decommissioned) {
- nodes[i].state = "down-decommissioned";
- } else {
- nodes[i].state = "down";
- }
- }
- }
-
- r.LiveNodes = node_map_to_array(JSON.parse(r.LiveNodes));
- augment_live_nodes(r.LiveNodes);
- r.DeadNodes = node_map_to_array(JSON.parse(r.DeadNodes));
- augment_dead_nodes(r.DeadNodes);
- r.DecomNodes = node_map_to_array(JSON.parse(r.DecomNodes));
- return r;
- }
-
- $.get(
- '/jmx?qry=Hadoop:service=NameNode,name=NameNodeInfo',
- guard_with_startup_progress(function (resp) {
- var data = workaround(resp.beans[0]);
- var base = dust.makeBase(HELPERS);
- dust.render('datanode-info', base.push(data), function(err, out) {
- $('#tab-datanode').html(out);
- $('#table-datanodes').dataTable( {
- 'lengthMenu': [ [25, 50, 100, -1], [25, 50, 100, "All"] ],
- 'columns': [
- { 'orderDataType': 'ng-value', 'searchable': true },
- { 'orderDataType': 'ng-value', 'type': 'numeric' },
- { 'orderDataType': 'ng-value', 'type': 'numeric' },
- { 'orderDataType': 'ng-value', 'type': 'numeric'}
- ]});
- $('#ui-tabs a[href="#tab-datanode"]').tab('show');
- });
- })).error(ajax_error_handler);
- }
-
- function load_mount_table() {
- var HELPERS = {}
-
- function workaround(resource) {
- resource.MountTable = JSON.parse(resource.MountTable)
- return resource;
- }
-
- $.get(
- '/jmx?qry=Hadoop:service=Router,name=FederationState',
- guard_with_startup_progress(function (resp) {
- var data = workaround(resp.beans[0]);
- var base = dust.makeBase(HELPERS);
- dust.render('mounttable', base.push(data), function(err, out) {
- $('#tab-mounttable').html(out);
- $('#ui-tabs a[href="#tab-mounttable"]').tab('show');
- });
- })).error(ajax_error_handler);
- }
-
- function toTitleCase(str) {
- return str.replace(/\w\S*/g, function(txt){
- return txt.charAt(0).toUpperCase() + txt.substr(1).toLowerCase();
- });
- }
-
- function show_err_msg(msg) {
- $('#alert-panel-body').html(msg);
- $('#alert-panel').show();
- }
-
- function ajax_error_handler(url, jqxhr, text, err) {
- show_err_msg('<p>Failed to retrieve data from ' + url + ', cause: ' + err + '</p>');
- }
-
- function guard_with_startup_progress(fn) {
- return function() {
- try {
- fn.apply(this, arguments);
- } catch (err) {
- if (err instanceof TypeError) {
- show_err_msg('Router error: ' + err);
- }
- }
- };
- }
-
- function load_page() {
- var hash = window.location.hash;
- switch(hash) {
- case "#tab-overview":
- load_overview();
- break;
- case "#tab-namenode":
- load_namenode_info();
- break;
- case "#tab-router":
- load_router_info();
- break;
- case "#tab-datanode":
- load_datanode_info();
- break;
- case "#tab-mounttable":
- load_mount_table();
- break;
- default:
- window.location.hash = "tab-overview";
- break;
- }
- }
- load_page();
-
- $(window).bind('hashchange', function () {
- load_page();
- });
-})();
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/index.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/index.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/index.html
deleted file mode 100644
index 4ee04ac..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/index.html
+++ /dev/null
@@ -1,24 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml">
-<head>
-<meta http-equiv="REFRESH" content="0;url=federationhealth.html" />
-<title>Hadoop Administration</title>
-</head>
-</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/robots.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/robots.txt b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/robots.txt
deleted file mode 100644
index 77470cb..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/robots.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-User-agent: *
-Disallow: /
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md
deleted file mode 100644
index fdaaa11..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md
+++ /dev/null
@@ -1,373 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-HDFS Router-based Federation
-============================
-
-<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
-
-Introduction
-------------
-
-NameNodes have scalability limits because of the metadata overhead comprised of inodes (files and directories) and file blocks, the number of Datanode heartbeats, and the number of HDFS RPC client requests.
-The common solution is to split the filesystem into smaller subclusters [HDFS Federation](./Federation.html) and provide a federated view [ViewFs](./ViewFs.html).
-The problem is how to maintain the split of the subclusters (e.g., namespace partition), which forces users to connect to multiple subclusters and manage the allocation of folders/files to them.
-
-
-Architecture
-------------
-
-A natural extension to this partitioned federation is to add a layer of software responsible for federating the namespaces.
-This extra layer allows users to access any subcluster transparently, lets subclusters manage their own block pools independently, and will support rebalancing of data across subclusters later
-(see more info in [HDFS-13123](https://issues.apache.org/jira/browse/HDFS-13123)). The subclusters in RBF are not required to be the independent HDFS clusters, a normal federation cluster
-(with multiple block pools) or a mixed cluster with federation and independent cluster is also allowed.
-To accomplish these goals, the federation layer directs block accesses to the proper subcluster, maintains the state of the namespaces, and provides mechanisms for data rebalancing.
-This layer must be scalable, highly available, and fault tolerant.
-
-This federation layer comprises multiple components.
-The _Router_ component that has the same interface as a NameNode, and forwards the client requests to the correct subcluster, based on ground-truth information from a State Store.
-The _State Store_ combines a remote _Mount Table_ (in the flavor of [ViewFs](./ViewFs.html), but shared between clients) and utilization (load/capacity) information about the subclusters.
-This approach has the same architecture as [YARN federation](../hadoop-yarn/Federation.html).
-
-
-
-
-### Example flow
-The simplest configuration deploys a Router on each NameNode machine.
-The Router monitors the local NameNode and heartbeats the state to the State Store.
-When a regular DFS client contacts any of the Routers to access a file in the federated filesystem, the Router checks the Mount Table in the State Store (i.e., the local cache) to find out which subcluster contains the file.
-Then it checks the Membership table in the State Store (i.e., the local cache) for the NameNode responsible for the subcluster.
-After it has identified the correct NameNode, the Router proxies the request.
-The client accesses Datanodes directly.
-
-
-### Router
-There can be multiple Routers in the system with soft state.
-Each Router has two roles:
-
-* Federated interface: expose a single, global NameNode interface to the clients and forward the requests to the active NameNode in the correct subcluster
-* NameNode heartbeat: maintain the information about a NameNode in the State Store
-
-#### Federated interface
-The Router receives a client request, checks the State Store for the correct subcluster, and forwards the request to the active NameNode of that subcluster.
-The reply from the NameNode then flows in the opposite direction.
-The Routers are stateless and can be behind a load balancer.
-For performance, the Router also caches remote mount table entries and the state of the subclusters.
-To make sure that changes have been propagated to all Routers, each Router heartbeats its state to the State Store.
-
-The communications between the Routers and the State Store are cached (with timed expiration for freshness).
-This improves the performance of the system.
-
-#### NameNode heartbeat
-For this role, the Router periodically checks the state of a NameNode (usually on the same server) and reports their high availability (HA) state and load/space status to the State Store.
-Note that this is an optional role, as a Router can be independent of any subcluster.
-For performance with NameNode HA, the Router uses the high availability state information in the State Store to forward the request to the NameNode that is most likely to be active.
-Note that this service can be embedded into the NameNode itself to simplify the operation.
-
-#### Availability and fault tolerance
-The Router operates with failures at multiple levels.
-
-* **Federated interface HA:**
-The Routers are stateless and metadata operations are atomic at the NameNodes.
-If a Router becomes unavailable, any Router can take over for it.
-The clients configure their DFS HA client (e.g., ConfiguredFailoverProvider or RequestHedgingProxyProvider) with all the Routers in the federation as endpoints.
-
-* **Unavailable State Store:**
-If a Router cannot contact the State Store, it will enter into a Safe Mode state which disallows it from serving requests.
-Clients will treat Routers in Safe Mode as it was an Standby NameNode and try another Router. There is a manual way to manage the Safe Mode for the Router.
-
-The Safe Mode state can be managed by using the following command:
-
- [hdfs]$ $HADOOP_HOME/bin/hdfs dfsrouteradmin -safemode enter | leave | get
-
-* **NameNode heartbeat HA:**
-For high availability and flexibility, multiple Routers can monitor the same NameNode and heartbeat the information to the State Store.
-This increases clients' resiliency to stale information, should a Router fail.
-Conflicting NameNode information in the State Store is resolved by each Router via a quorum.
-
-* **Unavailable NameNodes:**
-If a Router cannot contact the active NameNode, then it will try the other NameNodes in the subcluster.
-It will first try those reported as standby and then the unavailable ones.
-If the Router cannot reach any NameNode, then it throws an exception.
-
-* **Expired NameNodes:**
-If a NameNode heartbeat has not been recorded in the State Store for a multiple of the heartbeat interval, the monitoring Router will record that the NameNode has expired and no Routers will attempt to access it.
-If an updated heartbeat is subsequently recorded for the NameNode, the monitoring Router will restore the NameNode from the expired state.
-
-#### Interfaces
-To interact with the users and the administrators, the Router exposes multiple interfaces.
-
-* **RPC:**
-The Router RPC implements the most common interfaces clients use to interact with HDFS.
-The current implementation has been tested using analytics workloads written in plain MapReduce, Spark, and Hive (on Tez, Spark, and MapReduce).
-Advanced functions like snapshot, encryption and tiered storage are left for future versions.
-All unimplemented functions will throw exceptions.
-
-* **Admin:**
-Administrators can query information from clusters and add/remove entries from the mount table over RPC.
-This interface is also exposed through the command line to get and modify information from the federation.
-
-* **Web UI:**
-The Router exposes a Web UI visualizing the state of the federation, mimicking the current NameNode UI.
-It displays information about the mount table, membership information about each subcluster, and the status of the Routers.
-
-* **WebHDFS:**
-The Router provides the HDFS REST interface (WebHDFS) in addition to the RPC one.
-
-* **JMX:**
-It exposes metrics through JMX mimicking the NameNode.
-This is used by the Web UI to get the cluster status.
-
-Some operations are not available in Router-based federation.
-The Router throws exceptions for those.
-Examples users may encounter include the following.
-
-* Rename file/folder in two different nameservices.
-* Copy file/folder in two different nameservices.
-* Write into a file/folder being rebalanced.
-
-### Quota management
-Federation supports and controls global quota at mount table level.
-For performance reasons, the Router caches the quota usage and updates it periodically. These quota usage values
-will be used for quota-verification during each WRITE RPC call invoked in RouterRPCSever. See [HDFS Quotas Guide](./HdfsQuotaAdminGuide.html)
-for the quota detail.
-
-### State Store
-The (logically centralized, but physically distributed) State Store maintains:
-
-* The state of the subclusters in terms of their block access load, available disk space, HA state, etc.
-* The mapping between folder/files and subclusters, i.e. the remote mount table.
-
-The backend of the State Store is pluggable.
-We leverage the fault tolerance of the backend implementations.
-The main information stored in the State Store and its implementation:
-
-* **Membership**:
-The membership information encodes the state of the NameNodes in the federation.
-This includes information about the subcluster, such as storage capacity and the number of nodes.
-The Router periodically heartbeats this information about one or more NameNodes.
-Given that multiple Routers can monitor a single NameNode, the heartbeat from every Router is stored.
-The Routers apply a quorum of the data when querying this information from the State Store.
-The Routers discard the entries older than a certain threshold (e.g., ten Router heartbeat periods).
-
-* **Mount Table**:
-This table hosts the mapping between folders and subclusters.
-It is similar to the mount table in [ViewFs](.ViewFs.html) where it specifies the federated folder, the destination subcluster and the path in that folder.
-
-
-### Security
-Secure authentication and authorization are not supported yet, so the Router will not proxy to Hadoop clusters with security enabled.
-
-
-Deployment
-----------
-
-By default, the Router is ready to take requests and monitor the NameNode in the local machine.
-It needs to know the State Store endpoint by setting `dfs.federation.router.store.driver.class`.
-The rest of the options are documented in [hdfs-default.xml](./hdfs-default.xml).
-
-Once the Router is configured, it can be started:
-
- [hdfs]$ $HADOOP_PREFIX/sbin/hadoop-daemon.sh --script $HADOOP_PREFIX/bin/hdfs start dfsrouter
-
-And to stop it:
-
- [hdfs]$ $HADOOP_PREFIX/sbin/hadoop-daemon.sh --script $HADOOP_PREFIX/bin/hdfs stop dfsrouter
-
-### Mount table management
-
-The mount table entries are pretty much the same as in [ViewFs](./ViewFs.html).
-A good practice for simplifying the management is to name the federated namespace with the same names as the destination namespaces.
-For example, if we to mount `/data/app1` in the federated namespace, it is recommended to have that same name as in the destination namespace.
-
-The federation admin tool supports managing the mount table.
-For example, to create three mount points and list them:
-
- [hdfs]$ $HADOOP_HOME/bin/hdfs dfsrouteradmin -add /tmp ns1 /tmp
- [hdfs]$ $HADOOP_HOME/bin/hdfs dfsrouteradmin -add /data/app1 ns2 /data/app1
- [hdfs]$ $HADOOP_HOME/bin/hdfs dfsrouteradmin -add /data/app2 ns3 /data/app2
- [hdfs]$ $HADOOP_HOME/bin/hdfs dfsrouteradmin -ls
-
-It also supports mount points that disallow writes:
-
- [hdfs]$ $HADOOP_HOME/bin/hdfs dfsrouteradmin -add /readonly ns1 / -readonly
-
-If a mount point is not set, the Router will map it to the default namespace `dfs.federation.router.default.nameserviceId`.
-
-Mount table have UNIX-like *permissions*, which restrict which users and groups have access to the mount point. Write permissions allow users to add
-, update or remove mount point. Read permissions allow users to list mount point. Execute permissions are unused.
-
-Mount table permission can be set by following command:
-
- [hdfs]$ $HADOOP_HOME/bin/hdfs dfsrouteradmin -add /tmp ns1 /tmp -owner root -group supergroup -mode 0755
-
-The option mode is UNIX-style permissions for the mount table. Permissions are specified in octal, e.g. 0755. By default, this is set to 0755.
-
-Router-based federation supports global quota at mount table level. Mount table entries may spread multiple subclusters and the global quota will be
-accounted across these subclusters.
-
-The federation admin tool supports setting quotas for specified mount table entries:
-
- [hdfs]$ $HADOOP_HOME/bin/hdfs dfsrouteradmin -setQuota /path -nsQuota 100 -ssQuota 1024
-
-The above command means that we allow the path to have a maximum of 100 file/directories and use at most 1024 bytes storage space. The parameter for *ssQuota*
-supports multiple size-unit suffix (e.g. 1k is 1KB, 5m is 5MB). If no suffix is specified then bytes is assumed.
-
-Ls command will show below information for each mount table entry:
-
- Source Destinations Owner Group Mode Quota/Usage
- /path ns0->/path root supergroup rwxr-xr-x [NsQuota: 50/0, SsQuota: 100 B/0 B]
-
-Client configuration
---------------------
-
-For clients to use the federated namespace, they need to create a new one that points to the routers.
-For example, a cluster with 4 namespaces **ns0, ns1, ns2, ns3**, can add a new one to **hdfs-site.xml** called **ns-fed** which points to two of the routers:
-
-```xml
-<configuration>
- <property>
- <name>dfs.nameservices</name>
- <value>ns0,ns1,ns2,ns3,ns-fed</value>
- </property>
- <property>
- <name>dfs.ha.namenodes.ns-fed</name>
- <value>r1,r2</value>
- </property>
- <property>
- <name>dfs.namenode.rpc-address.ns-fed.r1</name>
- <value>router1:rpc-port</value>
- </property>
- <property>
- <name>dfs.namenode.rpc-address.ns-fed.r2</name>
- <value>router2:rpc-port</value>
- </property>
- <property>
- <name>dfs.client.failover.proxy.provider.ns-fed</name>
- <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
- </property>
- <property>
- <name>dfs.client.failover.random.order</name>
- <value>true</value>
- </property>
-</configuration>
-```
-
-The `dfs.client.failover.random.order` set to `true` allows distributing the load evenly across the routers.
-
-With this setting a user can interact with `ns-fed` as a regular namespace:
-
- $ $HADOOP_HOME/bin/hdfs dfs -ls hdfs://ns-fed/
- /tmp
- /data
-
-This federated namespace can also be set as the default one at **core-site.xml** using `fs.defaultFS`.
-
-
-Router configuration
---------------------
-
-One can add the configurations for Router-based federation to **hdfs-site.xml**.
-The main options are documented in [hdfs-default.xml](./hdfs-default.xml).
-The configuration values are described in this section.
-
-### RPC server
-
-The RPC server to receive connections from the clients.
-
-| Property | Default | Description|
-|:---- |:---- |:---- |
-| dfs.federation.router.default.nameserviceId | | Nameservice identifier of the default subcluster to monitor. |
-| dfs.federation.router.rpc.enable | `true` | If `true`, the RPC service to handle client requests in the router is enabled. |
-| dfs.federation.router.rpc-address | 0.0.0.0:8888 | RPC address that handles all clients requests. |
-| dfs.federation.router.rpc-bind-host | 0.0.0.0 | The actual address the RPC server will bind to. |
-| dfs.federation.router.handler.count | 10 | The number of server threads for the router to handle RPC requests from clients. |
-| dfs.federation.router.handler.queue.size | 100 | The size of the queue for the number of handlers to handle RPC client requests. |
-| dfs.federation.router.reader.count | 1 | The number of readers for the router to handle RPC client requests. |
-| dfs.federation.router.reader.queue.size | 100 | The size of the queue for the number of readers for the router to handle RPC client requests. |
-
-#### Connection to the Namenodes
-
-The Router forwards the client requests to the NameNodes.
-It uses a pool of connections to reduce the latency of creating them.
-
-| Property | Default | Description|
-|:---- |:---- |:---- |
-| dfs.federation.router.connection.pool-size | 1 | Size of the pool of connections from the router to namenodes. |
-| dfs.federation.router.connection.clean.ms | 10000 | Time interval, in milliseconds, to check if the connection pool should remove unused connections. |
-| dfs.federation.router.connection.pool.clean.ms | 60000 | Time interval, in milliseconds, to check if the connection manager should remove unused connection pools. |
-
-### Admin server
-
-The administration server to manage the Mount Table.
-
-| Property | Default | Description|
-|:---- |:---- |:---- |
-| dfs.federation.router.admin.enable | `true` | If `true`, the RPC admin service to handle client requests in the router is enabled. |
-| dfs.federation.router.admin-address | 0.0.0.0:8111 | RPC address that handles the admin requests. |
-| dfs.federation.router.admin-bind-host | 0.0.0.0 | The actual address the RPC admin server will bind to. |
-| dfs.federation.router.admin.handler.count | 1 | The number of server threads for the router to handle RPC requests from admin. |
-
-### State Store
-
-The connection to the State Store and the internal caching at the Router.
-
-| Property | Default | Description|
-|:---- |:---- |:---- |
-| dfs.federation.router.store.enable | `true` | If `true`, the Router connects to the State Store. |
-| dfs.federation.router.store.serializer | `org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreSerializerPBImpl` | Class to serialize State Store records. |
-| dfs.federation.router.store.driver.class | `org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl` | Class to implement the State Store. |
-| dfs.federation.router.store.connection.test | 60000 | How often to check for the connection to the State Store in milliseconds. |
-| dfs.federation.router.cache.ttl | 60000 | How often to refresh the State Store caches in milliseconds. |
-| dfs.federation.router.store.membership.expiration | 300000 | Expiration time in milliseconds for a membership record. |
-
-### Routing
-
-Forwarding client requests to the right subcluster.
-
-| Property | Default | Description|
-|:---- |:---- |:---- |
-| dfs.federation.router.file.resolver.client.class | `org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver` | Class to resolve files to subclusters. |
-| dfs.federation.router.namenode.resolver.client.class | `org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver` | Class to resolve the namenode for a subcluster. |
-
-### Namenode monitoring
-
-Monitor the namenodes in the subclusters for forwarding the client requests.
-
-| Property | Default | Description|
-|:---- |:---- |:---- |
-| dfs.federation.router.heartbeat.enable | `true` | If `true`, the Router heartbeats into the State Store. |
-| dfs.federation.router.heartbeat.interval | 5000 | How often the Router should heartbeat into the State Store in milliseconds. |
-| dfs.federation.router.monitor.namenode | | The identifier of the namenodes to monitor and heartbeat. |
-| dfs.federation.router.monitor.localnamenode.enable | `true` | If `true`, the Router should monitor the namenode in the local machine. |
-
-Note: The config *dfs.nameservice.id* is recommended to configure if *dfs.federation.router.monitor.localnamenode.enable* is enabled.
-This will allow the Router finding the local node directly. Otherwise, it will find the nameservice Id by matching namenode RPC address with the
-local node address. If multiple addresses are matched, the Router will fail to start. In addition, if the local node is in a HA mode, it is recommend
-to configure *dfs.ha.namenode.id*.
-
-### Quota
-
-Global quota supported in federation.
-
-| Property | Default | Description|
-|:---- |:---- |:---- |
-| dfs.federation.router.quota.enable | `false` | If `true`, the quota system enabled in the Router. |
-| dfs.federation.router.quota-cache.update.interval | 60s | How often the Router updates quota cache. This setting supports multiple time unit suffixes. If no suffix is specified then milliseconds is assumed. |
-
-Metrics
--------
-
-The Router and State Store statistics are exposed in metrics/JMX. These info will be very useful for monitoring.
-More metrics info can see [Router RPC Metrics](../../hadoop-project-dist/hadoop-common/Metrics.html#RouterRPCMetrics) and [State Store Metrics](../../hadoop-project-dist/hadoop-common/Metrics.html#StateStoreMetrics).
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/site/resources/images/routerfederation.png
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/resources/images/routerfederation.png b/hadoop-hdfs-project/hadoop-hdfs/src/site/resources/images/routerfederation.png
deleted file mode 100644
index 2b158bb..0000000
Binary files a/hadoop-hdfs-project/hadoop-hdfs/src/site/resources/images/routerfederation.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
deleted file mode 100644
index 0ead93e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
+++ /dev/null
@@ -1,246 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-
-import java.io.BufferedReader;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.lang.management.ManagementFactory;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.util.Date;
-import java.util.List;
-import java.util.Random;
-
-import javax.management.JMX;
-import javax.management.MBeanServer;
-import javax.management.MalformedObjectNameException;
-import javax.management.ObjectName;
-
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.UnsupportedFileSystemException;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
-import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport;
-import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
-import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.test.GenericTestUtils;
-
-import com.google.common.base.Supplier;
-
-/**
- * Helper utilities for testing HDFS Federation.
- */
-public final class FederationTestUtils {
-
- public final static String[] NAMESERVICES = {"ns0", "ns1"};
- public final static String[] NAMENODES = {"nn0", "nn1", "nn2", "nn3"};
- public final static String[] ROUTERS =
- {"router0", "router1", "router2", "router3"};
-
-
- private FederationTestUtils() {
- // Utility class
- }
-
- public static void verifyException(Object obj, String methodName,
- Class<? extends Exception> exceptionClass, Class<?>[] parameterTypes,
- Object[] arguments) {
-
- Throwable triggeredException = null;
- try {
- Method m = obj.getClass().getMethod(methodName, parameterTypes);
- m.invoke(obj, arguments);
- } catch (InvocationTargetException ex) {
- triggeredException = ex.getTargetException();
- } catch (Exception e) {
- triggeredException = e;
- }
- if (exceptionClass != null) {
- assertNotNull("No exception was triggered, expected exception"
- + exceptionClass.getName(), triggeredException);
- assertEquals(exceptionClass, triggeredException.getClass());
- } else {
- assertNull("Exception was triggered but no exception was expected",
- triggeredException);
- }
- }
-
- public static NamenodeStatusReport createNamenodeReport(String ns, String nn,
- HAServiceState state) {
- Random rand = new Random();
- NamenodeStatusReport report = new NamenodeStatusReport(ns, nn,
- "localhost:" + rand.nextInt(10000), "localhost:" + rand.nextInt(10000),
- "localhost:" + rand.nextInt(10000), "testwebaddress-" + ns + nn);
- if (state == null) {
- // Unavailable, no additional info
- return report;
- }
- report.setHAServiceState(state);
- NamespaceInfo nsInfo = new NamespaceInfo(
- 1, "tesclusterid", ns, 0, "testbuildvesion", "testsoftwareversion");
- report.setNamespaceInfo(nsInfo);
- return report;
- }
-
- /**
- * Wait for a namenode to be registered with a particular state.
- * @param resolver Active namenode resolver.
- * @param nsId Nameservice identifier.
- * @param nnId Namenode identifier.
- * @param finalState State to check for.
- * @throws Exception Failed to verify State Store registration of namenode
- * nsId:nnId for state.
- */
- public static void waitNamenodeRegistered(
- final ActiveNamenodeResolver resolver,
- final String nsId, final String nnId,
- final FederationNamenodeServiceState state) throws Exception {
-
- GenericTestUtils.waitFor(new Supplier<Boolean>() {
- @Override
- public Boolean get() {
- try {
- List<? extends FederationNamenodeContext> namenodes =
- resolver.getNamenodesForNameserviceId(nsId);
- if (namenodes != null) {
- for (FederationNamenodeContext namenode : namenodes) {
- // Check if this is the Namenode we are checking
- if (namenode.getNamenodeId() == nnId ||
- namenode.getNamenodeId().equals(nnId)) {
- return state == null || namenode.getState().equals(state);
- }
- }
- }
- } catch (IOException e) {
- // Ignore
- }
- return false;
- }
- }, 1000, 20 * 1000);
- }
-
- public static boolean verifyDate(Date d1, Date d2, long precision) {
- return Math.abs(d1.getTime() - d2.getTime()) < precision;
- }
-
- public static <T> T getBean(String name, Class<T> obj)
- throws MalformedObjectNameException {
- MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer();
- ObjectName poolName = new ObjectName(name);
- return JMX.newMXBeanProxy(mBeanServer, poolName, obj);
- }
-
- public static boolean addDirectory(FileSystem context, String path)
- throws IOException {
- context.mkdirs(new Path(path), new FsPermission("777"));
- return verifyFileExists(context, path);
- }
-
- public static FileStatus getFileStatus(FileSystem context, String path)
- throws IOException {
- return context.getFileStatus(new Path(path));
- }
-
- public static boolean verifyFileExists(FileSystem context, String path) {
- try {
- FileStatus status = getFileStatus(context, path);
- if (status != null) {
- return true;
- }
- } catch (Exception e) {
- return false;
- }
- return false;
- }
-
- public static boolean checkForFileInDirectory(
- FileSystem context, String testPath, String targetFile)
- throws IOException, AccessControlException, FileNotFoundException,
- UnsupportedFileSystemException, IllegalArgumentException {
-
- FileStatus[] fileStatus = context.listStatus(new Path(testPath));
- String file = null;
- String verifyPath = testPath + "/" + targetFile;
- if (testPath.equals("/")) {
- verifyPath = testPath + targetFile;
- }
-
- Boolean found = false;
- for (int i = 0; i < fileStatus.length; i++) {
- FileStatus f = fileStatus[i];
- file = Path.getPathWithoutSchemeAndAuthority(f.getPath()).toString();
- if (file.equals(verifyPath)) {
- found = true;
- }
- }
- return found;
- }
-
- public static int countContents(FileSystem context, String testPath)
- throws IOException {
- Path path = new Path(testPath);
- FileStatus[] fileStatus = context.listStatus(path);
- return fileStatus.length;
- }
-
- public static void createFile(FileSystem fs, String path, long length)
- throws IOException {
- FsPermission permissions = new FsPermission("700");
- FSDataOutputStream writeStream = fs.create(new Path(path), permissions,
- true, 1000, (short) 1, DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT, null);
- for (int i = 0; i < length; i++) {
- writeStream.write(i);
- }
- writeStream.close();
- }
-
- public static String readFile(FileSystem fs, String path) throws IOException {
- // Read the file from the filesystem via the active namenode
- Path fileName = new Path(path);
- InputStreamReader reader = new InputStreamReader(fs.open(fileName));
- BufferedReader bufferedReader = new BufferedReader(reader);
- StringBuilder data = new StringBuilder();
- String line;
-
- while ((line = bufferedReader.readLine()) != null) {
- data.append(line);
- }
-
- bufferedReader.close();
- reader.close();
- return data.toString();
- }
-
- public static boolean deleteFile(FileSystem fs, String path)
- throws IOException {
- return fs.delete(new Path(path), true);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
deleted file mode 100644
index 151d731..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
+++ /dev/null
@@ -1,313 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
-import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.NamenodePriorityComparator;
-import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport;
-import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
-import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
-import org.apache.hadoop.hdfs.server.federation.router.Router;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
-import org.apache.hadoop.util.Time;
-
-/**
- * In-memory cache/mock of a namenode and file resolver. Stores the most
- * recently updated NN information for each nameservice and block pool. It also
- * stores a virtual mount table for resolving global namespace paths to local NN
- * paths.
- */
-public class MockResolver
- implements ActiveNamenodeResolver, FileSubclusterResolver {
-
- private Map<String, List<? extends FederationNamenodeContext>> resolver =
- new HashMap<>();
- private Map<String, List<RemoteLocation>> locations = new HashMap<>();
- private Set<FederationNamespaceInfo> namespaces = new HashSet<>();
- private String defaultNamespace = null;
-
- public MockResolver() {
- this.cleanRegistrations();
- }
-
- public MockResolver(Configuration conf) {
- this();
- }
-
- public MockResolver(Configuration conf, StateStoreService store) {
- this();
- }
-
- public MockResolver(Configuration conf, Router router) {
- this();
- }
-
- public void addLocation(String mount, String nsId, String location) {
- List<RemoteLocation> locationsList = this.locations.get(mount);
- if (locationsList == null) {
- locationsList = new LinkedList<>();
- this.locations.put(mount, locationsList);
- }
-
- final RemoteLocation remoteLocation = new RemoteLocation(nsId, location);
- if (!locationsList.contains(remoteLocation)) {
- locationsList.add(remoteLocation);
- }
-
- if (this.defaultNamespace == null) {
- this.defaultNamespace = nsId;
- }
- }
-
- public synchronized void cleanRegistrations() {
- this.resolver = new HashMap<>();
- this.namespaces = new HashSet<>();
- }
-
- @Override
- public void updateActiveNamenode(
- String nsId, InetSocketAddress successfulAddress) {
-
- String address = successfulAddress.getHostName() + ":" +
- successfulAddress.getPort();
- String key = nsId;
- if (key != null) {
- // Update the active entry
- @SuppressWarnings("unchecked")
- List<FederationNamenodeContext> namenodes =
- (List<FederationNamenodeContext>) this.resolver.get(key);
- for (FederationNamenodeContext namenode : namenodes) {
- if (namenode.getRpcAddress().equals(address)) {
- MockNamenodeContext nn = (MockNamenodeContext) namenode;
- nn.setState(FederationNamenodeServiceState.ACTIVE);
- break;
- }
- }
- // This operation modifies the list so we need to be careful
- synchronized(namenodes) {
- Collections.sort(namenodes, new NamenodePriorityComparator());
- }
- }
- }
-
- @Override
- public List<? extends FederationNamenodeContext>
- getNamenodesForNameserviceId(String nameserviceId) {
- // Return a copy of the list because it is updated periodically
- List<? extends FederationNamenodeContext> namenodes =
- this.resolver.get(nameserviceId);
- return Collections.unmodifiableList(new ArrayList<>(namenodes));
- }
-
- @Override
- public List<? extends FederationNamenodeContext> getNamenodesForBlockPoolId(
- String blockPoolId) {
- // Return a copy of the list because it is updated periodically
- List<? extends FederationNamenodeContext> namenodes =
- this.resolver.get(blockPoolId);
- return Collections.unmodifiableList(new ArrayList<>(namenodes));
- }
-
- private static class MockNamenodeContext
- implements FederationNamenodeContext {
-
- private String namenodeId;
- private String nameserviceId;
-
- private String webAddress;
- private String rpcAddress;
- private String serviceAddress;
- private String lifelineAddress;
-
- private FederationNamenodeServiceState state;
- private long dateModified;
-
- MockNamenodeContext(
- String rpc, String service, String lifeline, String web,
- String ns, String nn, FederationNamenodeServiceState state) {
- this.rpcAddress = rpc;
- this.serviceAddress = service;
- this.lifelineAddress = lifeline;
- this.webAddress = web;
- this.namenodeId = nn;
- this.nameserviceId = ns;
- this.state = state;
- this.dateModified = Time.now();
- }
-
- public void setState(FederationNamenodeServiceState newState) {
- this.state = newState;
- this.dateModified = Time.now();
- }
-
- @Override
- public String getRpcAddress() {
- return rpcAddress;
- }
-
- @Override
- public String getServiceAddress() {
- return serviceAddress;
- }
-
- @Override
- public String getLifelineAddress() {
- return lifelineAddress;
- }
-
- @Override
- public String getWebAddress() {
- return webAddress;
- }
-
- @Override
- public String getNamenodeKey() {
- return nameserviceId + " " + namenodeId + " " + rpcAddress;
- }
-
- @Override
- public String getNameserviceId() {
- return nameserviceId;
- }
-
- @Override
- public String getNamenodeId() {
- return namenodeId;
- }
-
- @Override
- public FederationNamenodeServiceState getState() {
- return state;
- }
-
- @Override
- public long getDateModified() {
- return dateModified;
- }
- }
-
- @Override
- public synchronized boolean registerNamenode(NamenodeStatusReport report)
- throws IOException {
-
- MockNamenodeContext context = new MockNamenodeContext(
- report.getRpcAddress(), report.getServiceAddress(),
- report.getLifelineAddress(), report.getWebAddress(),
- report.getNameserviceId(), report.getNamenodeId(), report.getState());
-
- String nsId = report.getNameserviceId();
- String bpId = report.getBlockPoolId();
- String cId = report.getClusterId();
-
- @SuppressWarnings("unchecked")
- List<MockNamenodeContext> existingItems =
- (List<MockNamenodeContext>) this.resolver.get(nsId);
- if (existingItems == null) {
- existingItems = new ArrayList<>();
- this.resolver.put(bpId, existingItems);
- this.resolver.put(nsId, existingItems);
- }
- boolean added = false;
- for (int i=0; i<existingItems.size() && !added; i++) {
- MockNamenodeContext existing = existingItems.get(i);
- if (existing.getNamenodeKey().equals(context.getNamenodeKey())) {
- existingItems.set(i, context);
- added = true;
- }
- }
- if (!added) {
- existingItems.add(context);
- }
- Collections.sort(existingItems, new NamenodePriorityComparator());
-
- FederationNamespaceInfo info = new FederationNamespaceInfo(bpId, cId, nsId);
- this.namespaces.add(info);
- return true;
- }
-
- @Override
- public Set<FederationNamespaceInfo> getNamespaces() throws IOException {
- return this.namespaces;
- }
-
- @Override
- public PathLocation getDestinationForPath(String path) throws IOException {
- List<RemoteLocation> remoteLocations = new LinkedList<>();
- // We go from the leaves to the root
- List<String> keys = new ArrayList<>(this.locations.keySet());
- Collections.sort(keys, Collections.reverseOrder());
- for (String key : keys) {
- if (path.startsWith(key)) {
- for (RemoteLocation location : this.locations.get(key)) {
- String finalPath = location.getDest() + path.substring(key.length());
- String nameservice = location.getNameserviceId();
- RemoteLocation remoteLocation =
- new RemoteLocation(nameservice, finalPath);
- remoteLocations.add(remoteLocation);
- }
- break;
- }
- }
- if (remoteLocations.isEmpty()) {
- // Path isn't supported, mimic resolver behavior.
- return null;
- }
- return new PathLocation(path, remoteLocations);
- }
-
- @Override
- public List<String> getMountPoints(String path) throws IOException {
- List<String> mounts = new ArrayList<>();
- if (path.equals("/")) {
- // Mounts only supported under root level
- for (String mount : this.locations.keySet()) {
- if (mount.length() > 1) {
- // Remove leading slash, this is the behavior of the mount tree,
- // return only names.
- mounts.add(mount.replace("/", ""));
- }
- }
- }
- return mounts;
- }
-
- @Override
- public void setRouterId(String router) {
- }
-
- @Override
- public String getDefaultNamespace() {
- return defaultNamespace;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
deleted file mode 100644
index b332f1f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
-
-/**
- * Constructs a router configuration with individual features enabled/disabled.
- */
-public class RouterConfigBuilder {
-
- private Configuration conf;
-
- private boolean enableRpcServer = false;
- private boolean enableAdminServer = false;
- private boolean enableHttpServer = false;
- private boolean enableHeartbeat = false;
- private boolean enableLocalHeartbeat = false;
- private boolean enableStateStore = false;
- private boolean enableMetrics = false;
- private boolean enableQuota = false;
- private boolean enableSafemode = false;
-
- public RouterConfigBuilder(Configuration configuration) {
- this.conf = configuration;
- }
-
- public RouterConfigBuilder() {
- this.conf = new Configuration(false);
- }
-
- public RouterConfigBuilder all() {
- this.enableRpcServer = true;
- this.enableAdminServer = true;
- this.enableHttpServer = true;
- this.enableHeartbeat = true;
- this.enableLocalHeartbeat = true;
- this.enableStateStore = true;
- this.enableMetrics = true;
- this.enableSafemode = true;
- return this;
- }
-
- public RouterConfigBuilder enableLocalHeartbeat(boolean enable) {
- this.enableLocalHeartbeat = enable;
- return this;
- }
-
- public RouterConfigBuilder rpc(boolean enable) {
- this.enableRpcServer = enable;
- return this;
- }
-
- public RouterConfigBuilder admin(boolean enable) {
- this.enableAdminServer = enable;
- return this;
- }
-
- public RouterConfigBuilder http(boolean enable) {
- this.enableHttpServer = enable;
- return this;
- }
-
- public RouterConfigBuilder heartbeat(boolean enable) {
- this.enableHeartbeat = enable;
- return this;
- }
-
- public RouterConfigBuilder stateStore(boolean enable) {
- this.enableStateStore = enable;
- return this;
- }
-
- public RouterConfigBuilder metrics(boolean enable) {
- this.enableMetrics = enable;
- return this;
- }
-
- public RouterConfigBuilder quota(boolean enable) {
- this.enableQuota = enable;
- return this;
- }
-
- public RouterConfigBuilder safemode(boolean enable) {
- this.enableSafemode = enable;
- return this;
- }
-
- public RouterConfigBuilder rpc() {
- return this.rpc(true);
- }
-
- public RouterConfigBuilder admin() {
- return this.admin(true);
- }
-
- public RouterConfigBuilder http() {
- return this.http(true);
- }
-
- public RouterConfigBuilder heartbeat() {
- return this.heartbeat(true);
- }
-
- public RouterConfigBuilder stateStore() {
- // reset the State Store driver implementation class for testing
- conf.setClass(DFSConfigKeys.FEDERATION_STORE_DRIVER_CLASS,
- FederationStateStoreTestUtils.getTestDriverClass(),
- StateStoreDriver.class);
- return this.stateStore(true);
- }
-
- public RouterConfigBuilder metrics() {
- return this.metrics(true);
- }
-
- public RouterConfigBuilder quota() {
- return this.quota(true);
- }
-
- public RouterConfigBuilder safemode() {
- return this.safemode(true);
- }
-
- public Configuration build() {
- conf.setBoolean(DFSConfigKeys.DFS_ROUTER_STORE_ENABLE,
- this.enableStateStore);
- conf.setBoolean(DFSConfigKeys.DFS_ROUTER_RPC_ENABLE, this.enableRpcServer);
- conf.setBoolean(DFSConfigKeys.DFS_ROUTER_ADMIN_ENABLE,
- this.enableAdminServer);
- conf.setBoolean(DFSConfigKeys.DFS_ROUTER_HTTP_ENABLE,
- this.enableHttpServer);
- conf.setBoolean(DFSConfigKeys.DFS_ROUTER_HEARTBEAT_ENABLE,
- this.enableHeartbeat);
- conf.setBoolean(DFSConfigKeys.DFS_ROUTER_MONITOR_LOCAL_NAMENODE,
- this.enableLocalHeartbeat);
- conf.setBoolean(DFSConfigKeys.DFS_ROUTER_METRICS_ENABLE,
- this.enableMetrics);
- conf.setBoolean(DFSConfigKeys.DFS_ROUTER_QUOTA_ENABLE,
- this.enableQuota);
- conf.setBoolean(DFSConfigKeys.DFS_ROUTER_SAFEMODE_ENABLE,
- this.enableSafemode);
- return conf;
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMembershipState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMembershipState.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMembershipState.java
deleted file mode 100644
index 26f081b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMembershipState.java
+++ /dev/null
@@ -1,463 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store;
-
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMENODES;
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMESERVICES;
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.ROUTERS;
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyException;
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.clearRecords;
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.createMockRegistrationForNamenode;
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.synchronizeRecords;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateNamenodeRegistrationRequest;
-import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
-import org.apache.hadoop.util.Time;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- * Test the basic {@link MembershipStore} membership functionality.
- */
-public class TestStateStoreMembershipState extends TestStateStoreBase {
-
- private static MembershipStore membershipStore;
-
- @BeforeClass
- public static void create() {
- // Reduce expirations to 5 seconds
- getConf().setLong(
- DFSConfigKeys.FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS,
- TimeUnit.SECONDS.toMillis(5));
- }
-
- @Before
- public void setup() throws IOException, InterruptedException {
-
- membershipStore =
- getStateStore().getRegisteredRecordStore(MembershipStore.class);
-
- // Clear NN registrations
- assertTrue(clearRecords(getStateStore(), MembershipState.class));
- }
-
- @Test
- public void testNamenodeStateOverride() throws Exception {
- // Populate the state store
- // 1) ns0:nn0 - Standby
- String ns = "ns0";
- String nn = "nn0";
- MembershipState report = createRegistration(
- ns, nn, ROUTERS[1], FederationNamenodeServiceState.STANDBY);
- assertTrue(namenodeHeartbeat(report));
-
- // Load data into cache and calculate quorum
- assertTrue(getStateStore().loadCache(MembershipStore.class, true));
-
- MembershipState existingState = getNamenodeRegistration(ns, nn);
- assertEquals(
- FederationNamenodeServiceState.STANDBY, existingState.getState());
-
- // Override cache
- UpdateNamenodeRegistrationRequest request =
- UpdateNamenodeRegistrationRequest.newInstance(
- ns, nn, FederationNamenodeServiceState.ACTIVE);
- assertTrue(membershipStore.updateNamenodeRegistration(request).getResult());
-
- MembershipState newState = getNamenodeRegistration(ns, nn);
- assertEquals(FederationNamenodeServiceState.ACTIVE, newState.getState());
- }
-
- @Test
- public void testStateStoreDisconnected() throws Exception {
-
- // Close the data store driver
- getStateStore().closeDriver();
- assertFalse(getStateStore().isDriverReady());
-
- NamenodeHeartbeatRequest hbRequest = NamenodeHeartbeatRequest.newInstance();
- hbRequest.setNamenodeMembership(
- createMockRegistrationForNamenode(
- "test", "test", FederationNamenodeServiceState.UNAVAILABLE));
- verifyException(membershipStore, "namenodeHeartbeat",
- StateStoreUnavailableException.class,
- new Class[] {NamenodeHeartbeatRequest.class},
- new Object[] {hbRequest });
-
- // Information from cache, no exception should be triggered for these
- // TODO - should cached info expire at some point?
- GetNamenodeRegistrationsRequest getRequest =
- GetNamenodeRegistrationsRequest.newInstance();
- verifyException(membershipStore,
- "getNamenodeRegistrations", null,
- new Class[] {GetNamenodeRegistrationsRequest.class},
- new Object[] {getRequest});
-
- verifyException(membershipStore,
- "getExpiredNamenodeRegistrations", null,
- new Class[] {GetNamenodeRegistrationsRequest.class},
- new Object[] {getRequest});
-
- UpdateNamenodeRegistrationRequest overrideRequest =
- UpdateNamenodeRegistrationRequest.newInstance();
- verifyException(membershipStore,
- "updateNamenodeRegistration", null,
- new Class[] {UpdateNamenodeRegistrationRequest.class},
- new Object[] {overrideRequest});
- }
-
- private void registerAndLoadRegistrations(
- List<MembershipState> registrationList) throws IOException {
- // Populate
- assertTrue(synchronizeRecords(
- getStateStore(), registrationList, MembershipState.class));
-
- // Load into cache
- assertTrue(getStateStore().loadCache(MembershipStore.class, true));
- }
-
- private MembershipState createRegistration(String ns, String nn,
- String router, FederationNamenodeServiceState state) throws IOException {
- MembershipState record = MembershipState.newInstance(
- router, ns,
- nn, "testcluster", "testblock-" + ns, "testrpc-"+ ns + nn,
- "testservice-"+ ns + nn, "testlifeline-"+ ns + nn,
- "testweb-" + ns + nn, state, false);
- return record;
- }
-
- @Test
- public void testRegistrationMajorityQuorum()
- throws InterruptedException, IOException {
-
- // Populate the state store with a set of non-matching elements
- // 1) ns0:nn0 - Standby (newest)
- // 2) ns0:nn0 - Active (oldest)
- // 3) ns0:nn0 - Active (2nd oldest)
- // 4) ns0:nn0 - Active (3nd oldest element, newest active element)
- // Verify the selected entry is the newest majority opinion (4)
- String ns = "ns0";
- String nn = "nn0";
-
- // Active - oldest
- MembershipState report = createRegistration(
- ns, nn, ROUTERS[1], FederationNamenodeServiceState.ACTIVE);
- assertTrue(namenodeHeartbeat(report));
- Thread.sleep(1000);
-
- // Active - 2nd oldest
- report = createRegistration(
- ns, nn, ROUTERS[2], FederationNamenodeServiceState.ACTIVE);
- assertTrue(namenodeHeartbeat(report));
- Thread.sleep(1000);
-
- // Active - 3rd oldest, newest active element
- report = createRegistration(
- ns, nn, ROUTERS[3], FederationNamenodeServiceState.ACTIVE);
- assertTrue(namenodeHeartbeat(report));
-
- // standby - newest overall
- report = createRegistration(
- ns, nn, ROUTERS[0], FederationNamenodeServiceState.STANDBY);
- assertTrue(namenodeHeartbeat(report));
-
- // Load and calculate quorum
- assertTrue(getStateStore().loadCache(MembershipStore.class, true));
-
- // Verify quorum entry
- MembershipState quorumEntry = getNamenodeRegistration(
- report.getNameserviceId(), report.getNamenodeId());
- assertNotNull(quorumEntry);
- assertEquals(quorumEntry.getRouterId(), ROUTERS[3]);
- }
-
- @Test
- public void testRegistrationQuorumExcludesExpired()
- throws InterruptedException, IOException {
-
- // Populate the state store with some expired entries and verify the expired
- // entries are ignored.
- // 1) ns0:nn0 - Active
- // 2) ns0:nn0 - Expired
- // 3) ns0:nn0 - Expired
- // 4) ns0:nn0 - Expired
- // Verify the selected entry is the active entry
- List<MembershipState> registrationList = new ArrayList<>();
- String ns = "ns0";
- String nn = "nn0";
- String rpcAddress = "testrpcaddress";
- String serviceAddress = "testserviceaddress";
- String lifelineAddress = "testlifelineaddress";
- String blockPoolId = "testblockpool";
- String clusterId = "testcluster";
- String webAddress = "testwebaddress";
- boolean safemode = false;
-
- // Active
- MembershipState record = MembershipState.newInstance(
- ROUTERS[0], ns, nn, clusterId, blockPoolId,
- rpcAddress, serviceAddress, lifelineAddress, webAddress,
- FederationNamenodeServiceState.ACTIVE, safemode);
- registrationList.add(record);
-
- // Expired
- record = MembershipState.newInstance(
- ROUTERS[1], ns, nn, clusterId, blockPoolId,
- rpcAddress, serviceAddress, lifelineAddress, webAddress,
- FederationNamenodeServiceState.EXPIRED, safemode);
- registrationList.add(record);
-
- // Expired
- record = MembershipState.newInstance(
- ROUTERS[2], ns, nn, clusterId, blockPoolId,
- rpcAddress, serviceAddress, lifelineAddress, webAddress,
- FederationNamenodeServiceState.EXPIRED, safemode);
- registrationList.add(record);
-
- // Expired
- record = MembershipState.newInstance(
- ROUTERS[3], ns, nn, clusterId, blockPoolId,
- rpcAddress, serviceAddress, lifelineAddress, webAddress,
- FederationNamenodeServiceState.EXPIRED, safemode);
- registrationList.add(record);
- registerAndLoadRegistrations(registrationList);
-
- // Verify quorum entry chooses active membership
- MembershipState quorumEntry = getNamenodeRegistration(
- record.getNameserviceId(), record.getNamenodeId());
- assertNotNull(quorumEntry);
- assertEquals(ROUTERS[0], quorumEntry.getRouterId());
- }
-
- @Test
- public void testRegistrationQuorumAllExpired() throws IOException {
-
- // 1) ns0:nn0 - Expired (oldest)
- // 2) ns0:nn0 - Expired
- // 3) ns0:nn0 - Expired
- // 4) ns0:nn0 - Expired
- // Verify no entry is either selected or cached
- List<MembershipState> registrationList = new ArrayList<>();
- String ns = NAMESERVICES[0];
- String nn = NAMENODES[0];
- String rpcAddress = "testrpcaddress";
- String serviceAddress = "testserviceaddress";
- String lifelineAddress = "testlifelineaddress";
- String blockPoolId = "testblockpool";
- String clusterId = "testcluster";
- String webAddress = "testwebaddress";
- boolean safemode = false;
- long startingTime = Time.now();
-
- // Expired
- MembershipState record = MembershipState.newInstance(
- ROUTERS[0], ns, nn, clusterId, blockPoolId,
- rpcAddress, webAddress, lifelineAddress, webAddress,
- FederationNamenodeServiceState.EXPIRED, safemode);
- record.setDateModified(startingTime - 10000);
- registrationList.add(record);
-
- // Expired
- record = MembershipState.newInstance(
- ROUTERS[1], ns, nn, clusterId, blockPoolId,
- rpcAddress, serviceAddress, lifelineAddress, webAddress,
- FederationNamenodeServiceState.EXPIRED, safemode);
- record.setDateModified(startingTime);
- registrationList.add(record);
-
- // Expired
- record = MembershipState.newInstance(
- ROUTERS[2], ns, nn, clusterId, blockPoolId,
- rpcAddress, serviceAddress, lifelineAddress, webAddress,
- FederationNamenodeServiceState.EXPIRED, safemode);
- record.setDateModified(startingTime);
- registrationList.add(record);
-
- // Expired
- record = MembershipState.newInstance(
- ROUTERS[3], ns, nn, clusterId, blockPoolId,
- rpcAddress, serviceAddress, lifelineAddress, webAddress,
- FederationNamenodeServiceState.EXPIRED, safemode);
- record.setDateModified(startingTime);
- registrationList.add(record);
-
- registerAndLoadRegistrations(registrationList);
-
- // Verify no entry is found for this nameservice
- assertNull(getNamenodeRegistration(
- record.getNameserviceId(), record.getNamenodeId()));
- }
-
- @Test
- public void testRegistrationNoQuorum()
- throws InterruptedException, IOException {
-
- // Populate the state store with a set of non-matching elements
- // 1) ns0:nn0 - Standby (newest)
- // 2) ns0:nn0 - Standby (oldest)
- // 3) ns0:nn0 - Active (2nd oldest)
- // 4) ns0:nn0 - Active (3nd oldest element, newest active element)
- // Verify the selected entry is the newest entry (1)
- MembershipState report1 = createRegistration(
- NAMESERVICES[0], NAMENODES[0], ROUTERS[1],
- FederationNamenodeServiceState.STANDBY);
- assertTrue(namenodeHeartbeat(report1));
- Thread.sleep(100);
- MembershipState report2 = createRegistration(
- NAMESERVICES[0], NAMENODES[0], ROUTERS[2],
- FederationNamenodeServiceState.ACTIVE);
- assertTrue(namenodeHeartbeat(report2));
- Thread.sleep(100);
- MembershipState report3 = createRegistration(
- NAMESERVICES[0], NAMENODES[0], ROUTERS[3],
- FederationNamenodeServiceState.ACTIVE);
- assertTrue(namenodeHeartbeat(report3));
- Thread.sleep(100);
- MembershipState report4 = createRegistration(
- NAMESERVICES[0], NAMENODES[0], ROUTERS[0],
- FederationNamenodeServiceState.STANDBY);
- assertTrue(namenodeHeartbeat(report4));
-
- // Load and calculate quorum
- assertTrue(getStateStore().loadCache(MembershipStore.class, true));
-
- // Verify quorum entry uses the newest data, even though it is standby
- MembershipState quorumEntry = getNamenodeRegistration(
- report1.getNameserviceId(), report1.getNamenodeId());
- assertNotNull(quorumEntry);
- assertEquals(ROUTERS[0], quorumEntry.getRouterId());
- assertEquals(
- FederationNamenodeServiceState.STANDBY, quorumEntry.getState());
- }
-
- @Test
- public void testRegistrationExpired()
- throws InterruptedException, IOException {
-
- // Populate the state store with a single NN element
- // 1) ns0:nn0 - Active
- // Wait for the entry to expire without heartbeating
- // Verify the NN entry is populated as EXPIRED internally in the state store
-
- MembershipState report = createRegistration(
- NAMESERVICES[0], NAMENODES[0], ROUTERS[0],
- FederationNamenodeServiceState.ACTIVE);
- assertTrue(namenodeHeartbeat(report));
-
- // Load cache
- assertTrue(getStateStore().loadCache(MembershipStore.class, true));
-
- // Verify quorum and entry
- MembershipState quorumEntry = getNamenodeRegistration(
- report.getNameserviceId(), report.getNamenodeId());
- assertNotNull(quorumEntry);
- assertEquals(ROUTERS[0], quorumEntry.getRouterId());
- assertEquals(FederationNamenodeServiceState.ACTIVE, quorumEntry.getState());
-
- // Wait past expiration (set in conf to 5 seconds)
- Thread.sleep(6000);
- // Reload cache
- assertTrue(getStateStore().loadCache(MembershipStore.class, true));
-
- // Verify entry is now expired and is no longer in the cache
- quorumEntry = getNamenodeRegistration(NAMESERVICES[0], NAMENODES[0]);
- assertNull(quorumEntry);
-
- // Verify entry is now expired and can't be used by RPC service
- quorumEntry = getNamenodeRegistration(
- report.getNameserviceId(), report.getNamenodeId());
- assertNull(quorumEntry);
-
- // Heartbeat again, updates dateModified
- assertTrue(namenodeHeartbeat(report));
- // Reload cache
- assertTrue(getStateStore().loadCache(MembershipStore.class, true));
-
- // Verify updated entry marked as active and is accessible to RPC server
- quorumEntry = getNamenodeRegistration(
- report.getNameserviceId(), report.getNamenodeId());
- assertNotNull(quorumEntry);
- assertEquals(ROUTERS[0], quorumEntry.getRouterId());
- assertEquals(FederationNamenodeServiceState.ACTIVE, quorumEntry.getState());
- }
-
- /**
- * Get a single namenode membership record from the store.
- *
- * @param nsId The HDFS nameservice ID to search for
- * @param nnId The HDFS namenode ID to search for
- * @return The single NamenodeMembershipRecord that matches the query or null
- * if not found.
- * @throws IOException if the query could not be executed.
- */
- private MembershipState getNamenodeRegistration(
- final String nsId, final String nnId) throws IOException {
-
- MembershipState partial = MembershipState.newInstance();
- partial.setNameserviceId(nsId);
- partial.setNamenodeId(nnId);
- GetNamenodeRegistrationsRequest request =
- GetNamenodeRegistrationsRequest.newInstance(partial);
- GetNamenodeRegistrationsResponse response =
- membershipStore.getNamenodeRegistrations(request);
-
- List<MembershipState> results = response.getNamenodeMemberships();
- if (results != null && results.size() == 1) {
- MembershipState record = results.get(0);
- return record;
- }
- return null;
- }
-
- /**
- * Register a namenode heartbeat with the state store.
- *
- * @param store FederationMembershipStateStore instance to retrieve the
- * membership data records.
- * @param namenode A fully populated namenode membership record to be
- * committed to the data store.
- * @return True if successful, false otherwise.
- * @throws IOException if the state store query could not be performed.
- */
- private boolean namenodeHeartbeat(MembershipState namenode)
- throws IOException {
-
- NamenodeHeartbeatRequest request =
- NamenodeHeartbeatRequest.newInstance(namenode);
- NamenodeHeartbeatResponse response =
- membershipStore.namenodeHeartbeat(request);
- return response.getResult();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMountTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMountTable.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMountTable.java
deleted file mode 100644
index d30d6ba..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMountTable.java
+++ /dev/null
@@ -1,250 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store;
-
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMESERVICES;
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyException;
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.clearRecords;
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.createMockMountTable;
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.synchronizeRecords;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- * Test the basic {@link StateStoreService}
- * {@link MountTableStore} functionality.
- */
-public class TestStateStoreMountTable extends TestStateStoreBase {
-
- private static List<String> nameservices;
- private static MountTableStore mountStore;
-
- @BeforeClass
- public static void create() throws IOException {
- nameservices = new ArrayList<>();
- nameservices.add(NAMESERVICES[0]);
- nameservices.add(NAMESERVICES[1]);
- }
-
- @Before
- public void setup() throws IOException, InterruptedException {
- mountStore =
- getStateStore().getRegisteredRecordStore(MountTableStore.class);
- // Clear Mount table registrations
- assertTrue(clearRecords(getStateStore(), MountTable.class));
- }
-
- @Test
- public void testStateStoreDisconnected() throws Exception {
-
- // Close the data store driver
- getStateStore().closeDriver();
- assertFalse(getStateStore().isDriverReady());
-
- // Test APIs that access the store to check they throw the correct exception
- AddMountTableEntryRequest addRequest =
- AddMountTableEntryRequest.newInstance();
- verifyException(mountStore, "addMountTableEntry",
- StateStoreUnavailableException.class,
- new Class[] {AddMountTableEntryRequest.class},
- new Object[] {addRequest});
-
- UpdateMountTableEntryRequest updateRequest =
- UpdateMountTableEntryRequest.newInstance();
- verifyException(mountStore, "updateMountTableEntry",
- StateStoreUnavailableException.class,
- new Class[] {UpdateMountTableEntryRequest.class},
- new Object[] {updateRequest});
-
- RemoveMountTableEntryRequest removeRequest =
- RemoveMountTableEntryRequest.newInstance();
- verifyException(mountStore, "removeMountTableEntry",
- StateStoreUnavailableException.class,
- new Class[] {RemoveMountTableEntryRequest.class},
- new Object[] {removeRequest});
-
- GetMountTableEntriesRequest getRequest =
- GetMountTableEntriesRequest.newInstance();
- mountStore.loadCache(true);
- verifyException(mountStore, "getMountTableEntries",
- StateStoreUnavailableException.class,
- new Class[] {GetMountTableEntriesRequest.class},
- new Object[] {getRequest});
- }
-
- @Test
- public void testSynchronizeMountTable() throws IOException {
- // Synchronize and get mount table entries
- List<MountTable> entries = createMockMountTable(nameservices);
- assertTrue(synchronizeRecords(getStateStore(), entries, MountTable.class));
- for (MountTable e : entries) {
- mountStore.loadCache(true);
- MountTable entry = getMountTableEntry(e.getSourcePath());
- assertNotNull(entry);
- assertEquals(e.getDefaultLocation().getDest(),
- entry.getDefaultLocation().getDest());
- }
- }
-
- @Test
- public void testAddMountTableEntry() throws IOException {
-
- // Add 1
- List<MountTable> entries = createMockMountTable(nameservices);
- List<MountTable> entries1 = getMountTableEntries("/").getRecords();
- assertEquals(0, entries1.size());
- MountTable entry0 = entries.get(0);
- AddMountTableEntryRequest request =
- AddMountTableEntryRequest.newInstance(entry0);
- AddMountTableEntryResponse response =
- mountStore.addMountTableEntry(request);
- assertTrue(response.getStatus());
-
- mountStore.loadCache(true);
- List<MountTable> entries2 = getMountTableEntries("/").getRecords();
- assertEquals(1, entries2.size());
- }
-
- @Test
- public void testRemoveMountTableEntry() throws IOException {
-
- // Add many
- List<MountTable> entries = createMockMountTable(nameservices);
- synchronizeRecords(getStateStore(), entries, MountTable.class);
- mountStore.loadCache(true);
- List<MountTable> entries1 = getMountTableEntries("/").getRecords();
- assertEquals(entries.size(), entries1.size());
-
- // Remove 1
- RemoveMountTableEntryRequest request =
- RemoveMountTableEntryRequest.newInstance();
- request.setSrcPath(entries.get(0).getSourcePath());
- assertTrue(mountStore.removeMountTableEntry(request).getStatus());
-
- // Verify remove
- mountStore.loadCache(true);
- List<MountTable> entries2 = getMountTableEntries("/").getRecords();
- assertEquals(entries.size() - 1, entries2.size());
- }
-
- @Test
- public void testUpdateMountTableEntry() throws IOException {
-
- // Add 1
- List<MountTable> entries = createMockMountTable(nameservices);
- MountTable entry0 = entries.get(0);
- String srcPath = entry0.getSourcePath();
- String nsId = entry0.getDefaultLocation().getNameserviceId();
- AddMountTableEntryRequest request =
- AddMountTableEntryRequest.newInstance(entry0);
- AddMountTableEntryResponse response =
- mountStore.addMountTableEntry(request);
- assertTrue(response.getStatus());
-
- // Verify
- mountStore.loadCache(true);
- MountTable matchingEntry0 = getMountTableEntry(srcPath);
- assertNotNull(matchingEntry0);
- assertEquals(nsId, matchingEntry0.getDefaultLocation().getNameserviceId());
-
- // Edit destination nameservice for source path
- Map<String, String> destMap =
- Collections.singletonMap("testnameservice", "/");
- MountTable replacement =
- MountTable.newInstance(srcPath, destMap);
- UpdateMountTableEntryRequest updateRequest =
- UpdateMountTableEntryRequest.newInstance(replacement);
- UpdateMountTableEntryResponse updateResponse =
- mountStore.updateMountTableEntry(updateRequest);
- assertTrue(updateResponse.getStatus());
-
- // Verify
- mountStore.loadCache(true);
- MountTable matchingEntry1 = getMountTableEntry(srcPath);
- assertNotNull(matchingEntry1);
- assertEquals("testnameservice",
- matchingEntry1.getDefaultLocation().getNameserviceId());
- }
-
- /**
- * Gets an existing mount table record in the state store.
- *
- * @param mount The mount point of the record to remove.
- * @return The matching record if found, null if it is not found.
- * @throws IOException If the state store could not be accessed.
- */
- private MountTable getMountTableEntry(String mount) throws IOException {
- GetMountTableEntriesRequest request =
- GetMountTableEntriesRequest.newInstance(mount);
- GetMountTableEntriesResponse response =
- mountStore.getMountTableEntries(request);
- List<MountTable> results = response.getEntries();
- if (results.size() > 0) {
- // First result is sorted to have the shortest mount string length
- return results.get(0);
- }
- return null;
- }
-
- /**
- * Fetch all mount table records beneath a root path.
- *
- * @param store FederationMountTableStore instance to commit the data.
- * @param mount The root search path, enter "/" to return all mount table
- * records.
- *
- * @return A list of all mount table records found below the root mount.
- *
- * @throws IOException If the state store could not be accessed.
- */
- private QueryResult<MountTable> getMountTableEntries(String mount)
- throws IOException {
- if (mount == null) {
- throw new IOException("Please specify a root search path");
- }
- GetMountTableEntriesRequest request =
- GetMountTableEntriesRequest.newInstance();
- request.setSrcPath(mount);
- GetMountTableEntriesResponse response =
- mountStore.getMountTableEntries(request);
- List<MountTable> records = response.getEntries();
- long timestamp = response.getTimestamp();
- return new QueryResult<MountTable>(records, timestamp);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreRouterState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreRouterState.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreRouterState.java
deleted file mode 100644
index cbc5e7d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreRouterState.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store;
-
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyException;
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.clearRecords;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.server.federation.router.FederationUtil;
-import org.apache.hadoop.hdfs.server.federation.router.RouterServiceState;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationsRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatRequest;
-import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
-import org.apache.hadoop.util.Time;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- * Test the basic {@link StateStoreService} {@link RouterStore} functionality.
- */
-public class TestStateStoreRouterState extends TestStateStoreBase {
-
- private static RouterStore routerStore;
-
- @BeforeClass
- public static void create() {
- // Reduce expirations to 5 seconds
- getConf().setTimeDuration(
- DFSConfigKeys.FEDERATION_STORE_ROUTER_EXPIRATION_MS,
- 5, TimeUnit.SECONDS);
- }
-
- @Before
- public void setup() throws IOException, InterruptedException {
-
- if (routerStore == null) {
- routerStore =
- getStateStore().getRegisteredRecordStore(RouterStore.class);
- }
-
- // Clear router status registrations
- assertTrue(clearRecords(getStateStore(), RouterState.class));
- }
-
- @Test
- public void testStateStoreDisconnected() throws Exception {
-
- // Close the data store driver
- getStateStore().closeDriver();
- assertEquals(false, getStateStore().isDriverReady());
-
- // Test all APIs that access the data store to ensure they throw the correct
- // exception.
- GetRouterRegistrationRequest getSingleRequest =
- GetRouterRegistrationRequest.newInstance();
- verifyException(routerStore, "getRouterRegistration",
- StateStoreUnavailableException.class,
- new Class[] {GetRouterRegistrationRequest.class},
- new Object[] {getSingleRequest});
-
- GetRouterRegistrationsRequest getRequest =
- GetRouterRegistrationsRequest.newInstance();
- routerStore.loadCache(true);
- verifyException(routerStore, "getRouterRegistrations",
- StateStoreUnavailableException.class,
- new Class[] {GetRouterRegistrationsRequest.class},
- new Object[] {getRequest});
-
- RouterHeartbeatRequest hbRequest = RouterHeartbeatRequest.newInstance(
- RouterState.newInstance("test", 0, RouterServiceState.UNINITIALIZED));
- verifyException(routerStore, "routerHeartbeat",
- StateStoreUnavailableException.class,
- new Class[] {RouterHeartbeatRequest.class},
- new Object[] {hbRequest});
- }
-
- //
- // Router
- //
- @Test
- public void testUpdateRouterStatus()
- throws IllegalStateException, IOException {
-
- long dateStarted = Time.now();
- String address = "testaddress";
-
- // Set
- RouterHeartbeatRequest request = RouterHeartbeatRequest.newInstance(
- RouterState.newInstance(
- address, dateStarted, RouterServiceState.RUNNING));
- assertTrue(routerStore.routerHeartbeat(request).getStatus());
-
- // Verify
- GetRouterRegistrationRequest getRequest =
- GetRouterRegistrationRequest.newInstance(address);
- RouterState record =
- routerStore.getRouterRegistration(getRequest).getRouter();
- assertNotNull(record);
- assertEquals(RouterServiceState.RUNNING, record.getStatus());
- assertEquals(address, record.getAddress());
- assertEquals(FederationUtil.getCompileInfo(), record.getCompileInfo());
- // Build version may vary a bit
- assertFalse(record.getVersion().isEmpty());
- }
-
- @Test
- public void testRouterStateExpired()
- throws IOException, InterruptedException {
-
- long dateStarted = Time.now();
- String address = "testaddress";
-
- RouterHeartbeatRequest request = RouterHeartbeatRequest.newInstance(
- RouterState.newInstance(
- address, dateStarted, RouterServiceState.RUNNING));
- // Set
- assertTrue(routerStore.routerHeartbeat(request).getStatus());
-
- // Verify
- GetRouterRegistrationRequest getRequest =
- GetRouterRegistrationRequest.newInstance(address);
- RouterState record =
- routerStore.getRouterRegistration(getRequest).getRouter();
- assertNotNull(record);
-
- // Wait past expiration (set to 5 sec in config)
- Thread.sleep(6000);
-
- // Verify expired
- RouterState r = routerStore.getRouterRegistration(getRequest).getRouter();
- assertEquals(RouterServiceState.EXPIRED, r.getStatus());
-
- // Heartbeat again and this shouldn't be EXPIRED anymore
- assertTrue(routerStore.routerHeartbeat(request).getStatus());
- r = routerStore.getRouterRegistration(getRequest).getRouter();
- assertEquals(RouterServiceState.RUNNING, r.getStatus());
- }
-
- @Test
- public void testGetAllRouterStates()
- throws StateStoreUnavailableException, IOException {
-
- // Set 2 entries
- RouterHeartbeatRequest heartbeatRequest1 =
- RouterHeartbeatRequest.newInstance(
- RouterState.newInstance(
- "testaddress1", Time.now(), RouterServiceState.RUNNING));
- assertTrue(routerStore.routerHeartbeat(heartbeatRequest1).getStatus());
-
- RouterHeartbeatRequest heartbeatRequest2 =
- RouterHeartbeatRequest.newInstance(
- RouterState.newInstance(
- "testaddress2", Time.now(), RouterServiceState.RUNNING));
- assertTrue(routerStore.routerHeartbeat(heartbeatRequest2).getStatus());
-
- // Verify
- routerStore.loadCache(true);
- GetRouterRegistrationsRequest request =
- GetRouterRegistrationsRequest.newInstance();
- List<RouterState> entries =
- routerStore.getRouterRegistrations(request).getRouters();
- assertEquals(2, entries.size());
- Collections.sort(entries);
- assertEquals("testaddress1", entries.get(0).getAddress());
- assertEquals("testaddress2", entries.get(1).getAddress());
- assertEquals(RouterServiceState.RUNNING, entries.get(0).getStatus());
- assertEquals(RouterServiceState.RUNNING, entries.get(1).getStatus());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
deleted file mode 100644
index fd29e37..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
+++ /dev/null
@@ -1,613 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.driver;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.lang.reflect.Method;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Random;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.server.federation.metrics.StateStoreMetrics;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
-import org.apache.hadoop.hdfs.server.federation.router.RouterServiceState;
-import org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
-import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
-import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
-import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
-import org.apache.hadoop.hdfs.server.federation.store.records.Query;
-import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
-import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
-import org.apache.hadoop.hdfs.server.federation.store.records.StateStoreVersion;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Base tests for the driver. The particular implementations will use this to
- * test their functionality.
- */
-public class TestStateStoreDriverBase {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(TestStateStoreDriverBase.class);
-
- private static StateStoreService stateStore;
- private static Configuration conf;
-
- private static final Random RANDOM = new Random();
-
-
- /**
- * Get the State Store driver.
- * @return State Store driver.
- */
- protected StateStoreDriver getStateStoreDriver() {
- return stateStore.getDriver();
- }
-
- @After
- public void cleanMetrics() {
- if (stateStore != null) {
- StateStoreMetrics metrics = stateStore.getMetrics();
- metrics.reset();
- }
- }
-
- @AfterClass
- public static void tearDownCluster() {
- if (stateStore != null) {
- stateStore.stop();
- }
- }
-
- /**
- * Get a new State Store using this configuration.
- *
- * @param config Configuration for the State Store.
- * @throws Exception If we cannot get the State Store.
- */
- public static void getStateStore(Configuration config) throws Exception {
- conf = config;
- stateStore = FederationStateStoreTestUtils.newStateStore(conf);
- }
-
- private String generateRandomString() {
- String randomString = "randomString-" + RANDOM.nextInt();
- return randomString;
- }
-
- private long generateRandomLong() {
- return RANDOM.nextLong();
- }
-
- @SuppressWarnings("rawtypes")
- private <T extends Enum> T generateRandomEnum(Class<T> enumClass) {
- int x = RANDOM.nextInt(enumClass.getEnumConstants().length);
- T data = enumClass.getEnumConstants()[x];
- return data;
- }
-
- @SuppressWarnings("unchecked")
- private <T extends BaseRecord> T generateFakeRecord(Class<T> recordClass)
- throws IllegalArgumentException, IllegalAccessException, IOException {
-
- if (recordClass == MembershipState.class) {
- return (T) MembershipState.newInstance(generateRandomString(),
- generateRandomString(), generateRandomString(),
- generateRandomString(), generateRandomString(),
- generateRandomString(), generateRandomString(),
- generateRandomString(), generateRandomString(),
- generateRandomEnum(FederationNamenodeServiceState.class), false);
- } else if (recordClass == MountTable.class) {
- String src = "/" + generateRandomString();
- Map<String, String> destMap = Collections.singletonMap(
- generateRandomString(), "/" + generateRandomString());
- return (T) MountTable.newInstance(src, destMap);
- } else if (recordClass == RouterState.class) {
- RouterState routerState = RouterState.newInstance(generateRandomString(),
- generateRandomLong(), generateRandomEnum(RouterServiceState.class));
- StateStoreVersion version = generateFakeRecord(StateStoreVersion.class);
- routerState.setStateStoreVersion(version);
- return (T) routerState;
- }
-
- return null;
- }
-
- /**
- * Validate if a record is the same.
- *
- * @param original Original record.
- * @param committed Committed record.
- * @param assertEquals Assert if the records are equal or just return.
- * @return If the record is successfully validated.
- */
- private boolean validateRecord(
- BaseRecord original, BaseRecord committed, boolean assertEquals) {
-
- boolean ret = true;
-
- Map<String, Class<?>> fields = getFields(original);
- for (String key : fields.keySet()) {
- if (key.equals("dateModified") ||
- key.equals("dateCreated") ||
- key.equals("proto")) {
- // Fields are updated/set on commit and fetch and may not match
- // the fields that are initialized in a non-committed object.
- continue;
- }
- Object data1 = getField(original, key);
- Object data2 = getField(committed, key);
- if (assertEquals) {
- assertEquals("Field " + key + " does not match", data1, data2);
- } else if (!data1.equals(data2)) {
- ret = false;
- }
- }
-
- long now = stateStore.getDriver().getTime();
- assertTrue(
- committed.getDateCreated() <= now && committed.getDateCreated() > 0);
- assertTrue(committed.getDateModified() >= committed.getDateCreated());
-
- return ret;
- }
-
- public static void removeAll(StateStoreDriver driver) throws IOException {
- driver.removeAll(MembershipState.class);
- driver.removeAll(MountTable.class);
- }
-
- public <T extends BaseRecord> void testInsert(
- StateStoreDriver driver, Class<T> recordClass)
- throws IllegalArgumentException, IllegalAccessException, IOException {
-
- assertTrue(driver.removeAll(recordClass));
- QueryResult<T> queryResult0 = driver.get(recordClass);
- List<T> records0 = queryResult0.getRecords();
- assertTrue(records0.isEmpty());
-
- // Insert single
- BaseRecord record = generateFakeRecord(recordClass);
- driver.put(record, true, false);
-
- // Verify
- QueryResult<T> queryResult1 = driver.get(recordClass);
- List<T> records1 = queryResult1.getRecords();
- assertEquals(1, records1.size());
- T record0 = records1.get(0);
- validateRecord(record, record0, true);
-
- // Insert multiple
- List<T> insertList = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- T newRecord = generateFakeRecord(recordClass);
- insertList.add(newRecord);
- }
- driver.putAll(insertList, true, false);
-
- // Verify
- QueryResult<T> queryResult2 = driver.get(recordClass);
- List<T> records2 = queryResult2.getRecords();
- assertEquals(11, records2.size());
- }
-
- public <T extends BaseRecord> void testFetchErrors(StateStoreDriver driver,
- Class<T> clazz) throws IllegalAccessException, IOException {
-
- // Fetch empty list
- driver.removeAll(clazz);
- QueryResult<T> result0 = driver.get(clazz);
- assertNotNull(result0);
- List<T> records0 = result0.getRecords();
- assertEquals(records0.size(), 0);
-
- // Insert single
- BaseRecord record = generateFakeRecord(clazz);
- assertTrue(driver.put(record, true, false));
-
- // Verify
- QueryResult<T> result1 = driver.get(clazz);
- List<T> records1 = result1.getRecords();
- assertEquals(1, records1.size());
- validateRecord(record, records1.get(0), true);
-
- // Test fetch single object with a bad query
- final T fakeRecord = generateFakeRecord(clazz);
- final Query<T> query = new Query<T>(fakeRecord);
- T getRecord = driver.get(clazz, query);
- assertNull(getRecord);
-
- // Test fetch multiple objects does not exist returns empty list
- assertEquals(driver.getMultiple(clazz, query).size(), 0);
- }
-
- public <T extends BaseRecord> void testPut(
- StateStoreDriver driver, Class<T> clazz)
- throws IllegalArgumentException, ReflectiveOperationException,
- IOException, SecurityException {
-
- driver.removeAll(clazz);
- QueryResult<T> records = driver.get(clazz);
- assertTrue(records.getRecords().isEmpty());
-
- // Insert multiple
- List<T> insertList = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- T newRecord = generateFakeRecord(clazz);
- insertList.add(newRecord);
- }
-
- // Verify
- assertTrue(driver.putAll(insertList, false, true));
- records = driver.get(clazz);
- assertEquals(records.getRecords().size(), 10);
-
- // Generate a new record with the same PK fields as an existing record
- BaseRecord updatedRecord = generateFakeRecord(clazz);
- BaseRecord existingRecord = records.getRecords().get(0);
- Map<String, String> primaryKeys = existingRecord.getPrimaryKeys();
- for (Entry<String, String> entry : primaryKeys.entrySet()) {
- String key = entry.getKey();
- String value = entry.getValue();
- Class<?> fieldType = getFieldType(existingRecord, key);
- Object field = fromString(value, fieldType);
- assertTrue(setField(updatedRecord, key, field));
- }
-
- // Attempt an update of an existing entry, but it is not allowed.
- assertFalse(driver.put(updatedRecord, false, true));
-
- // Verify no update occurred, all original records are unchanged
- QueryResult<T> newRecords = driver.get(clazz);
- assertTrue(newRecords.getRecords().size() == 10);
- assertEquals("A single entry was improperly updated in the store", 10,
- countMatchingEntries(records.getRecords(), newRecords.getRecords()));
-
- // Update the entry (allowing updates)
- assertTrue(driver.put(updatedRecord, true, false));
-
- // Verify that one entry no longer matches the original set
- newRecords = driver.get(clazz);
- assertEquals(10, newRecords.getRecords().size());
- assertEquals(
- "Record of type " + clazz + " not updated in the store", 9,
- countMatchingEntries(records.getRecords(), newRecords.getRecords()));
- }
-
- private int countMatchingEntries(
- Collection<? extends BaseRecord> committedList,
- Collection<? extends BaseRecord> matchList) {
-
- int matchingCount = 0;
- for (BaseRecord committed : committedList) {
- for (BaseRecord match : matchList) {
- try {
- if (match.getPrimaryKey().equals(committed.getPrimaryKey())) {
- if (validateRecord(match, committed, false)) {
- matchingCount++;
- }
- break;
- }
- } catch (Exception ex) {
- }
- }
- }
- return matchingCount;
- }
-
- public <T extends BaseRecord> void testRemove(
- StateStoreDriver driver, Class<T> clazz)
- throws IllegalArgumentException, IllegalAccessException, IOException {
-
- // Remove all
- assertTrue(driver.removeAll(clazz));
- QueryResult<T> records = driver.get(clazz);
- assertTrue(records.getRecords().isEmpty());
-
- // Insert multiple
- List<T> insertList = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- T newRecord = generateFakeRecord(clazz);
- insertList.add(newRecord);
- }
-
- // Verify
- assertTrue(driver.putAll(insertList, false, true));
- records = driver.get(clazz);
- assertEquals(records.getRecords().size(), 10);
-
- // Remove Single
- assertTrue(driver.remove(records.getRecords().get(0)));
-
- // Verify
- records = driver.get(clazz);
- assertEquals(records.getRecords().size(), 9);
-
- // Remove with filter
- final T firstRecord = records.getRecords().get(0);
- final Query<T> query0 = new Query<T>(firstRecord);
- assertTrue(driver.remove(clazz, query0) > 0);
-
- final T secondRecord = records.getRecords().get(1);
- final Query<T> query1 = new Query<T>(secondRecord);
- assertTrue(driver.remove(clazz, query1) > 0);
-
- // Verify
- records = driver.get(clazz);
- assertEquals(records.getRecords().size(), 7);
-
- // Remove all
- assertTrue(driver.removeAll(clazz));
-
- // Verify
- records = driver.get(clazz);
- assertTrue(records.getRecords().isEmpty());
- }
-
- public void testInsert(StateStoreDriver driver)
- throws IllegalArgumentException, IllegalAccessException, IOException {
- testInsert(driver, MembershipState.class);
- testInsert(driver, MountTable.class);
- }
-
- public void testPut(StateStoreDriver driver)
- throws IllegalArgumentException, ReflectiveOperationException,
- IOException, SecurityException {
- testPut(driver, MembershipState.class);
- testPut(driver, MountTable.class);
- }
-
- public void testRemove(StateStoreDriver driver)
- throws IllegalArgumentException, IllegalAccessException, IOException {
- testRemove(driver, MembershipState.class);
- testRemove(driver, MountTable.class);
- }
-
- public void testFetchErrors(StateStoreDriver driver)
- throws IllegalArgumentException, IllegalAccessException, IOException {
- testFetchErrors(driver, MembershipState.class);
- testFetchErrors(driver, MountTable.class);
- }
-
- public void testMetrics(StateStoreDriver driver)
- throws IOException, IllegalArgumentException, IllegalAccessException {
-
- MountTable insertRecord =
- this.generateFakeRecord(MountTable.class);
-
- // Put single
- StateStoreMetrics metrics = stateStore.getMetrics();
- assertEquals(0, metrics.getWriteOps());
- driver.put(insertRecord, true, false);
- assertEquals(1, metrics.getWriteOps());
-
- // Put multiple
- metrics.reset();
- assertEquals(0, metrics.getWriteOps());
- driver.put(insertRecord, true, false);
- assertEquals(1, metrics.getWriteOps());
-
- // Get Single
- metrics.reset();
- assertEquals(0, metrics.getReadOps());
-
- final String querySourcePath = insertRecord.getSourcePath();
- MountTable partial = MountTable.newInstance();
- partial.setSourcePath(querySourcePath);
- final Query<MountTable> query = new Query<>(partial);
- driver.get(MountTable.class, query);
- assertEquals(1, metrics.getReadOps());
-
- // GetAll
- metrics.reset();
- assertEquals(0, metrics.getReadOps());
- driver.get(MountTable.class);
- assertEquals(1, metrics.getReadOps());
-
- // GetMultiple
- metrics.reset();
- assertEquals(0, metrics.getReadOps());
- driver.getMultiple(MountTable.class, query);
- assertEquals(1, metrics.getReadOps());
-
- // Insert fails
- metrics.reset();
- assertEquals(0, metrics.getFailureOps());
- driver.put(insertRecord, false, true);
- assertEquals(1, metrics.getFailureOps());
-
- // Remove single
- metrics.reset();
- assertEquals(0, metrics.getRemoveOps());
- driver.remove(insertRecord);
- assertEquals(1, metrics.getRemoveOps());
-
- // Remove multiple
- metrics.reset();
- driver.put(insertRecord, true, false);
- assertEquals(0, metrics.getRemoveOps());
- driver.remove(MountTable.class, query);
- assertEquals(1, metrics.getRemoveOps());
-
- // Remove all
- metrics.reset();
- driver.put(insertRecord, true, false);
- assertEquals(0, metrics.getRemoveOps());
- driver.removeAll(MountTable.class);
- assertEquals(1, metrics.getRemoveOps());
- }
-
- /**
- * Sets the value of a field on the object.
- *
- * @param fieldName The string name of the field.
- * @param data The data to pass to the field's setter.
- *
- * @return True if successful, fails if failed.
- */
- private static boolean setField(
- BaseRecord record, String fieldName, Object data) {
-
- Method m = locateSetter(record, fieldName);
- if (m != null) {
- try {
- m.invoke(record, data);
- } catch (Exception e) {
- LOG.error("Cannot set field " + fieldName + " on object "
- + record.getClass().getName() + " to data " + data + " of type "
- + data.getClass(), e);
- return false;
- }
- }
- return true;
- }
-
- /**
- * Finds the appropriate setter for a field name.
- *
- * @param fieldName The legacy name of the field.
- * @return The matching setter or null if not found.
- */
- private static Method locateSetter(BaseRecord record, String fieldName) {
- for (Method m : record.getClass().getMethods()) {
- if (m.getName().equalsIgnoreCase("set" + fieldName)) {
- return m;
- }
- }
- return null;
- }
-
- /**
- * Returns all serializable fields in the object.
- *
- * @return Map with the fields.
- */
- private static Map<String, Class<?>> getFields(BaseRecord record) {
- Map<String, Class<?>> getters = new HashMap<>();
- for (Method m : record.getClass().getDeclaredMethods()) {
- if (m.getName().startsWith("get")) {
- try {
- Class<?> type = m.getReturnType();
- char[] c = m.getName().substring(3).toCharArray();
- c[0] = Character.toLowerCase(c[0]);
- String key = new String(c);
- getters.put(key, type);
- } catch (Exception e) {
- LOG.error("Cannot execute getter " + m.getName()
- + " on object " + record);
- }
- }
- }
- return getters;
- }
-
- /**
- * Get the type of a field.
- *
- * @param fieldName
- * @return Field type
- */
- private static Class<?> getFieldType(BaseRecord record, String fieldName) {
- Method m = locateGetter(record, fieldName);
- return m.getReturnType();
- }
-
- /**
- * Fetches the value for a field name.
- *
- * @param fieldName the legacy name of the field.
- * @return The field data or null if not found.
- */
- private static Object getField(BaseRecord record, String fieldName) {
- Object result = null;
- Method m = locateGetter(record, fieldName);
- if (m != null) {
- try {
- result = m.invoke(record);
- } catch (Exception e) {
- LOG.error("Cannot get field " + fieldName + " on object " + record);
- }
- }
- return result;
- }
-
- /**
- * Finds the appropriate getter for a field name.
- *
- * @param fieldName The legacy name of the field.
- * @return The matching getter or null if not found.
- */
- private static Method locateGetter(BaseRecord record, String fieldName) {
- for (Method m : record.getClass().getMethods()) {
- if (m.getName().equalsIgnoreCase("get" + fieldName)) {
- return m;
- }
- }
- return null;
- }
-
- /**
- * Expands a data object from the store into an record object. Default store
- * data type is a String. Override if additional serialization is required.
- *
- * @param data Object containing the serialized data. Only string is
- * supported.
- * @param clazz Target object class to hold the deserialized data.
- * @return An instance of the target data object initialized with the
- * deserialized data.
- */
- @Deprecated
- @SuppressWarnings({ "unchecked", "rawtypes" })
- private static <T> T fromString(String data, Class<T> clazz) {
-
- if (data.equals("null")) {
- return null;
- } else if (clazz == String.class) {
- return (T) data;
- } else if (clazz == Long.class || clazz == long.class) {
- return (T) Long.valueOf(data);
- } else if (clazz == Integer.class || clazz == int.class) {
- return (T) Integer.valueOf(data);
- } else if (clazz == Double.class || clazz == double.class) {
- return (T) Double.valueOf(data);
- } else if (clazz == Float.class || clazz == float.class) {
- return (T) Float.valueOf(data);
- } else if (clazz == Boolean.class || clazz == boolean.class) {
- return (T) Boolean.valueOf(data);
- } else if (clazz.isEnum()) {
- return (T) Enum.valueOf((Class<Enum>) clazz, data);
- }
- return null;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFile.java
deleted file mode 100644
index a8a9020..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFile.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.driver;
-
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.getStateStoreConfiguration;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- * Test the FileSystem (e.g., HDFS) implementation of the State Store driver.
- */
-public class TestStateStoreFile extends TestStateStoreDriverBase {
-
- @BeforeClass
- public static void setupCluster() throws Exception {
- Configuration conf = getStateStoreConfiguration(StateStoreFileImpl.class);
- getStateStore(conf);
- }
-
- @Before
- public void startup() throws IOException {
- removeAll(getStateStoreDriver());
- }
-
- @Test
- public void testInsert()
- throws IllegalArgumentException, IllegalAccessException, IOException {
- testInsert(getStateStoreDriver());
- }
-
- @Test
- public void testUpdate()
- throws IllegalArgumentException, ReflectiveOperationException,
- IOException, SecurityException {
- testPut(getStateStoreDriver());
- }
-
- @Test
- public void testDelete()
- throws IllegalArgumentException, IllegalAccessException, IOException {
- testRemove(getStateStoreDriver());
- }
-
- @Test
- public void testFetchErrors()
- throws IllegalArgumentException, IllegalAccessException, IOException {
- testFetchErrors(getStateStoreDriver());
- }
-
- @Test
- public void testMetrics()
- throws IllegalArgumentException, IllegalAccessException, IOException {
- testMetrics(getStateStoreDriver());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileBase.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileBase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileBase.java
deleted file mode 100644
index 9adfe33..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileBase.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.driver;
-
-import static org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileBaseImpl.isOldTempRecord;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.util.Time;
-import org.junit.Test;
-
-/**
- * Tests for the State Store file based implementation.
- */
-public class TestStateStoreFileBase {
-
- @Test
- public void testTempOld() {
- assertFalse(isOldTempRecord("test.txt"));
- assertFalse(isOldTempRecord("testfolder/test.txt"));
-
- long tnow = Time.now();
- String tmpFile1 = "test." + tnow + ".tmp";
- assertFalse(isOldTempRecord(tmpFile1));
-
- long told = Time.now() - TimeUnit.MINUTES.toMillis(1);
- String tmpFile2 = "test." + told + ".tmp";
- assertTrue(isOldTempRecord(tmpFile2));
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileSystem.java
deleted file mode 100644
index 8c4b188..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileSystem.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.driver;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils;
-import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileSystemImpl;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- * Test the FileSystem (e.g., HDFS) implementation of the State Store driver.
- */
-public class TestStateStoreFileSystem extends TestStateStoreDriverBase {
-
- private static MiniDFSCluster dfsCluster;
-
- @BeforeClass
- public static void setupCluster() throws Exception {
- Configuration conf = FederationStateStoreTestUtils
- .getStateStoreConfiguration(StateStoreFileSystemImpl.class);
- conf.set(StateStoreFileSystemImpl.FEDERATION_STORE_FS_PATH,
- "/hdfs-federation/");
-
- // Create HDFS cluster to back the state tore
- MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
- builder.numDataNodes(1);
- dfsCluster = builder.build();
- dfsCluster.waitClusterUp();
- getStateStore(conf);
- }
-
- @AfterClass
- public static void tearDownCluster() {
- if (dfsCluster != null) {
- dfsCluster.shutdown();
- }
- }
-
- @Before
- public void startup() throws IOException {
- removeAll(getStateStoreDriver());
- }
-
- @Test
- public void testInsert()
- throws IllegalArgumentException, IllegalAccessException, IOException {
- testInsert(getStateStoreDriver());
- }
-
- @Test
- public void testUpdate() throws IllegalArgumentException, IOException,
- SecurityException, ReflectiveOperationException {
- testPut(getStateStoreDriver());
- }
-
- @Test
- public void testDelete()
- throws IllegalArgumentException, IllegalAccessException, IOException {
- testRemove(getStateStoreDriver());
- }
-
- @Test
- public void testFetchErrors()
- throws IllegalArgumentException, IllegalAccessException, IOException {
- testFetchErrors(getStateStoreDriver());
- }
-
- @Test
- public void testMetrics()
- throws IllegalArgumentException, IllegalAccessException, IOException {
- testMetrics(getStateStoreDriver());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreZK.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreZK.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreZK.java
deleted file mode 100644
index 36353ff..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreZK.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.driver;
-
-import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.getStateStoreConfiguration;
-
-import java.io.IOException;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.retry.RetryNTimes;
-import org.apache.curator.test.TestingServer;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- * Test the ZooKeeper implementation of the State Store driver.
- */
-public class TestStateStoreZK extends TestStateStoreDriverBase {
-
- private static TestingServer curatorTestingServer;
- private static CuratorFramework curatorFramework;
-
- @BeforeClass
- public static void setupCluster() throws Exception {
- curatorTestingServer = new TestingServer();
- curatorTestingServer.start();
- String connectString = curatorTestingServer.getConnectString();
- curatorFramework = CuratorFrameworkFactory.builder()
- .connectString(connectString)
- .retryPolicy(new RetryNTimes(100, 100))
- .build();
- curatorFramework.start();
-
- // Create the ZK State Store
- Configuration conf =
- getStateStoreConfiguration(StateStoreZooKeeperImpl.class);
- conf.set(CommonConfigurationKeys.ZK_ADDRESS, connectString);
- // Disable auto-repair of connection
- conf.setLong(DFSConfigKeys.FEDERATION_STORE_CONNECTION_TEST_MS,
- TimeUnit.HOURS.toMillis(1));
- getStateStore(conf);
- }
-
- @AfterClass
- public static void tearDownCluster() {
- curatorFramework.close();
- try {
- curatorTestingServer.stop();
- } catch (IOException e) {
- }
- }
-
- @Before
- public void startup() throws IOException {
- removeAll(getStateStoreDriver());
- }
-
- @Test
- public void testInsert()
- throws IllegalArgumentException, IllegalAccessException, IOException {
- testInsert(getStateStoreDriver());
- }
-
- @Test
- public void testUpdate()
- throws IllegalArgumentException, ReflectiveOperationException,
- IOException, SecurityException {
- testPut(getStateStoreDriver());
- }
-
- @Test
- public void testDelete()
- throws IllegalArgumentException, IllegalAccessException, IOException {
- testRemove(getStateStoreDriver());
- }
-
- @Test
- public void testFetchErrors()
- throws IllegalArgumentException, IllegalAccessException, IOException {
- testFetchErrors(getStateStoreDriver());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMembershipState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMembershipState.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMembershipState.java
deleted file mode 100644
index d922414..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMembershipState.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.records;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
-import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-import org.junit.Test;
-
-/**
- * Test the Membership State records.
- */
-public class TestMembershipState {
-
- private static final String ROUTER = "router";
- private static final String NAMESERVICE = "nameservice";
- private static final String NAMENODE = "namenode";
- private static final String CLUSTER_ID = "cluster";
- private static final String BLOCKPOOL_ID = "blockpool";
- private static final String RPC_ADDRESS = "rpcaddress";
- private static final String SERVICE_ADDRESS = "serviceaddress";
- private static final String LIFELINE_ADDRESS = "lifelineaddress";
- private static final String WEB_ADDRESS = "webaddress";
- private static final boolean SAFE_MODE = false;
-
- private static final long DATE_CREATED = 100;
- private static final long DATE_MODIFIED = 200;
-
- private static final long NUM_BLOCKS = 300;
- private static final long NUM_FILES = 400;
- private static final int NUM_DEAD = 500;
- private static final int NUM_ACTIVE = 600;
- private static final int NUM_DECOM = 700;
- private static final int NUM_DECOM_ACTIVE = 800;
- private static final int NUM_DECOM_DEAD = 900;
- private static final long NUM_BLOCK_MISSING = 1000;
-
- private static final long TOTAL_SPACE = 1100;
- private static final long AVAILABLE_SPACE = 1200;
-
- private static final FederationNamenodeServiceState STATE =
- FederationNamenodeServiceState.ACTIVE;
-
- private MembershipState createRecord() throws IOException {
-
- MembershipState record = MembershipState.newInstance(
- ROUTER, NAMESERVICE, NAMENODE, CLUSTER_ID,
- BLOCKPOOL_ID, RPC_ADDRESS, SERVICE_ADDRESS, LIFELINE_ADDRESS,
- WEB_ADDRESS, STATE, SAFE_MODE);
- record.setDateCreated(DATE_CREATED);
- record.setDateModified(DATE_MODIFIED);
-
- MembershipStats stats = MembershipStats.newInstance();
- stats.setNumOfBlocks(NUM_BLOCKS);
- stats.setNumOfFiles(NUM_FILES);
- stats.setNumOfActiveDatanodes(NUM_ACTIVE);
- stats.setNumOfDeadDatanodes(NUM_DEAD);
- stats.setNumOfDecommissioningDatanodes(NUM_DECOM);
- stats.setNumOfDecomActiveDatanodes(NUM_DECOM_ACTIVE);
- stats.setNumOfDecomDeadDatanodes(NUM_DECOM_DEAD);
- stats.setNumOfBlocksMissing(NUM_BLOCK_MISSING);
- stats.setTotalSpace(TOTAL_SPACE);
- stats.setAvailableSpace(AVAILABLE_SPACE);
- record.setStats(stats);
- return record;
- }
-
- private void validateRecord(MembershipState record) throws IOException {
-
- assertEquals(ROUTER, record.getRouterId());
- assertEquals(NAMESERVICE, record.getNameserviceId());
- assertEquals(CLUSTER_ID, record.getClusterId());
- assertEquals(BLOCKPOOL_ID, record.getBlockPoolId());
- assertEquals(RPC_ADDRESS, record.getRpcAddress());
- assertEquals(WEB_ADDRESS, record.getWebAddress());
- assertEquals(STATE, record.getState());
- assertEquals(SAFE_MODE, record.getIsSafeMode());
- assertEquals(DATE_CREATED, record.getDateCreated());
- assertEquals(DATE_MODIFIED, record.getDateModified());
-
- MembershipStats stats = record.getStats();
- assertEquals(NUM_BLOCKS, stats.getNumOfBlocks());
- assertEquals(NUM_FILES, stats.getNumOfFiles());
- assertEquals(NUM_ACTIVE, stats.getNumOfActiveDatanodes());
- assertEquals(NUM_DEAD, stats.getNumOfDeadDatanodes());
- assertEquals(NUM_DECOM, stats.getNumOfDecommissioningDatanodes());
- assertEquals(NUM_DECOM_ACTIVE, stats.getNumOfDecomActiveDatanodes());
- assertEquals(NUM_DECOM_DEAD, stats.getNumOfDecomDeadDatanodes());
- assertEquals(TOTAL_SPACE, stats.getTotalSpace());
- assertEquals(AVAILABLE_SPACE, stats.getAvailableSpace());
- }
-
- @Test
- public void testGetterSetter() throws IOException {
- MembershipState record = createRecord();
- validateRecord(record);
- }
-
- @Test
- public void testSerialization() throws IOException {
-
- MembershipState record = createRecord();
-
- StateStoreSerializer serializer = StateStoreSerializer.getSerializer();
- String serializedString = serializer.serializeString(record);
- MembershipState newRecord =
- serializer.deserialize(serializedString, MembershipState.class);
-
- validateRecord(newRecord);
- }
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
deleted file mode 100644
index aaf2817..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
+++ /dev/null
@@ -1,362 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HEARTBEAT_INTERVAL_MS;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_HEARTBEAT_INTERVAL_MS_DEFAULT;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.URI;
-import java.util.Map;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ha.HAServiceProtocol;
-import org.apache.hadoop.ha.HAServiceStatus;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.NameNodeProxies;
-import org.apache.hadoop.hdfs.protocol.ClientProtocol;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
-import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport;
-import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
-import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
-import org.apache.hadoop.hdfs.tools.NNHAServiceTarget;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * The {@link Router} periodically checks the state of a Namenode (usually on
- * the same server) and reports their high availability (HA) state and
- * load/space status to the
- * {@link org.apache.hadoop.hdfs.server.federation.store.StateStoreService}
- * . Note that this is an optional role as a Router can be independent of any
- * subcluster.
- * <p>
- * For performance with Namenode HA, the Router uses the high availability state
- * information in the State Store to forward the request to the Namenode that is
- * most likely to be active.
- * <p>
- * Note that this service can be embedded into the Namenode itself to simplify
- * the operation.
- */
-public class NamenodeHeartbeatService extends PeriodicService {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(NamenodeHeartbeatService.class);
-
-
- /** Configuration for the heartbeat. */
- private Configuration conf;
-
- /** Router performing the heartbeating. */
- private final ActiveNamenodeResolver resolver;
-
- /** Interface to the tracked NN. */
- private final String nameserviceId;
- private final String namenodeId;
-
- /** Namenode HA target. */
- private NNHAServiceTarget localTarget;
- /** RPC address for the namenode. */
- private String rpcAddress;
- /** Service RPC address for the namenode. */
- private String serviceAddress;
- /** Service RPC address for the namenode. */
- private String lifelineAddress;
- /** HTTP address for the namenode. */
- private String webAddress;
-
- /**
- * Create a new Namenode status updater.
- * @param resolver Namenode resolver service to handle NN registration.
- * @param nsId Identifier of the nameservice.
- * @param nnId Identifier of the namenode in HA.
- */
- public NamenodeHeartbeatService(
- ActiveNamenodeResolver resolver, String nsId, String nnId) {
- super(NamenodeHeartbeatService.class.getSimpleName() +
- (nsId == null ? "" : " " + nsId) +
- (nnId == null ? "" : " " + nnId));
-
- this.resolver = resolver;
-
- this.nameserviceId = nsId;
- this.namenodeId = nnId;
-
- }
-
- @Override
- protected void serviceInit(Configuration configuration) throws Exception {
-
- this.conf = configuration;
-
- String nnDesc = nameserviceId;
- if (this.namenodeId != null && !this.namenodeId.isEmpty()) {
- this.localTarget = new NNHAServiceTarget(
- conf, nameserviceId, namenodeId);
- nnDesc += "-" + namenodeId;
- } else {
- this.localTarget = null;
- }
-
- // Get the RPC address for the clients to connect
- this.rpcAddress = getRpcAddress(conf, nameserviceId, namenodeId);
- LOG.info("{} RPC address: {}", nnDesc, rpcAddress);
-
- // Get the Service RPC address for monitoring
- this.serviceAddress =
- DFSUtil.getNamenodeServiceAddr(conf, nameserviceId, namenodeId);
- if (this.serviceAddress == null) {
- LOG.error("Cannot locate RPC service address for NN {}, " +
- "using RPC address {}", nnDesc, this.rpcAddress);
- this.serviceAddress = this.rpcAddress;
- }
- LOG.info("{} Service RPC address: {}", nnDesc, serviceAddress);
-
- // Get the Lifeline RPC address for faster monitoring
- this.lifelineAddress =
- DFSUtil.getNamenodeLifelineAddr(conf, nameserviceId, namenodeId);
- if (this.lifelineAddress == null) {
- this.lifelineAddress = this.serviceAddress;
- }
- LOG.info("{} Lifeline RPC address: {}", nnDesc, lifelineAddress);
-
- // Get the Web address for UI
- this.webAddress =
- DFSUtil.getNamenodeWebAddr(conf, nameserviceId, namenodeId);
- LOG.info("{} Web address: {}", nnDesc, webAddress);
-
- this.setIntervalMs(conf.getLong(
- DFS_ROUTER_HEARTBEAT_INTERVAL_MS,
- DFS_ROUTER_HEARTBEAT_INTERVAL_MS_DEFAULT));
-
-
- super.serviceInit(configuration);
- }
-
- @Override
- public void periodicInvoke() {
- updateState();
- }
-
- /**
- * Get the RPC address for a Namenode.
- * @param conf Configuration.
- * @param nsId Name service identifier.
- * @param nnId Name node identifier.
- * @return RPC address in format hostname:1234.
- */
- private static String getRpcAddress(
- Configuration conf, String nsId, String nnId) {
-
- // Get it from the regular RPC setting
- String confKey = DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY;
- String ret = conf.get(confKey);
-
- if (nsId != null || nnId != null) {
- // Get if for the proper nameservice and namenode
- confKey = DFSUtil.addKeySuffixes(confKey, nsId, nnId);
- ret = conf.get(confKey);
-
- // If not available, get it from the map
- if (ret == null) {
- Map<String, InetSocketAddress> rpcAddresses =
- DFSUtil.getRpcAddressesForNameserviceId(conf, nsId, null);
- InetSocketAddress sockAddr = null;
- if (nnId != null) {
- sockAddr = rpcAddresses.get(nnId);
- } else if (rpcAddresses.size() == 1) {
- // Get the only namenode in the namespace
- sockAddr = rpcAddresses.values().iterator().next();
- }
- if (sockAddr != null) {
- InetAddress addr = sockAddr.getAddress();
- ret = addr.getHostName() + ":" + sockAddr.getPort();
- }
- }
- }
- return ret;
- }
-
- /**
- * Update the state of the Namenode.
- */
- private void updateState() {
- NamenodeStatusReport report = getNamenodeStatusReport();
- if (!report.registrationValid()) {
- // Not operational
- LOG.error("Namenode is not operational: {}", getNamenodeDesc());
- } else if (report.haStateValid()) {
- // block and HA status available
- LOG.debug("Received service state: {} from HA namenode: {}",
- report.getState(), getNamenodeDesc());
- } else if (localTarget == null) {
- // block info available, HA status not expected
- LOG.debug(
- "Reporting non-HA namenode as operational: " + getNamenodeDesc());
- } else {
- // block info available, HA status should be available, but was not
- // fetched do nothing and let the current state stand
- return;
- }
- try {
- if (!resolver.registerNamenode(report)) {
- LOG.warn("Cannot register namenode {}", report);
- }
- } catch (IOException e) {
- LOG.info("Cannot register namenode in the State Store");
- } catch (Exception ex) {
- LOG.error("Unhandled exception updating NN registration for {}",
- getNamenodeDesc(), ex);
- }
- }
-
- /**
- * Get the status report for the Namenode monitored by this heartbeater.
- * @return Namenode status report.
- */
- protected NamenodeStatusReport getNamenodeStatusReport() {
- NamenodeStatusReport report = new NamenodeStatusReport(nameserviceId,
- namenodeId, rpcAddress, serviceAddress, lifelineAddress, webAddress);
-
- try {
- LOG.debug("Probing NN at service address: {}", serviceAddress);
-
- URI serviceURI = new URI("hdfs://" + serviceAddress);
- // Read the filesystem info from RPC (required)
- NamenodeProtocol nn = NameNodeProxies
- .createProxy(this.conf, serviceURI, NamenodeProtocol.class)
- .getProxy();
-
- if (nn != null) {
- NamespaceInfo info = nn.versionRequest();
- if (info != null) {
- report.setNamespaceInfo(info);
- }
- }
- if (!report.registrationValid()) {
- return report;
- }
-
- // Check for safemode from the client protocol. Currently optional, but
- // should be required at some point for QoS
- try {
- ClientProtocol client = NameNodeProxies
- .createProxy(this.conf, serviceURI, ClientProtocol.class)
- .getProxy();
- if (client != null) {
- boolean isSafeMode = client.setSafeMode(
- SafeModeAction.SAFEMODE_GET, false);
- report.setSafeMode(isSafeMode);
- }
- } catch (Exception e) {
- LOG.error("Cannot fetch safemode state for {}", getNamenodeDesc(), e);
- }
-
- // Read the stats from JMX (optional)
- updateJMXParameters(webAddress, report);
-
- if (localTarget != null) {
- // Try to get the HA status
- try {
- // Determine if NN is active
- // TODO: dynamic timeout
- HAServiceProtocol haProtocol = localTarget.getProxy(conf, 30*1000);
- HAServiceStatus status = haProtocol.getServiceStatus();
- report.setHAServiceState(status.getState());
- } catch (Throwable e) {
- if (e.getMessage().startsWith("HA for namenode is not enabled")) {
- LOG.error("HA for {} is not enabled", getNamenodeDesc());
- localTarget = null;
- } else {
- // Failed to fetch HA status, ignoring failure
- LOG.error("Cannot fetch HA status for {}: {}",
- getNamenodeDesc(), e.getMessage(), e);
- }
- }
- }
- } catch(IOException e) {
- LOG.error("Cannot communicate with {}: {}",
- getNamenodeDesc(), e.getMessage());
- } catch(Throwable e) {
- // Generic error that we don't know about
- LOG.error("Unexpected exception while communicating with {}: {}",
- getNamenodeDesc(), e.getMessage(), e);
- }
- return report;
- }
-
- /**
- * Get the description of the Namenode to monitor.
- * @return Description of the Namenode to monitor.
- */
- public String getNamenodeDesc() {
- if (namenodeId != null && !namenodeId.isEmpty()) {
- return nameserviceId + "-" + namenodeId + ":" + serviceAddress;
- } else {
- return nameserviceId + ":" + serviceAddress;
- }
- }
-
- /**
- * Get the parameters for a Namenode from JMX and add them to the report.
- * @param address Web interface of the Namenode to monitor.
- * @param report Namenode status report to update with JMX data.
- */
- private void updateJMXParameters(
- String address, NamenodeStatusReport report) {
- try {
- // TODO part of this should be moved to its own utility
- String query = "Hadoop:service=NameNode,name=FSNamesystem*";
- JSONArray aux = FederationUtil.getJmx(query, address);
- if (aux != null) {
- for (int i = 0; i < aux.length(); i++) {
- JSONObject jsonObject = aux.getJSONObject(i);
- String name = jsonObject.getString("name");
- if (name.equals("Hadoop:service=NameNode,name=FSNamesystemState")) {
- report.setDatanodeInfo(
- jsonObject.getInt("NumLiveDataNodes"),
- jsonObject.getInt("NumDeadDataNodes"),
- jsonObject.getInt("NumDecommissioningDataNodes"),
- jsonObject.getInt("NumDecomLiveDataNodes"),
- jsonObject.getInt("NumDecomDeadDataNodes"));
- } else if (name.equals(
- "Hadoop:service=NameNode,name=FSNamesystem")) {
- report.setNamesystemInfo(
- jsonObject.getLong("CapacityRemaining"),
- jsonObject.getLong("CapacityTotal"),
- jsonObject.getLong("FilesTotal"),
- jsonObject.getLong("BlocksTotal"),
- jsonObject.getLong("MissingBlocks"),
- jsonObject.getLong("PendingReplicationBlocks"),
- jsonObject.getLong("UnderReplicatedBlocks"),
- jsonObject.getLong("PendingDeletionBlocks"),
- jsonObject.getLong("ProvidedCapacityTotal"));
- }
- }
- }
- } catch (Exception e) {
- LOG.error("Cannot get stat from {} using JMX", getNamenodeDesc(), e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java
deleted file mode 100644
index 5e12222..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.service.ServiceStateException;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-
-/**
- * Service to periodically execute a runnable.
- */
-public abstract class PeriodicService extends AbstractService {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(PeriodicService.class);
-
- /** Default interval in milliseconds for the periodic service. */
- private static final long DEFAULT_INTERVAL_MS = TimeUnit.MINUTES.toMillis(1);
-
-
- /** Interval for running the periodic service in milliseconds. */
- private long intervalMs;
- /** Name of the service. */
- private final String serviceName;
-
- /** Scheduler for the periodic service. */
- private final ScheduledExecutorService scheduler;
-
- /** If the service is running. */
- private volatile boolean isRunning = false;
-
- /** How many times we run. */
- private long runCount;
- /** How many errors we got. */
- private long errorCount;
- /** When was the last time we executed this service successfully. */
- private long lastRun;
-
- /**
- * Create a new periodic update service.
- *
- * @param name Name of the service.
- */
- public PeriodicService(String name) {
- this(name, DEFAULT_INTERVAL_MS);
- }
-
- /**
- * Create a new periodic update service.
- *
- * @param name Name of the service.
- * @param interval Interval for the periodic service in milliseconds.
- */
- public PeriodicService(String name, long interval) {
- super(name);
- this.serviceName = name;
- this.intervalMs = interval;
-
- ThreadFactory threadFactory = new ThreadFactoryBuilder()
- .setNameFormat(this.getName() + "-%d")
- .build();
- this.scheduler = Executors.newScheduledThreadPool(1, threadFactory);
- }
-
- /**
- * Set the interval for the periodic service.
- *
- * @param interval Interval in milliseconds.
- */
- protected void setIntervalMs(long interval) {
- if (getServiceState() == STATE.STARTED) {
- throw new ServiceStateException("Periodic service already started");
- } else {
- this.intervalMs = interval;
- }
- }
-
- /**
- * Get the interval for the periodic service.
- *
- * @return Interval in milliseconds.
- */
- protected long getIntervalMs() {
- return this.intervalMs;
- }
-
- /**
- * Get how many times we failed to run the periodic service.
- *
- * @return Times we failed to run the periodic service.
- */
- protected long getErrorCount() {
- return this.errorCount;
- }
-
- /**
- * Get how many times we run the periodic service.
- *
- * @return Times we run the periodic service.
- */
- protected long getRunCount() {
- return this.runCount;
- }
-
- /**
- * Get the last time the periodic service was executed.
- *
- * @return Last time the periodic service was executed.
- */
- protected long getLastUpdate() {
- return this.lastRun;
- }
-
- @Override
- protected void serviceStart() throws Exception {
- super.serviceStart();
- LOG.info("Starting periodic service {}", this.serviceName);
- startPeriodic();
- }
-
- @Override
- protected void serviceStop() throws Exception {
- stopPeriodic();
- LOG.info("Stopping periodic service {}", this.serviceName);
- super.serviceStop();
- }
-
- /**
- * Stop the periodic task.
- */
- protected synchronized void stopPeriodic() {
- if (this.isRunning) {
- LOG.info("{} is shutting down", this.serviceName);
- this.isRunning = false;
- this.scheduler.shutdownNow();
- }
- }
-
- /**
- * Start the periodic execution.
- */
- protected synchronized void startPeriodic() {
- stopPeriodic();
-
- // Create the runnable service
- Runnable updateRunnable = new Runnable() {
- @Override
- public void run() {
- LOG.debug("Running {} update task", serviceName);
- try {
- if (!isRunning) {
- return;
- }
- periodicInvoke();
- runCount++;
- lastRun = Time.now();
- } catch (Exception ex) {
- errorCount++;
- LOG.warn(serviceName + " service threw an exception", ex);
- }
- }
- };
-
- // Start the execution of the periodic service
- this.isRunning = true;
- this.scheduler.scheduleWithFixedDelay(
- updateRunnable, 0, this.intervalMs, TimeUnit.MILLISECONDS);
- }
-
- /**
- * Method that the service will run periodically.
- */
- protected abstract void periodicInvoke();
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java
deleted file mode 100644
index dbb6ffa..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.hadoop.fs.QuotaUsage;
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.hdfs.protocol.ClientProtocol;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
-import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Module that implements the quota relevant RPC calls
- * {@link ClientProtocol#setQuota(String, long, long, StorageType)}
- * and
- * {@link ClientProtocol#getQuotaUsage(String)}
- * in the {@link RouterRpcServer}.
- */
-public class Quota {
- private static final Logger LOG = LoggerFactory.getLogger(Quota.class);
-
- /** RPC server to receive client calls. */
- private final RouterRpcServer rpcServer;
- /** RPC clients to connect to the Namenodes. */
- private final RouterRpcClient rpcClient;
- /** Router used in RouterRpcServer. */
- private final Router router;
-
- public Quota(Router router, RouterRpcServer server) {
- this.router = router;
- this.rpcServer = server;
- this.rpcClient = server.getRPCClient();
- }
-
- /**
- * Set quota for the federation path.
- * @param path Federation path.
- * @param namespaceQuota Name space quota.
- * @param storagespaceQuota Storage space quota.
- * @param type StorageType that the space quota is intended to be set on.
- * @throws IOException
- */
- public void setQuota(String path, long namespaceQuota,
- long storagespaceQuota, StorageType type) throws IOException {
- rpcServer.checkOperation(OperationCategory.WRITE);
-
- // Set quota for current path and its children mount table path.
- final List<RemoteLocation> locations = getQuotaRemoteLocations(path);
- if (LOG.isDebugEnabled()) {
- for (RemoteLocation loc : locations) {
- LOG.debug("Set quota for path: nsId: {}, dest: {}.",
- loc.getNameserviceId(), loc.getDest());
- }
- }
-
- RemoteMethod method = new RemoteMethod("setQuota",
- new Class<?>[] {String.class, long.class, long.class,
- StorageType.class},
- new RemoteParam(), namespaceQuota, storagespaceQuota, type);
- rpcClient.invokeConcurrent(locations, method, false, false);
- }
-
- /**
- * Get quota usage for the federation path.
- * @param path Federation path.
- * @return Aggregated quota.
- * @throws IOException
- */
- public QuotaUsage getQuotaUsage(String path) throws IOException {
- final List<RemoteLocation> quotaLocs = getValidQuotaLocations(path);
- RemoteMethod method = new RemoteMethod("getQuotaUsage",
- new Class<?>[] {String.class}, new RemoteParam());
- Map<RemoteLocation, QuotaUsage> results = rpcClient.invokeConcurrent(
- quotaLocs, method, true, false, QuotaUsage.class);
-
- return aggregateQuota(results);
- }
-
- /**
- * Get valid quota remote locations used in {@link #getQuotaUsage(String)}.
- * Differentiate the method {@link #getQuotaRemoteLocations(String)}, this
- * method will do some additional filtering.
- * @param path Federation path.
- * @return List of valid quota remote locations.
- * @throws IOException
- */
- private List<RemoteLocation> getValidQuotaLocations(String path)
- throws IOException {
- final List<RemoteLocation> locations = getQuotaRemoteLocations(path);
-
- // NameService -> Locations
- Map<String, List<RemoteLocation>> validLocations = new HashMap<>();
- for (RemoteLocation loc : locations) {
- String nsId = loc.getNameserviceId();
- List<RemoteLocation> dests = validLocations.get(nsId);
- if (dests == null) {
- dests = new LinkedList<>();
- dests.add(loc);
- validLocations.put(nsId, dests);
- } else {
- // Ensure the paths in the same nameservice is different.
- // Don't include parent-child paths.
- boolean isChildPath = false;
- for (RemoteLocation d : dests) {
- if (loc.getDest().startsWith(d.getDest())) {
- isChildPath = true;
- break;
- }
- }
-
- if (!isChildPath) {
- dests.add(loc);
- }
- }
- }
-
- List<RemoteLocation> quotaLocs = new LinkedList<>();
- for (List<RemoteLocation> locs : validLocations.values()) {
- quotaLocs.addAll(locs);
- }
-
- return quotaLocs;
- }
-
- /**
- * Aggregate quota that queried from sub-clusters.
- * @param results Quota query result.
- * @return Aggregated Quota.
- */
- private QuotaUsage aggregateQuota(Map<RemoteLocation, QuotaUsage> results) {
- long nsCount = 0;
- long ssCount = 0;
- boolean hasQuotaUnSet = false;
-
- for (Map.Entry<RemoteLocation, QuotaUsage> entry : results.entrySet()) {
- RemoteLocation loc = entry.getKey();
- QuotaUsage usage = entry.getValue();
- if (usage != null) {
- // If quota is not set in real FileSystem, the usage
- // value will return -1.
- if (usage.getQuota() == -1 && usage.getSpaceQuota() == -1) {
- hasQuotaUnSet = true;
- }
-
- nsCount += usage.getFileAndDirectoryCount();
- ssCount += usage.getSpaceConsumed();
- LOG.debug(
- "Get quota usage for path: nsId: {}, dest: {},"
- + " nsCount: {}, ssCount: {}.",
- loc.getNameserviceId(), loc.getDest(),
- usage.getFileAndDirectoryCount(), usage.getSpaceConsumed());
- }
- }
-
- QuotaUsage.Builder builder = new QuotaUsage.Builder()
- .fileAndDirectoryCount(nsCount).spaceConsumed(ssCount);
- if (hasQuotaUnSet) {
- builder.quota(HdfsConstants.QUOTA_DONT_SET);
- }
-
- return builder.build();
- }
-
- /**
- * Get all quota remote locations across subclusters under given
- * federation path.
- * @param path Federation path.
- * @return List of quota remote locations.
- * @throws IOException
- */
- private List<RemoteLocation> getQuotaRemoteLocations(String path)
- throws IOException {
- List<RemoteLocation> locations = new LinkedList<>();
- RouterQuotaManager manager = this.router.getQuotaManager();
- if (manager != null) {
- Set<String> childrenPaths = manager.getPaths(path);
- for (String childPath : childrenPaths) {
- locations.addAll(rpcServer.getLocationsForPath(childPath, true));
- }
- }
-
- return locations;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
deleted file mode 100644
index a90c460..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-
-/**
- * Base class for objects that are unique to a namespace.
- */
-public abstract class RemoteLocationContext
- implements Comparable<RemoteLocationContext> {
-
- /**
- * Returns an identifier for a unique namespace.
- *
- * @return Namespace identifier.
- */
- public abstract String getNameserviceId();
-
- /**
- * Destination in this location. For example the path in a remote namespace.
- *
- * @return Destination in this location.
- */
- public abstract String getDest();
-
- @Override
- public int hashCode() {
- return new HashCodeBuilder(17, 31)
- .append(getNameserviceId())
- .append(getDest())
- .toHashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj instanceof RemoteLocationContext) {
- RemoteLocationContext other = (RemoteLocationContext) obj;
- return this.getNameserviceId().equals(other.getNameserviceId()) &&
- this.getDest().equals(other.getDest());
- }
- return false;
- }
-
- @Override
- public int compareTo(RemoteLocationContext info) {
- int ret = this.getNameserviceId().compareTo(info.getNameserviceId());
- if (ret == 0) {
- ret = this.getDest().compareTo(info.getDest());
- }
- return ret;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java
deleted file mode 100644
index cd57d45..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import java.io.IOException;
-import java.lang.reflect.Method;
-import java.util.Arrays;
-
-import org.apache.hadoop.hdfs.protocol.ClientProtocol;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Determines the remote client protocol method and the parameter list for a
- * specific location.
- */
-public class RemoteMethod {
-
- private static final Logger LOG = LoggerFactory.getLogger(RemoteMethod.class);
-
-
- /** List of parameters: static and dynamic values, matchings types. */
- private final Object[] params;
- /** List of method parameters types, matches parameters. */
- private final Class<?>[] types;
- /** String name of the ClientProtocol method. */
- private final String methodName;
-
- /**
- * Create a method with no parameters.
- *
- * @param method The string name of the ClientProtocol method.
- */
- public RemoteMethod(String method) {
- this.params = null;
- this.types = null;
- this.methodName = method;
- }
-
- /**
- * Creates a remote method generator.
- *
- * @param method The string name of the ClientProtocol method.
- * @param pTypes A list of types to use to locate the specific method.
- * @param pParams A list of parameters for the method. The order of the
- * parameter list must match the order and number of the types.
- * Parameters are grouped into 2 categories:
- * <ul>
- * <li>Static parameters that are immutable across locations.
- * <li>Dynamic parameters that are determined for each location by a
- * RemoteParam object. To specify a dynamic parameter, pass an
- * instance of RemoteParam in place of the parameter value.
- * </ul>
- * @throws IOException If the types and parameter lists are not valid.
- */
- public RemoteMethod(String method, Class<?>[] pTypes, Object... pParams)
- throws IOException {
-
- if (pParams.length != pTypes.length) {
- throw new IOException("Invalid parameters for method " + method);
- }
-
- this.params = pParams;
- this.types = pTypes;
- this.methodName = method;
- }
-
- /**
- * Get the represented java method.
- *
- * @return Method
- * @throws IOException If the method cannot be found.
- */
- public Method getMethod() throws IOException {
- try {
- if (types != null) {
- return ClientProtocol.class.getDeclaredMethod(methodName, types);
- } else {
- return ClientProtocol.class.getDeclaredMethod(methodName);
- }
- } catch (NoSuchMethodException e) {
- // Re-throw as an IOException
- LOG.error("Cannot get method {} with types {}",
- methodName, Arrays.toString(types), e);
- throw new IOException(e);
- } catch (SecurityException e) {
- LOG.error("Cannot access method {} with types {}",
- methodName, Arrays.toString(types), e);
- throw new IOException(e);
- }
- }
-
- /**
- * Get the calling types for this method.
- *
- * @return An array of calling types.
- */
- public Class<?>[] getTypes() {
- return this.types;
- }
-
- /**
- * Generate a list of parameters for this specific location using no context.
- *
- * @return A list of parameters for the method customized for the location.
- */
- public Object[] getParams() {
- return this.getParams(null);
- }
-
- /**
- * Get the name of the method.
- *
- * @return Name of the method.
- */
- public String getMethodName() {
- return this.methodName;
- }
-
- /**
- * Generate a list of parameters for this specific location. Parameters are
- * grouped into 2 categories:
- * <ul>
- * <li>Static parameters that are immutable across locations.
- * <li>Dynamic parameters that are determined for each location by a
- * RemoteParam object.
- * </ul>
- *
- * @param context The context identifying the location.
- * @return A list of parameters for the method customized for the location.
- */
- public Object[] getParams(RemoteLocationContext context) {
- if (this.params == null) {
- return new Object[] {};
- }
- Object[] objList = new Object[this.params.length];
- for (int i = 0; i < this.params.length; i++) {
- Object currentObj = this.params[i];
- if (currentObj instanceof RemoteParam) {
- // Map the parameter using the context
- RemoteParam paramGetter = (RemoteParam) currentObj;
- objList[i] = paramGetter.getParameterForContext(context);
- } else {
- objList[i] = currentObj;
- }
- }
- return objList;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteParam.java
deleted file mode 100644
index 8816ff6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteParam.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import java.util.Map;
-
-/**
- * A dynamically assignable parameter that is location-specific.
- * <p>
- * There are 2 ways this mapping is determined:
- * <ul>
- * <li>Default: Uses the RemoteLocationContext's destination
- * <li>Map: Uses the value of the RemoteLocationContext key provided in the
- * parameter map.
- * </ul>
- */
-public class RemoteParam {
-
- private final Map<? extends Object, ? extends Object> paramMap;
-
- /**
- * Constructs a default remote parameter. Always maps the value to the
- * destination of the provided RemoveLocationContext.
- */
- public RemoteParam() {
- this.paramMap = null;
- }
-
- /**
- * Constructs a map based remote parameter. Determines the value using the
- * provided RemoteLocationContext as a key into the map.
- *
- * @param map Map with RemoteLocationContext keys.
- */
- public RemoteParam(
- Map<? extends RemoteLocationContext, ? extends Object> map) {
- this.paramMap = map;
- }
-
- /**
- * Determine the appropriate value for this parameter based on the location.
- *
- * @param context Context identifying the location.
- * @return A parameter specific to this location.
- */
- public Object getParameterForContext(RemoteLocationContext context) {
- if (context == null) {
- return null;
- } else if (this.paramMap != null) {
- return this.paramMap.get(context);
- } else {
- // Default case
- return context.getDest();
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
deleted file mode 100644
index be9adc1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
+++ /dev/null
@@ -1,656 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import static org.apache.hadoop.hdfs.server.federation.router.FederationUtil.newActiveNamenodeResolver;
-import static org.apache.hadoop.hdfs.server.federation.router.FederationUtil.newFileSubclusterResolver;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.UnknownHostException;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.HAUtil;
-import org.apache.hadoop.hdfs.server.federation.metrics.FederationMetrics;
-import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
-import org.apache.hadoop.hdfs.server.federation.store.RouterStore;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.source.JvmMetrics;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.util.JvmPauseMonitor;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.annotations.VisibleForTesting;
-
-/**
- * Router that provides a unified view of multiple federated HDFS clusters. It
- * has two main roles: (1) federated interface and (2) NameNode heartbeat.
- * <p>
- * For the federated interface, the Router receives a client request, checks the
- * State Store for the correct subcluster, and forwards the request to the
- * active Namenode of that subcluster. The reply from the Namenode then flows in
- * the opposite direction. The Routers are stateless and can be behind a load
- * balancer. HDFS clients connect to the router using the same interfaces as are
- * used to communicate with a namenode, namely the ClientProtocol RPC interface
- * and the WebHdfs HTTP interface exposed by the router. {@link RouterRpcServer}
- * {@link RouterHttpServer}
- * <p>
- * For NameNode heartbeat, the Router periodically checks the state of a
- * NameNode (usually on the same server) and reports their high availability
- * (HA) state and load/space status to the State Store. Note that this is an
- * optional role as a Router can be independent of any subcluster.
- * {@link StateStoreService} {@link NamenodeHeartbeatService}
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class Router extends CompositeService {
-
- private static final Logger LOG = LoggerFactory.getLogger(Router.class);
-
-
- /** Configuration for the Router. */
- private Configuration conf;
-
- /** Router address/identifier. */
- private String routerId;
-
- /** RPC interface to the client. */
- private RouterRpcServer rpcServer;
- private InetSocketAddress rpcAddress;
-
- /** RPC interface for the admin. */
- private RouterAdminServer adminServer;
- private InetSocketAddress adminAddress;
-
- /** HTTP interface and web application. */
- private RouterHttpServer httpServer;
-
- /** Interface with the State Store. */
- private StateStoreService stateStore;
-
- /** Interface to map global name space to HDFS subcluster name spaces. */
- private FileSubclusterResolver subclusterResolver;
-
- /** Interface to identify the active NN for a nameservice or blockpool ID. */
- private ActiveNamenodeResolver namenodeResolver;
- /** Updates the namenode status in the namenode resolver. */
- private Collection<NamenodeHeartbeatService> namenodeHeartbeatServices;
-
- /** Router metrics. */
- private RouterMetricsService metrics;
-
- /** JVM pauses (GC and others). */
- private JvmPauseMonitor pauseMonitor;
-
- /** Quota usage update service. */
- private RouterQuotaUpdateService quotaUpdateService;
- /** Quota cache manager. */
- private RouterQuotaManager quotaManager;
-
- /** Manages the current state of the router. */
- private RouterStore routerStateManager;
- /** Heartbeat our run status to the router state manager. */
- private RouterHeartbeatService routerHeartbeatService;
- /** Enter/exit safemode. */
- private RouterSafemodeService safemodeService;
-
- /** The start time of the namesystem. */
- private final long startTime = Time.now();
-
- /** State of the Router. */
- private RouterServiceState state = RouterServiceState.UNINITIALIZED;
-
-
- /////////////////////////////////////////////////////////
- // Constructor
- /////////////////////////////////////////////////////////
-
- public Router() {
- super(Router.class.getName());
- }
-
- /////////////////////////////////////////////////////////
- // Service management
- /////////////////////////////////////////////////////////
-
- @Override
- protected void serviceInit(Configuration configuration) throws Exception {
- this.conf = configuration;
- updateRouterState(RouterServiceState.INITIALIZING);
-
- if (conf.getBoolean(
- DFSConfigKeys.DFS_ROUTER_STORE_ENABLE,
- DFSConfigKeys.DFS_ROUTER_STORE_ENABLE_DEFAULT)) {
- // Service that maintains the State Store connection
- this.stateStore = new StateStoreService();
- addService(this.stateStore);
- }
-
- // Resolver to track active NNs
- this.namenodeResolver = newActiveNamenodeResolver(
- this.conf, this.stateStore);
- if (this.namenodeResolver == null) {
- throw new IOException("Cannot find namenode resolver.");
- }
-
- // Lookup interface to map between the global and subcluster name spaces
- this.subclusterResolver = newFileSubclusterResolver(this.conf, this);
- if (this.subclusterResolver == null) {
- throw new IOException("Cannot find subcluster resolver");
- }
-
- if (conf.getBoolean(
- DFSConfigKeys.DFS_ROUTER_RPC_ENABLE,
- DFSConfigKeys.DFS_ROUTER_RPC_ENABLE_DEFAULT)) {
- // Create RPC server
- this.rpcServer = createRpcServer();
- addService(this.rpcServer);
- this.setRpcServerAddress(rpcServer.getRpcAddress());
- }
-
- if (conf.getBoolean(
- DFSConfigKeys.DFS_ROUTER_ADMIN_ENABLE,
- DFSConfigKeys.DFS_ROUTER_ADMIN_ENABLE_DEFAULT)) {
- // Create admin server
- this.adminServer = createAdminServer();
- addService(this.adminServer);
- }
-
- if (conf.getBoolean(
- DFSConfigKeys.DFS_ROUTER_HTTP_ENABLE,
- DFSConfigKeys.DFS_ROUTER_HTTP_ENABLE_DEFAULT)) {
- // Create HTTP server
- this.httpServer = createHttpServer();
- addService(this.httpServer);
- }
-
- if (conf.getBoolean(
- DFSConfigKeys.DFS_ROUTER_HEARTBEAT_ENABLE,
- DFSConfigKeys.DFS_ROUTER_HEARTBEAT_ENABLE_DEFAULT)) {
-
- // Create status updater for each monitored Namenode
- this.namenodeHeartbeatServices = createNamenodeHeartbeatServices();
- for (NamenodeHeartbeatService hearbeatService :
- this.namenodeHeartbeatServices) {
- addService(hearbeatService);
- }
-
- if (this.namenodeHeartbeatServices.isEmpty()) {
- LOG.error("Heartbeat is enabled but there are no namenodes to monitor");
- }
-
- // Periodically update the router state
- this.routerHeartbeatService = new RouterHeartbeatService(this);
- addService(this.routerHeartbeatService);
- }
-
- // Router metrics system
- if (conf.getBoolean(
- DFSConfigKeys.DFS_ROUTER_METRICS_ENABLE,
- DFSConfigKeys.DFS_ROUTER_METRICS_ENABLE_DEFAULT)) {
-
- DefaultMetricsSystem.initialize("Router");
-
- this.metrics = new RouterMetricsService(this);
- addService(this.metrics);
-
- // JVM pause monitor
- this.pauseMonitor = new JvmPauseMonitor();
- this.pauseMonitor.init(conf);
- }
-
- // Initial quota relevant service
- if (conf.getBoolean(DFSConfigKeys.DFS_ROUTER_QUOTA_ENABLE,
- DFSConfigKeys.DFS_ROUTER_QUOTA_ENABLED_DEFAULT)) {
- this.quotaManager = new RouterQuotaManager();
- this.quotaUpdateService = new RouterQuotaUpdateService(this);
- addService(this.quotaUpdateService);
- }
-
- // Safemode service to refuse RPC calls when the router is out of sync
- if (conf.getBoolean(
- DFSConfigKeys.DFS_ROUTER_SAFEMODE_ENABLE,
- DFSConfigKeys.DFS_ROUTER_SAFEMODE_ENABLE_DEFAULT)) {
- // Create safemode monitoring service
- this.safemodeService = new RouterSafemodeService(this);
- addService(this.safemodeService);
- }
-
- super.serviceInit(conf);
- }
-
- @Override
- protected void serviceStart() throws Exception {
-
- if (this.safemodeService == null) {
- // Router is running now
- updateRouterState(RouterServiceState.RUNNING);
- }
-
- if (this.pauseMonitor != null) {
- this.pauseMonitor.start();
- JvmMetrics jvmMetrics = this.metrics.getJvmMetrics();
- if (jvmMetrics != null) {
- jvmMetrics.setPauseMonitor(pauseMonitor);
- }
- }
-
- super.serviceStart();
- }
-
- @Override
- protected void serviceStop() throws Exception {
-
- // Update state
- updateRouterState(RouterServiceState.SHUTDOWN);
-
- // JVM pause monitor
- if (this.pauseMonitor != null) {
- this.pauseMonitor.stop();
- }
-
- super.serviceStop();
- }
-
- /**
- * Shutdown the router.
- */
- public void shutDown() {
- new Thread() {
- @Override
- public void run() {
- Router.this.stop();
- }
- }.start();
- }
-
- /////////////////////////////////////////////////////////
- // RPC Server
- /////////////////////////////////////////////////////////
-
- /**
- * Create a new Router RPC server to proxy ClientProtocol requests.
- *
- * @return New Router RPC Server.
- * @throws IOException If the router RPC server was not started.
- */
- protected RouterRpcServer createRpcServer() throws IOException {
- return new RouterRpcServer(this.conf, this, this.getNamenodeResolver(),
- this.getSubclusterResolver());
- }
-
- /**
- * Get the Router RPC server.
- *
- * @return Router RPC server.
- */
- public RouterRpcServer getRpcServer() {
- return this.rpcServer;
- }
-
- /**
- * Set the current RPC socket for the router.
- *
- * @param address RPC address.
- */
- protected void setRpcServerAddress(InetSocketAddress address) {
- this.rpcAddress = address;
-
- // Use the RPC address as our unique router Id
- if (this.rpcAddress != null) {
- try {
- String hostname = InetAddress.getLocalHost().getHostName();
- setRouterId(hostname + ":" + this.rpcAddress.getPort());
- } catch (UnknownHostException ex) {
- LOG.error("Cannot set unique router ID, address not resolvable {}",
- this.rpcAddress);
- }
- }
- }
-
- /**
- * Get the current RPC socket address for the router.
- *
- * @return InetSocketAddress
- */
- public InetSocketAddress getRpcServerAddress() {
- return this.rpcAddress;
- }
-
- /////////////////////////////////////////////////////////
- // Admin server
- /////////////////////////////////////////////////////////
-
- /**
- * Create a new router admin server to handle the router admin interface.
- *
- * @return RouterAdminServer
- * @throws IOException If the admin server was not successfully started.
- */
- protected RouterAdminServer createAdminServer() throws IOException {
- return new RouterAdminServer(this.conf, this);
- }
-
- /**
- * Set the current Admin socket for the router.
- *
- * @param address Admin RPC address.
- */
- protected void setAdminServerAddress(InetSocketAddress address) {
- this.adminAddress = address;
- }
-
- /**
- * Get the current Admin socket address for the router.
- *
- * @return InetSocketAddress Admin address.
- */
- public InetSocketAddress getAdminServerAddress() {
- return adminAddress;
- }
-
- /////////////////////////////////////////////////////////
- // HTTP server
- /////////////////////////////////////////////////////////
-
- /**
- * Create an HTTP server for this Router.
- *
- * @return HTTP server for this Router.
- */
- protected RouterHttpServer createHttpServer() {
- return new RouterHttpServer(this);
- }
-
- /**
- * Get the current HTTP socket address for the router.
- *
- * @return InetSocketAddress HTTP address.
- */
- public InetSocketAddress getHttpServerAddress() {
- if (httpServer != null) {
- return httpServer.getHttpAddress();
- }
- return null;
- }
-
- /////////////////////////////////////////////////////////
- // Namenode heartbeat monitors
- /////////////////////////////////////////////////////////
-
- /**
- * Create each of the services that will monitor a Namenode.
- *
- * @return List of heartbeat services.
- */
- protected Collection<NamenodeHeartbeatService>
- createNamenodeHeartbeatServices() {
-
- Map<String, NamenodeHeartbeatService> ret = new HashMap<>();
-
- if (conf.getBoolean(
- DFSConfigKeys.DFS_ROUTER_MONITOR_LOCAL_NAMENODE,
- DFSConfigKeys.DFS_ROUTER_MONITOR_LOCAL_NAMENODE_DEFAULT)) {
- // Create a local heartbet service
- NamenodeHeartbeatService localHeartbeatService =
- createLocalNamenodeHearbeatService();
- if (localHeartbeatService != null) {
- String nnDesc = localHeartbeatService.getNamenodeDesc();
- ret.put(nnDesc, localHeartbeatService);
- }
- }
-
- // Create heartbeat services for a list specified by the admin
- String namenodes = this.conf.get(
- DFSConfigKeys.DFS_ROUTER_MONITOR_NAMENODE);
- if (namenodes != null) {
- for (String namenode : namenodes.split(",")) {
- String[] namenodeSplit = namenode.split("\\.");
- String nsId = null;
- String nnId = null;
- if (namenodeSplit.length == 2) {
- nsId = namenodeSplit[0];
- nnId = namenodeSplit[1];
- } else if (namenodeSplit.length == 1) {
- nsId = namenode;
- } else {
- LOG.error("Wrong Namenode to monitor: {}", namenode);
- }
- if (nsId != null) {
- NamenodeHeartbeatService heartbeatService =
- createNamenodeHearbeatService(nsId, nnId);
- if (heartbeatService != null) {
- ret.put(heartbeatService.getNamenodeDesc(), heartbeatService);
- }
- }
- }
- }
-
- return ret.values();
- }
-
- /**
- * Create a new status updater for the local Namenode.
- *
- * @return Updater of the status for the local Namenode.
- */
- protected NamenodeHeartbeatService createLocalNamenodeHearbeatService() {
- // Detect NN running in this machine
- String nsId = DFSUtil.getNamenodeNameServiceId(conf);
- String nnId = null;
- if (HAUtil.isHAEnabled(conf, nsId)) {
- nnId = HAUtil.getNameNodeId(conf, nsId);
- if (nnId == null) {
- LOG.error("Cannot find namenode id for local {}", nsId);
- }
- }
-
- return createNamenodeHearbeatService(nsId, nnId);
- }
-
- /**
- * Create a heartbeat monitor for a particular Namenode.
- *
- * @param nsId Identifier of the nameservice to monitor.
- * @param nnId Identifier of the namenode (HA) to monitor.
- * @return Updater of the status for the specified Namenode.
- */
- protected NamenodeHeartbeatService createNamenodeHearbeatService(
- String nsId, String nnId) {
-
- LOG.info("Creating heartbeat service for Namenode {} in {}", nnId, nsId);
- NamenodeHeartbeatService ret = new NamenodeHeartbeatService(
- namenodeResolver, nsId, nnId);
- return ret;
- }
-
- /////////////////////////////////////////////////////////
- // Router State Management
- /////////////////////////////////////////////////////////
-
- /**
- * Update the router state and heartbeat to the state store.
- *
- * @param state The new router state.
- */
- public void updateRouterState(RouterServiceState newState) {
- this.state = newState;
- if (this.routerHeartbeatService != null) {
- this.routerHeartbeatService.updateStateAsync();
- }
- }
-
- /**
- * Get the status of the router.
- *
- * @return Status of the router.
- */
- public RouterServiceState getRouterState() {
- return this.state;
- }
-
- /////////////////////////////////////////////////////////
- // Submodule getters
- /////////////////////////////////////////////////////////
-
- /**
- * Get the State Store service.
- *
- * @return State Store service.
- */
- public StateStoreService getStateStore() {
- return this.stateStore;
- }
-
- /**
- * Get the metrics system for the Router.
- *
- * @return Router metrics.
- */
- public RouterMetrics getRouterMetrics() {
- if (this.metrics != null) {
- return this.metrics.getRouterMetrics();
- }
- return null;
- }
-
- /**
- * Get the federation metrics.
- *
- * @return Federation metrics.
- */
- public FederationMetrics getMetrics() {
- if (this.metrics != null) {
- return this.metrics.getFederationMetrics();
- }
- return null;
- }
-
- /**
- * Get the subcluster resolver for files.
- *
- * @return Subcluster resolver for files.
- */
- public FileSubclusterResolver getSubclusterResolver() {
- return this.subclusterResolver;
- }
-
- /**
- * Get the namenode resolver for a subcluster.
- *
- * @return The namenode resolver for a subcluster.
- */
- public ActiveNamenodeResolver getNamenodeResolver() {
- return this.namenodeResolver;
- }
-
- /**
- * Get the state store interface for the router heartbeats.
- *
- * @return FederationRouterStateStore state store API handle.
- */
- public RouterStore getRouterStateManager() {
- if (this.routerStateManager == null && this.stateStore != null) {
- this.routerStateManager = this.stateStore.getRegisteredRecordStore(
- RouterStore.class);
- }
- return this.routerStateManager;
- }
-
- /////////////////////////////////////////////////////////
- // Router info
- /////////////////////////////////////////////////////////
-
- /**
- * Get the start date of the Router.
- *
- * @return Start date of the router.
- */
- public long getStartTime() {
- return this.startTime;
- }
-
- /**
- * Unique ID for the router, typically the hostname:port string for the
- * router's RPC server. This ID may be null on router startup before the RPC
- * server has bound to a port.
- *
- * @return Router identifier.
- */
- public String getRouterId() {
- return this.routerId;
- }
-
- /**
- * Sets a unique ID for this router.
- *
- * @param id Identifier of the Router.
- */
- public void setRouterId(String id) {
- this.routerId = id;
- if (this.stateStore != null) {
- this.stateStore.setIdentifier(this.routerId);
- }
- if (this.namenodeResolver != null) {
- this.namenodeResolver.setRouterId(this.routerId);
- }
- }
-
- /**
- * If the quota system is enabled in Router.
- */
- public boolean isQuotaEnabled() {
- return this.quotaManager != null;
- }
-
- /**
- * Get route quota manager.
- * @return RouterQuotaManager Quota manager.
- */
- public RouterQuotaManager getQuotaManager() {
- return this.quotaManager;
- }
-
- /**
- * Get quota cache update service.
- */
- @VisibleForTesting
- RouterQuotaUpdateService getQuotaCacheUpdateService() {
- return this.quotaUpdateService;
- }
-
- /**
- * Get the list of namenode heartbeat service.
- */
- @VisibleForTesting
- Collection<NamenodeHeartbeatService> getNamenodeHearbeatServices() {
- return this.namenodeHeartbeatServices;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
deleted file mode 100644
index 0c0448c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
+++ /dev/null
@@ -1,298 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_ENABLED_DEFAULT;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.protocol.proto.RouterProtocolProtos.RouterAdminProtocolService;
-import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolPB;
-import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolServerSideTranslatorPB;
-import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
-import org.apache.hadoop.hdfs.server.federation.store.MountTableStore;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.RPC.Server;
-import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.AbstractService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.protobuf.BlockingService;
-
-/**
- * This class is responsible for handling all of the Admin calls to the HDFS
- * router. It is created, started, and stopped by {@link Router}.
- */
-public class RouterAdminServer extends AbstractService
- implements MountTableManager, RouterStateManager {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(RouterAdminServer.class);
-
- private Configuration conf;
-
- private final Router router;
-
- private MountTableStore mountTableStore;
-
- /** The Admin server that listens to requests from clients. */
- private final Server adminServer;
- private final InetSocketAddress adminAddress;
-
- /**
- * Permission related info used for constructing new router permission
- * checker instance.
- */
- private static String routerOwner;
- private static String superGroup;
- private static boolean isPermissionEnabled;
-
- public RouterAdminServer(Configuration conf, Router router)
- throws IOException {
- super(RouterAdminServer.class.getName());
-
- this.conf = conf;
- this.router = router;
-
- int handlerCount = this.conf.getInt(
- DFSConfigKeys.DFS_ROUTER_ADMIN_HANDLER_COUNT_KEY,
- DFSConfigKeys.DFS_ROUTER_ADMIN_HANDLER_COUNT_DEFAULT);
-
- RPC.setProtocolEngine(this.conf, RouterAdminProtocolPB.class,
- ProtobufRpcEngine.class);
-
- RouterAdminProtocolServerSideTranslatorPB routerAdminProtocolTranslator =
- new RouterAdminProtocolServerSideTranslatorPB(this);
- BlockingService clientNNPbService = RouterAdminProtocolService.
- newReflectiveBlockingService(routerAdminProtocolTranslator);
-
- InetSocketAddress confRpcAddress = conf.getSocketAddr(
- DFSConfigKeys.DFS_ROUTER_ADMIN_BIND_HOST_KEY,
- DFSConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_KEY,
- DFSConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_DEFAULT,
- DFSConfigKeys.DFS_ROUTER_ADMIN_PORT_DEFAULT);
-
- String bindHost = conf.get(
- DFSConfigKeys.DFS_ROUTER_ADMIN_BIND_HOST_KEY,
- confRpcAddress.getHostName());
- LOG.info("Admin server binding to {}:{}",
- bindHost, confRpcAddress.getPort());
-
- initializePermissionSettings(this.conf);
- this.adminServer = new RPC.Builder(this.conf)
- .setProtocol(RouterAdminProtocolPB.class)
- .setInstance(clientNNPbService)
- .setBindAddress(bindHost)
- .setPort(confRpcAddress.getPort())
- .setNumHandlers(handlerCount)
- .setVerbose(false)
- .build();
-
- // The RPC-server port can be ephemeral... ensure we have the correct info
- InetSocketAddress listenAddress = this.adminServer.getListenerAddress();
- this.adminAddress = new InetSocketAddress(
- confRpcAddress.getHostName(), listenAddress.getPort());
- router.setAdminServerAddress(this.adminAddress);
- }
-
- /**
- * Initialize permission related settings.
- *
- * @param routerConf
- * @throws IOException
- */
- private static void initializePermissionSettings(Configuration routerConf)
- throws IOException {
- routerOwner = UserGroupInformation.getCurrentUser().getShortUserName();
- superGroup = routerConf.get(
- DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_KEY,
- DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT);
- isPermissionEnabled = routerConf.getBoolean(DFS_PERMISSIONS_ENABLED_KEY,
- DFS_PERMISSIONS_ENABLED_DEFAULT);
- }
-
- /** Allow access to the client RPC server for testing. */
- @VisibleForTesting
- Server getAdminServer() {
- return this.adminServer;
- }
-
- private MountTableStore getMountTableStore() throws IOException {
- if (this.mountTableStore == null) {
- this.mountTableStore = router.getStateStore().getRegisteredRecordStore(
- MountTableStore.class);
- if (this.mountTableStore == null) {
- throw new IOException("Mount table state store is not available.");
- }
- }
- return this.mountTableStore;
- }
-
- /**
- * Get the RPC address of the admin service.
- * @return Administration service RPC address.
- */
- public InetSocketAddress getRpcAddress() {
- return this.adminAddress;
- }
-
- @Override
- protected void serviceInit(Configuration configuration) throws Exception {
- this.conf = configuration;
- super.serviceInit(conf);
- }
-
- @Override
- protected void serviceStart() throws Exception {
- this.adminServer.start();
- super.serviceStart();
- }
-
- @Override
- protected void serviceStop() throws Exception {
- if (this.adminServer != null) {
- this.adminServer.stop();
- }
- super.serviceStop();
- }
-
- @Override
- public AddMountTableEntryResponse addMountTableEntry(
- AddMountTableEntryRequest request) throws IOException {
- return getMountTableStore().addMountTableEntry(request);
- }
-
- @Override
- public UpdateMountTableEntryResponse updateMountTableEntry(
- UpdateMountTableEntryRequest request) throws IOException {
- return getMountTableStore().updateMountTableEntry(request);
- }
-
- @Override
- public RemoveMountTableEntryResponse removeMountTableEntry(
- RemoveMountTableEntryRequest request) throws IOException {
- return getMountTableStore().removeMountTableEntry(request);
- }
-
- @Override
- public GetMountTableEntriesResponse getMountTableEntries(
- GetMountTableEntriesRequest request) throws IOException {
- return getMountTableStore().getMountTableEntries(request);
- }
-
- @Override
- public EnterSafeModeResponse enterSafeMode(EnterSafeModeRequest request)
- throws IOException {
- this.router.updateRouterState(RouterServiceState.SAFEMODE);
- this.router.getRpcServer().setSafeMode(true);
- return EnterSafeModeResponse.newInstance(verifySafeMode(true));
- }
-
- @Override
- public LeaveSafeModeResponse leaveSafeMode(LeaveSafeModeRequest request)
- throws IOException {
- this.router.updateRouterState(RouterServiceState.RUNNING);
- this.router.getRpcServer().setSafeMode(false);
- return LeaveSafeModeResponse.newInstance(verifySafeMode(false));
- }
-
- @Override
- public GetSafeModeResponse getSafeMode(GetSafeModeRequest request)
- throws IOException {
- boolean isInSafeMode = this.router.getRpcServer().isInSafeMode();
- return GetSafeModeResponse.newInstance(isInSafeMode);
- }
-
- /**
- * Verify if Router set safe mode state correctly.
- * @param isInSafeMode Expected state to be set.
- * @return
- */
- private boolean verifySafeMode(boolean isInSafeMode) {
- boolean serverInSafeMode = this.router.getRpcServer().isInSafeMode();
- RouterServiceState currentState = this.router.getRouterState();
-
- return (isInSafeMode && currentState == RouterServiceState.SAFEMODE
- && serverInSafeMode)
- || (!isInSafeMode && currentState != RouterServiceState.SAFEMODE
- && !serverInSafeMode);
- }
-
- /**
- * Get a new permission checker used for making mount table access
- * control. This method will be invoked during each RPC call in router
- * admin server.
- *
- * @return Router permission checker
- * @throws AccessControlException
- */
- public static RouterPermissionChecker getPermissionChecker()
- throws AccessControlException {
- if (!isPermissionEnabled) {
- return null;
- }
-
- try {
- return new RouterPermissionChecker(routerOwner, superGroup,
- NameNode.getRemoteUser());
- } catch (IOException e) {
- throw new AccessControlException(e);
- }
- }
-
- /**
- * Get super user name.
- *
- * @return String super user name.
- */
- public static String getSuperUser() {
- return routerOwner;
- }
-
- /**
- * Get super group name.
- *
- * @return String super group name.
- */
- public static String getSuperGroup(){
- return superGroup;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClient.java
deleted file mode 100644
index b36e459..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClient.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolPB;
-import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolTranslatorPB;
-import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-
-/**
- * Client to connect to the {@link Router} via the admin protocol.
- */
-@Private
-public class RouterClient implements Closeable {
-
- private final RouterAdminProtocolTranslatorPB proxy;
- private final UserGroupInformation ugi;
-
- private static RouterAdminProtocolTranslatorPB createRouterProxy(
- InetSocketAddress address, Configuration conf, UserGroupInformation ugi)
- throws IOException {
-
- RPC.setProtocolEngine(
- conf, RouterAdminProtocolPB.class, ProtobufRpcEngine.class);
-
- AtomicBoolean fallbackToSimpleAuth = new AtomicBoolean(false);
- final long version = RPC.getProtocolVersion(RouterAdminProtocolPB.class);
- RouterAdminProtocolPB proxy = RPC.getProtocolProxy(
- RouterAdminProtocolPB.class, version, address, ugi, conf,
- NetUtils.getDefaultSocketFactory(conf),
- RPC.getRpcTimeout(conf), null,
- fallbackToSimpleAuth).getProxy();
-
- return new RouterAdminProtocolTranslatorPB(proxy);
- }
-
- public RouterClient(InetSocketAddress address, Configuration conf)
- throws IOException {
- this.ugi = UserGroupInformation.getCurrentUser();
- this.proxy = createRouterProxy(address, conf, ugi);
- }
-
- public MountTableManager getMountTableManager() {
- return proxy;
- }
-
- public RouterStateManager getRouterStateManager() {
- return proxy;
- }
-
- @Override
- public synchronized void close() throws IOException {
- RPC.stopProxy(proxy);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHeartbeatService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHeartbeatService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHeartbeatService.java
deleted file mode 100644
index 6e44984..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHeartbeatService.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.server.federation.store.CachedRecordStore;
-import org.apache.hadoop.hdfs.server.federation.store.MembershipStore;
-import org.apache.hadoop.hdfs.server.federation.store.MountTableStore;
-import org.apache.hadoop.hdfs.server.federation.store.RecordStore;
-import org.apache.hadoop.hdfs.server.federation.store.RouterStore;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
-import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
-import org.apache.hadoop.hdfs.server.federation.store.records.StateStoreVersion;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Service to periodically update the Router current state in the State Store.
- */
-public class RouterHeartbeatService extends PeriodicService {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(RouterHeartbeatService.class);
-
- /** Router we are hearbeating. */
- private final Router router;
-
- /**
- * Create a new Router heartbeat service.
- *
- * @param router Router to heartbeat.
- */
- public RouterHeartbeatService(Router router) {
- super(RouterHeartbeatService.class.getSimpleName());
- this.router = router;
- }
-
- /**
- * Trigger the update of the Router state asynchronously.
- */
- protected void updateStateAsync() {
- Thread thread = new Thread(new Runnable() {
- @Override
- public void run() {
- updateStateStore();
- }
- }, "Router Heartbeat Async");
- thread.setDaemon(true);
- thread.start();
- }
-
- /**
- * Update the state of the Router in the State Store.
- */
- @VisibleForTesting
- synchronized void updateStateStore() {
- String routerId = router.getRouterId();
- if (routerId == null) {
- LOG.error("Cannot heartbeat for router: unknown router id");
- return;
- }
- if (isStoreAvailable()) {
- RouterStore routerStore = router.getRouterStateManager();
- try {
- RouterState record = RouterState.newInstance(
- routerId, router.getStartTime(), router.getRouterState());
- StateStoreVersion stateStoreVersion = StateStoreVersion.newInstance(
- getStateStoreVersion(MembershipStore.class),
- getStateStoreVersion(MountTableStore.class));
- record.setStateStoreVersion(stateStoreVersion);
- RouterHeartbeatRequest request =
- RouterHeartbeatRequest.newInstance(record);
- RouterHeartbeatResponse response = routerStore.routerHeartbeat(request);
- if (!response.getStatus()) {
- LOG.warn("Cannot heartbeat router {}", routerId);
- } else {
- LOG.debug("Router heartbeat for router {}", routerId);
- }
- } catch (IOException e) {
- LOG.error("Cannot heartbeat router {}: {}", routerId, e.getMessage());
- }
- } else {
- LOG.warn("Cannot heartbeat router {}: State Store unavailable", routerId);
- }
- }
-
- /**
- * Get the version of the data in the State Store.
- *
- * @param clazz Class in the State Store.
- * @return Version of the data.
- */
- private <R extends BaseRecord, S extends RecordStore<R>>
- long getStateStoreVersion(final Class<S> clazz) {
- long version = -1;
- try {
- StateStoreService stateStore = router.getStateStore();
- S recordStore = stateStore.getRegisteredRecordStore(clazz);
- if (recordStore != null) {
- if (recordStore instanceof CachedRecordStore) {
- CachedRecordStore<R> cachedRecordStore =
- (CachedRecordStore<R>) recordStore;
- List<R> records = cachedRecordStore.getCachedRecords();
- for (BaseRecord record : records) {
- if (record.getDateModified() > version) {
- version = record.getDateModified();
- }
- }
- }
- }
- } catch (Exception e) {
- LOG.error("Cannot get version for {}: {}", clazz, e.getMessage());
- }
- return version;
- }
-
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
-
- long interval = conf.getTimeDuration(
- DFSConfigKeys.DFS_ROUTER_HEARTBEAT_STATE_INTERVAL_MS,
- DFSConfigKeys.DFS_ROUTER_HEARTBEAT_STATE_INTERVAL_MS_DEFAULT,
- TimeUnit.MILLISECONDS);
- this.setIntervalMs(interval);
-
- super.serviceInit(conf);
- }
-
- @Override
- public void periodicInvoke() {
- updateStateStore();
- }
-
- private boolean isStoreAvailable() {
- if (router.getRouterStateManager() == null) {
- return false;
- }
- if (router.getStateStore() == null) {
- return false;
- }
- return router.getStateStore().isDriverReady();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java
deleted file mode 100644
index 046f0ba..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import java.net.InetSocketAddress;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.server.common.JspHelper;
-import org.apache.hadoop.http.HttpServer2;
-import org.apache.hadoop.service.AbstractService;
-
-/**
- * Web interface for the {@link Router}. It exposes the Web UI and the WebHDFS
- * methods from {@link RouterWebHdfsMethods}.
- */
-public class RouterHttpServer extends AbstractService {
-
- protected static final String NAMENODE_ATTRIBUTE_KEY = "name.node";
-
-
- /** Configuration for the Router HTTP server. */
- private Configuration conf;
-
- /** Router using this HTTP server. */
- private final Router router;
-
- /** HTTP server. */
- private HttpServer2 httpServer;
-
- /** HTTP addresses. */
- private InetSocketAddress httpAddress;
- private InetSocketAddress httpsAddress;
-
-
- public RouterHttpServer(Router router) {
- super(RouterHttpServer.class.getName());
- this.router = router;
- }
-
- @Override
- protected void serviceInit(Configuration configuration) throws Exception {
- this.conf = configuration;
-
- // Get HTTP address
- this.httpAddress = conf.getSocketAddr(
- DFSConfigKeys.DFS_ROUTER_HTTP_BIND_HOST_KEY,
- DFSConfigKeys.DFS_ROUTER_HTTP_ADDRESS_KEY,
- DFSConfigKeys.DFS_ROUTER_HTTP_ADDRESS_DEFAULT,
- DFSConfigKeys.DFS_ROUTER_HTTP_PORT_DEFAULT);
-
- // Get HTTPs address
- this.httpsAddress = conf.getSocketAddr(
- DFSConfigKeys.DFS_ROUTER_HTTPS_BIND_HOST_KEY,
- DFSConfigKeys.DFS_ROUTER_HTTPS_ADDRESS_KEY,
- DFSConfigKeys.DFS_ROUTER_HTTPS_ADDRESS_DEFAULT,
- DFSConfigKeys.DFS_ROUTER_HTTPS_PORT_DEFAULT);
-
- super.serviceInit(conf);
- }
-
- @Override
- protected void serviceStart() throws Exception {
- // Build and start server
- String webApp = "router";
- HttpServer2.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(
- this.conf, this.httpAddress, this.httpsAddress, webApp,
- DFSConfigKeys.DFS_NAMENODE_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY,
- DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY);
-
- this.httpServer = builder.build();
-
- this.httpServer.setAttribute(NAMENODE_ATTRIBUTE_KEY, this.router);
- this.httpServer.setAttribute(JspHelper.CURRENT_CONF, this.conf);
- setupServlets(this.httpServer, this.conf);
-
- this.httpServer.start();
-
- // The server port can be ephemeral... ensure we have the correct info
- InetSocketAddress listenAddress = this.httpServer.getConnectorAddress(0);
- if (listenAddress != null) {
- this.httpAddress = new InetSocketAddress(this.httpAddress.getHostName(),
- listenAddress.getPort());
- }
- super.serviceStart();
- }
-
- @Override
- protected void serviceStop() throws Exception {
- if(this.httpServer != null) {
- this.httpServer.stop();
- }
- super.serviceStop();
- }
-
- private static void setupServlets(
- HttpServer2 httpServer, Configuration conf) {
- // TODO Add servlets for FSCK, etc
- }
-
- public InetSocketAddress getHttpAddress() {
- return this.httpAddress;
- }
-
- public InetSocketAddress getHttpsAddress() {
- return this.httpsAddress;
- }
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/LocalResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/LocalResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/LocalResolver.java
deleted file mode 100644
index 3508eab..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/LocalResolver.java
+++ /dev/null
@@ -1,297 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.resolver.order;
-
-import static org.apache.hadoop.util.Time.monotonicNow;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.net.HostAndPort;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.security.PrivilegedAction;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
-import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
-import org.apache.hadoop.hdfs.server.federation.router.Router;
-import org.apache.hadoop.hdfs.server.federation.router.RouterRpcServer;
-import org.apache.hadoop.hdfs.server.federation.store.MembershipStore;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsRequest;
-import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsResponse;
-import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
-import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
-import org.apache.hadoop.ipc.RPC.Server;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * The local subcluster (where the writer is) should be tried first. The writer
- * is defined from the RPC query received in the RPC server.
- */
-public class LocalResolver implements OrderedResolver {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(LocalResolver.class);
-
- /** Configuration key to set the minimum time to update the local cache.*/
- public static final String MIN_UPDATE_PERIOD_KEY =
- DFSConfigKeys.FEDERATION_ROUTER_PREFIX + "local-resolver.update-period";
- /** 10 seconds by default. */
- private static final long MIN_UPDATE_PERIOD_DEFAULT =
- TimeUnit.SECONDS.toMillis(10);
-
-
- /** Router service. */
- private final Router router;
- /** Minimum update time. */
- private final long minUpdateTime;
-
- /** Node IP -> Subcluster. */
- private Map<String, String> nodeSubcluster = null;
- /** Last time the subcluster map was updated. */
- private long lastUpdated;
-
-
- public LocalResolver(final Configuration conf, final Router routerService) {
- this.minUpdateTime = conf.getTimeDuration(
- MIN_UPDATE_PERIOD_KEY, MIN_UPDATE_PERIOD_DEFAULT,
- TimeUnit.MILLISECONDS);
- this.router = routerService;
- }
-
- /**
- * Get the local name space. This relies on the RPC Server to get the address
- * from the client.
- *
- * TODO we only support DN and NN locations, we need to add others like
- * Resource Managers.
- *
- * @param path Path ignored by this policy.
- * @param loc Federated location with multiple destinations.
- * @return Local name space. Null if we don't know about this machine.
- */
- @Override
- public String getFirstNamespace(final String path, final PathLocation loc) {
- String localSubcluster = null;
- String clientAddr = getClientAddr();
- Map<String, String> nodeToSubcluster = getSubclusterMappings();
- if (nodeToSubcluster != null) {
- localSubcluster = nodeToSubcluster.get(clientAddr);
- if (localSubcluster != null) {
- LOG.debug("Local namespace for {} is {}", clientAddr, localSubcluster);
- } else {
- LOG.error("Cannot get local namespace for {}", clientAddr);
- }
- } else {
- LOG.error("Cannot get node mapping when resolving {} at {} from {}",
- path, loc, clientAddr);
- }
- return localSubcluster;
- }
-
- @VisibleForTesting
- String getClientAddr() {
- return Server.getRemoteAddress();
- }
-
- /**
- * Get the mapping from nodes to subcluster. It gets this mapping from the
- * subclusters through expensive calls (e.g., RPC) and uses caching to avoid
- * too many calls. The cache might be updated asynchronously to reduce
- * latency.
- *
- * @return Node IP -> Subcluster.
- */
- @VisibleForTesting
- synchronized Map<String, String> getSubclusterMappings() {
- if (nodeSubcluster == null ||
- (monotonicNow() - lastUpdated) > minUpdateTime) {
- // Fetch the mapping asynchronously
- Thread updater = new Thread(new Runnable() {
- @Override
- public void run() {
- Map<String, String> mapping = new HashMap<>();
-
- Map<String, String> dnSubcluster = getDatanodesSubcluster();
- if (dnSubcluster != null) {
- mapping.putAll(dnSubcluster);
- }
-
- Map<String, String> nnSubcluster = getNamenodesSubcluster();
- if (nnSubcluster != null) {
- mapping.putAll(nnSubcluster);
- }
- nodeSubcluster = mapping;
- lastUpdated = monotonicNow();
- }
- });
- updater.start();
-
- // Wait until initialized
- if (nodeSubcluster == null) {
- try {
- LOG.debug("Wait to get the mapping for the first time");
- updater.join();
- } catch (InterruptedException e) {
- LOG.error("Cannot wait for the updater to finish");
- }
- }
- }
- return nodeSubcluster;
- }
-
- /**
- * Get the Datanode mapping from the subclusters from the Namenodes. This
- * needs to be done as a privileged action to use the user for the Router and
- * not the one from the client in the RPC call.
- *
- * @return DN IP -> Subcluster.
- */
- private Map<String, String> getDatanodesSubcluster() {
-
- final RouterRpcServer rpcServer = getRpcServer();
- if (rpcServer == null) {
- LOG.error("Cannot access the Router RPC server");
- return null;
- }
-
- Map<String, String> ret = new HashMap<>();
- try {
- // We need to get the DNs as a privileged user
- UserGroupInformation loginUser = UserGroupInformation.getLoginUser();
- Map<String, DatanodeStorageReport[]> dnMap = loginUser.doAs(
- new PrivilegedAction<Map<String, DatanodeStorageReport[]>>() {
- @Override
- public Map<String, DatanodeStorageReport[]> run() {
- try {
- return rpcServer.getDatanodeStorageReportMap(
- DatanodeReportType.ALL);
- } catch (IOException e) {
- LOG.error("Cannot get the datanodes from the RPC server", e);
- return null;
- }
- }
- });
- for (Entry<String, DatanodeStorageReport[]> entry : dnMap.entrySet()) {
- String nsId = entry.getKey();
- DatanodeStorageReport[] dns = entry.getValue();
- for (DatanodeStorageReport dn : dns) {
- DatanodeInfo dnInfo = dn.getDatanodeInfo();
- String ipAddr = dnInfo.getIpAddr();
- ret.put(ipAddr, nsId);
- }
- }
- } catch (IOException e) {
- LOG.error("Cannot get Datanodes from the Namenodes: {}", e.getMessage());
- }
- return ret;
- }
-
- /**
- * Get the Namenode mapping from the subclusters from the Membership store. As
- * the Routers are usually co-located with Namenodes, we also check for the
- * local address for this Router here.
- *
- * @return NN IP -> Subcluster.
- */
- private Map<String, String> getNamenodesSubcluster() {
-
- final MembershipStore membershipStore = getMembershipStore();
- if (membershipStore == null) {
- LOG.error("Cannot access the Membership store");
- return null;
- }
-
- // Manage requests from this hostname (127.0.0.1)
- String localIp = "127.0.0.1";
- String localHostname = localIp;
- try {
- localHostname = InetAddress.getLocalHost().getHostName();
- } catch (UnknownHostException e) {
- LOG.error("Cannot get local host name");
- }
-
- Map<String, String> ret = new HashMap<>();
- try {
- // Get the values from the store
- GetNamenodeRegistrationsRequest request =
- GetNamenodeRegistrationsRequest.newInstance();
- GetNamenodeRegistrationsResponse response =
- membershipStore.getNamenodeRegistrations(request);
- final List<MembershipState> nns = response.getNamenodeMemberships();
- for (MembershipState nn : nns) {
- try {
- String nsId = nn.getNameserviceId();
- String rpcAddress = nn.getRpcAddress();
- String hostname = HostAndPort.fromString(rpcAddress).getHostText();
- ret.put(hostname, nsId);
- if (hostname.equals(localHostname)) {
- ret.put(localIp, nsId);
- }
-
- InetAddress addr = InetAddress.getByName(hostname);
- String ipAddr = addr.getHostAddress();
- ret.put(ipAddr, nsId);
- } catch (Exception e) {
- LOG.error("Cannot get address for {}: {}", nn, e.getMessage());
- }
- }
- } catch (IOException ioe) {
- LOG.error("Cannot get Namenodes from the State Store: {}",
- ioe.getMessage());
- }
- return ret;
- }
-
- /**
- * Get the Router RPC server.
- *
- * @return Router RPC server. Null if not possible.
- */
- private RouterRpcServer getRpcServer() {
- if (this.router == null) {
- return null;
- }
- return router.getRpcServer();
- }
-
- /**
- * Get the Membership store.
- *
- * @return Membership store.
- */
- private MembershipStore getMembershipStore() {
- StateStoreService stateStore = router.getStateStore();
- if (stateStore == null) {
- return null;
- }
- return stateStore.getRegisteredRecordStore(MembershipStore.class);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/OrderedResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/OrderedResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/OrderedResolver.java
deleted file mode 100644
index 3a3ccf7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/OrderedResolver.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.resolver.order;
-
-import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
-
-
-/**
- * Policy that decides which should be the first location accessed given
- * multiple destinations.
- */
-public interface OrderedResolver {
-
- /**
- * Get the first namespace based on this resolver approach.
- *
- * @param path Path to check.
- * @param loc Federated location with multiple destinations.
- * @return First namespace out of the locations.
- */
- String getFirstNamespace(String path, PathLocation loc);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/RandomResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/RandomResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/RandomResolver.java
deleted file mode 100644
index 022aa48..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/RandomResolver.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.resolver.order;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
-
-import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * Order the destinations randomly.
- */
-public class RandomResolver implements OrderedResolver {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(RandomResolver.class);
-
-
- /** Random number generator. */
- private static final Random RANDOM = new Random();
-
- /**
- * Get a random name space from the path.
- *
- * @param path Path ignored by this policy.
- * @param loc Federated location with multiple destinations.
- * @return Random name space.
- */
- public String getFirstNamespace(final String path, final PathLocation loc) {
- if (loc == null) {
- return null;
- }
- Set<String> namespaces = loc.getNamespaces();
- if (namespaces == null || namespaces.isEmpty()) {
- LOG.error("Cannot get namespaces for {}", loc);
- return null;
- }
- List<String> nssList = new ArrayList<>(namespaces);
- int index = RANDOM.nextInt(nssList.size());
- return nssList.get(index);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/package-info.java
deleted file mode 100644
index f90152f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/package-info.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * A federated location can be resolved to multiple subclusters. This package
- * takes care of the order in which this multiple destinations should be used.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-
-package org.apache.hadoop.hdfs.server.federation.resolver.order;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/package-info.java
deleted file mode 100644
index d8be9e3..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/package-info.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * The resolver package contains indepedent data resolvers used in HDFS
- * federation. The data resolvers collect data from the cluster, including from
- * the state store. The resolvers expose APIs used by HDFS federation to collect
- * aggregated, cached data for use in Real-time request processing. The
- * resolvers are perf-sensitive and are used in the flow of the
- * {@link RouterRpcServer} request path.
- * <p>
- * The principal resolvers are:
- * <ul>
- * <li>{@link ActiveNamenodeResolver} Real-time interface for locating the most
- * recently active NN for a nameservice.
- * <li>{@link FileSubclusterResolver} Real-time interface for determining the NN
- * and local file path for a given file/folder based on the global namespace
- * path.
- * </ul>
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-package org.apache.hadoop.hdfs.server.federation.resolver;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java
deleted file mode 100644
index 1d27b51..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import org.apache.hadoop.hdfs.NameNodeProxiesClient.ProxyAndInfo;
-import org.apache.hadoop.hdfs.protocol.ClientProtocol;
-import org.apache.hadoop.ipc.RPC;
-
-/**
- * Context to track a connection in a {@link ConnectionPool}. When a client uses
- * a connection, it increments a counter to mark it as active. Once the client
- * is done with the connection, it decreases the counter. It also takes care of
- * closing the connection once is not active.
- */
-public class ConnectionContext {
-
- /** Client for the connection. */
- private final ProxyAndInfo<ClientProtocol> client;
- /** How many threads are using this connection. */
- private int numThreads = 0;
- /** If the connection is closed. */
- private boolean closed = false;
-
-
- public ConnectionContext(ProxyAndInfo<ClientProtocol> connection) {
- this.client = connection;
- }
-
- /**
- * Check if the connection is active.
- *
- * @return True if the connection is active.
- */
- public synchronized boolean isActive() {
- return this.numThreads > 0;
- }
-
- /**
- * Check if the connection is closed.
- *
- * @return If the connection is closed.
- */
- public synchronized boolean isClosed() {
- return this.closed;
- }
-
- /**
- * Check if the connection can be used. It checks if the connection is used by
- * another thread or already closed.
- *
- * @return True if the connection can be used.
- */
- public synchronized boolean isUsable() {
- return !isActive() && !isClosed();
- }
-
- /**
- * Get the connection client.
- *
- * @return Connection client.
- */
- public synchronized ProxyAndInfo<ClientProtocol> getClient() {
- this.numThreads++;
- return this.client;
- }
-
- /**
- * Release this connection. If the connection was closed, close the proxy.
- * Otherwise, mark the connection as not used by us anymore.
- */
- public synchronized void release() {
- if (--this.numThreads == 0 && this.closed) {
- close();
- }
- }
-
- /**
- * We will not use this connection anymore. If it's not being used, we close
- * it. Otherwise, we let release() do it once we are done with it.
- */
- public synchronized void close() {
- this.closed = true;
- if (this.numThreads == 0) {
- ClientProtocol proxy = this.client.getProxy();
- // Nobody should be using this anymore so it should close right away
- RPC.stopProxy(proxy);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java
deleted file mode 100644
index 594f489..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java
+++ /dev/null
@@ -1,436 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.TreeMap;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.Time;
-import org.eclipse.jetty.util.ajax.JSON;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Implements a pool of connections for the {@link Router} to be able to open
- * many connections to many Namenodes.
- */
-public class ConnectionManager {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ConnectionManager.class);
-
- /** Number of parallel new connections to create. */
- protected static final int MAX_NEW_CONNECTIONS = 100;
-
- /** Minimum amount of active connections: 50%. */
- protected static final float MIN_ACTIVE_RATIO = 0.5f;
-
-
- /** Configuration for the connection manager, pool and sockets. */
- private final Configuration conf;
-
- /** Min number of connections per user + nn. */
- private final int minSize = 1;
- /** Max number of connections per user + nn. */
- private final int maxSize;
-
- /** How often we close a pool for a particular user + nn. */
- private final long poolCleanupPeriodMs;
- /** How often we close a connection in a pool. */
- private final long connectionCleanupPeriodMs;
-
- /** Map of connection pools, one pool per user + NN. */
- private final Map<ConnectionPoolId, ConnectionPool> pools;
- /** Lock for accessing pools. */
- private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
- private final Lock readLock = readWriteLock.readLock();
- private final Lock writeLock = readWriteLock.writeLock();
-
- /** Queue for creating new connections. */
- private final BlockingQueue<ConnectionPool> creatorQueue =
- new ArrayBlockingQueue<>(MAX_NEW_CONNECTIONS);
- /** Create new connections asynchronously. */
- private final ConnectionCreator creator;
- /** Periodic executor to remove stale connection pools. */
- private final ScheduledThreadPoolExecutor cleaner =
- new ScheduledThreadPoolExecutor(1);
-
- /** If the connection manager is running. */
- private boolean running = false;
-
-
- /**
- * Creates a proxy client connection pool manager.
- *
- * @param config Configuration for the connections.
- */
- public ConnectionManager(Configuration config) {
- this.conf = config;
-
- // Configure minimum and maximum connection pools
- this.maxSize = this.conf.getInt(
- DFSConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_POOL_SIZE,
- DFSConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_POOL_SIZE_DEFAULT);
-
- // Map with the connections indexed by UGI and Namenode
- this.pools = new HashMap<>();
-
- // Create connections in a thread asynchronously
- this.creator = new ConnectionCreator(creatorQueue);
- this.creator.setDaemon(true);
-
- // Cleanup periods
- this.poolCleanupPeriodMs = this.conf.getLong(
- DFSConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_POOL_CLEAN,
- DFSConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_POOL_CLEAN_DEFAULT);
- LOG.info("Cleaning connection pools every {} seconds",
- TimeUnit.MILLISECONDS.toSeconds(this.poolCleanupPeriodMs));
- this.connectionCleanupPeriodMs = this.conf.getLong(
- DFSConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_CLEAN_MS,
- DFSConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_CLEAN_MS_DEFAULT);
- LOG.info("Cleaning connections every {} seconds",
- TimeUnit.MILLISECONDS.toSeconds(this.connectionCleanupPeriodMs));
- }
-
- /**
- * Start the connection manager.
- */
- public void start() {
- // Start the thread that creates connections asynchronously
- this.creator.start();
-
- // Schedule a task to remove stale connection pools and sockets
- long recyleTimeMs = Math.min(
- poolCleanupPeriodMs, connectionCleanupPeriodMs);
- LOG.info("Cleaning every {} seconds",
- TimeUnit.MILLISECONDS.toSeconds(recyleTimeMs));
- this.cleaner.scheduleAtFixedRate(
- new CleanupTask(), 0, recyleTimeMs, TimeUnit.MILLISECONDS);
-
- // Mark the manager as running
- this.running = true;
- }
-
- /**
- * Stop the connection manager by closing all the pools.
- */
- public void close() {
- this.creator.shutdown();
- this.cleaner.shutdown();
- this.running = false;
-
- writeLock.lock();
- try {
- for (ConnectionPool pool : this.pools.values()) {
- pool.close();
- }
- this.pools.clear();
- } finally {
- writeLock.unlock();
- }
- }
-
- /**
- * Fetches the next available proxy client in the pool. Each client connection
- * is reserved for a single user and cannot be reused until free.
- *
- * @param ugi User group information.
- * @param nnAddress Namenode address for the connection.
- * @return Proxy client to connect to nnId as UGI.
- * @throws IOException If the connection cannot be obtained.
- */
- public ConnectionContext getConnection(
- UserGroupInformation ugi, String nnAddress) throws IOException {
-
- // Check if the manager is shutdown
- if (!this.running) {
- LOG.error(
- "Cannot get a connection to {} because the manager isn't running",
- nnAddress);
- return null;
- }
-
- // Try to get the pool if created
- ConnectionPoolId connectionId = new ConnectionPoolId(ugi, nnAddress);
- ConnectionPool pool = null;
- readLock.lock();
- try {
- pool = this.pools.get(connectionId);
- } finally {
- readLock.unlock();
- }
-
- // Create the pool if not created before
- if (pool == null) {
- writeLock.lock();
- try {
- pool = this.pools.get(connectionId);
- if (pool == null) {
- pool = new ConnectionPool(
- this.conf, nnAddress, ugi, this.minSize, this.maxSize);
- this.pools.put(connectionId, pool);
- }
- } finally {
- writeLock.unlock();
- }
- }
-
- ConnectionContext conn = pool.getConnection();
-
- // Add a new connection to the pool if it wasn't usable
- if (conn == null || !conn.isUsable()) {
- if (!this.creatorQueue.offer(pool)) {
- LOG.error("Cannot add more than {} connections at the same time",
- MAX_NEW_CONNECTIONS);
- }
- }
-
- if (conn != null && conn.isClosed()) {
- LOG.error("We got a closed connection from {}", pool);
- conn = null;
- }
-
- return conn;
- }
-
- /**
- * Get the number of connection pools.
- *
- * @return Number of connection pools.
- */
- public int getNumConnectionPools() {
- readLock.lock();
- try {
- return pools.size();
- } finally {
- readLock.unlock();
- }
- }
-
- /**
- * Get number of open connections.
- *
- * @return Number of open connections.
- */
- public int getNumConnections() {
- int total = 0;
- readLock.lock();
- try {
- for (ConnectionPool pool : this.pools.values()) {
- total += pool.getNumConnections();
- }
- } finally {
- readLock.unlock();
- }
- return total;
- }
-
- /**
- * Get number of active connections.
- *
- * @return Number of active connections.
- */
- public int getNumActiveConnections() {
- int total = 0;
- readLock.lock();
- try {
- for (ConnectionPool pool : this.pools.values()) {
- total += pool.getNumActiveConnections();
- }
- } finally {
- readLock.unlock();
- }
- return total;
- }
-
- /**
- * Get the number of connections to be created.
- *
- * @return Number of connections to be created.
- */
- public int getNumCreatingConnections() {
- return this.creatorQueue.size();
- }
-
- /**
- * Get a JSON representation of the connection pool.
- *
- * @return JSON representation of all the connection pools.
- */
- public String getJSON() {
- final Map<String, String> info = new TreeMap<>();
- readLock.lock();
- try {
- for (Entry<ConnectionPoolId, ConnectionPool> entry :
- this.pools.entrySet()) {
- ConnectionPoolId connectionPoolId = entry.getKey();
- ConnectionPool pool = entry.getValue();
- info.put(connectionPoolId.toString(), pool.getJSON());
- }
- } finally {
- readLock.unlock();
- }
- return JSON.toString(info);
- }
-
- @VisibleForTesting
- Map<ConnectionPoolId, ConnectionPool> getPools() {
- return this.pools;
- }
-
- /**
- * Clean the unused connections for this pool.
- *
- * @param pool Connection pool to cleanup.
- */
- @VisibleForTesting
- void cleanup(ConnectionPool pool) {
- if (pool.getNumConnections() > pool.getMinSize()) {
- // Check if the pool hasn't been active in a while or not 50% are used
- long timeSinceLastActive = Time.now() - pool.getLastActiveTime();
- int total = pool.getNumConnections();
- int active = pool.getNumActiveConnections();
- if (timeSinceLastActive > connectionCleanupPeriodMs ||
- active < MIN_ACTIVE_RATIO * total) {
- // Remove and close 1 connection
- List<ConnectionContext> conns = pool.removeConnections(1);
- for (ConnectionContext conn : conns) {
- conn.close();
- }
- LOG.debug("Removed connection {} used {} seconds ago. " +
- "Pool has {}/{} connections", pool.getConnectionPoolId(),
- TimeUnit.MILLISECONDS.toSeconds(timeSinceLastActive),
- pool.getNumConnections(), pool.getMaxSize());
- }
- }
- }
-
- /**
- * Removes stale connections not accessed recently from the pool. This is
- * invoked periodically.
- */
- private class CleanupTask implements Runnable {
-
- @Override
- public void run() {
- long currentTime = Time.now();
- List<ConnectionPoolId> toRemove = new LinkedList<>();
-
- // Look for stale pools
- readLock.lock();
- try {
- for (Entry<ConnectionPoolId, ConnectionPool> entry : pools.entrySet()) {
- ConnectionPool pool = entry.getValue();
- long lastTimeActive = pool.getLastActiveTime();
- boolean isStale =
- currentTime > (lastTimeActive + poolCleanupPeriodMs);
- if (lastTimeActive > 0 && isStale) {
- // Remove this pool
- LOG.debug("Closing and removing stale pool {}", pool);
- pool.close();
- ConnectionPoolId poolId = entry.getKey();
- toRemove.add(poolId);
- } else {
- // Keep this pool but clean connections inside
- LOG.debug("Cleaning up {}", pool);
- cleanup(pool);
- }
- }
- } finally {
- readLock.unlock();
- }
-
- // Remove stale pools
- if (!toRemove.isEmpty()) {
- writeLock.lock();
- try {
- for (ConnectionPoolId poolId : toRemove) {
- pools.remove(poolId);
- }
- } finally {
- writeLock.unlock();
- }
- }
- }
- }
-
- /**
- * Thread that creates connections asynchronously.
- */
- private static class ConnectionCreator extends Thread {
- /** If the creator is running. */
- private boolean running = true;
- /** Queue to push work to. */
- private BlockingQueue<ConnectionPool> queue;
-
- ConnectionCreator(BlockingQueue<ConnectionPool> blockingQueue) {
- super("Connection creator");
- this.queue = blockingQueue;
- }
-
- @Override
- public void run() {
- while (this.running) {
- try {
- ConnectionPool pool = this.queue.take();
- try {
- int total = pool.getNumConnections();
- int active = pool.getNumActiveConnections();
- if (pool.getNumConnections() < pool.getMaxSize() &&
- active >= MIN_ACTIVE_RATIO * total) {
- ConnectionContext conn = pool.newConnection();
- pool.addConnection(conn);
- } else {
- LOG.debug("Cannot add more than {} connections to {}",
- pool.getMaxSize(), pool);
- }
- } catch (IOException e) {
- LOG.error("Cannot create a new connection", e);
- }
- } catch (InterruptedException e) {
- LOG.error("The connection creator was interrupted");
- this.running = false;
- }
- }
- }
-
- /**
- * Stop this connection creator.
- */
- public void shutdown() {
- this.running = false;
- this.interrupt();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java
deleted file mode 100644
index 5af8a86..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java
+++ /dev/null
@@ -1,337 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import javax.net.SocketFactory;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.NameNodeProxiesClient.ProxyAndInfo;
-import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
-import org.apache.hadoop.hdfs.protocol.ClientProtocol;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB;
-import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.io.retry.RetryUtils;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.SaslRpcServer;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.Time;
-import org.eclipse.jetty.util.ajax.JSON;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Maintains a pool of connections for each User (including tokens) + NN. The
- * RPC client maintains a single socket, to achieve throughput similar to a NN,
- * each request is multiplexed across multiple sockets/connections from a
- * pool.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class ConnectionPool {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ConnectionPool.class);
-
-
- /** Configuration settings for the connection pool. */
- private final Configuration conf;
-
- /** Identifier for this connection pool. */
- private final ConnectionPoolId connectionPoolId;
- /** Namenode this pool connects to. */
- private final String namenodeAddress;
- /** User for this connections. */
- private final UserGroupInformation ugi;
-
- /** Pool of connections. We mimic a COW array. */
- private volatile List<ConnectionContext> connections = new ArrayList<>();
- /** Connection index for round-robin. */
- private final AtomicInteger clientIndex = new AtomicInteger(0);
-
- /** Min number of connections per user. */
- private final int minSize;
- /** Max number of connections per user. */
- private final int maxSize;
-
- /** The last time a connection was active. */
- private volatile long lastActiveTime = 0;
-
-
- protected ConnectionPool(Configuration config, String address,
- UserGroupInformation user, int minPoolSize, int maxPoolSize)
- throws IOException {
-
- this.conf = config;
-
- // Connection pool target
- this.ugi = user;
- this.namenodeAddress = address;
- this.connectionPoolId =
- new ConnectionPoolId(this.ugi, this.namenodeAddress);
-
- // Set configuration parameters for the pool
- this.minSize = minPoolSize;
- this.maxSize = maxPoolSize;
-
- // Add minimum connections to the pool
- for (int i=0; i<this.minSize; i++) {
- ConnectionContext newConnection = newConnection();
- this.connections.add(newConnection);
- }
- LOG.debug("Created connection pool \"{}\" with {} connections",
- this.connectionPoolId, this.minSize);
- }
-
- /**
- * Get the maximum number of connections allowed in this pool.
- *
- * @return Maximum number of connections.
- */
- protected int getMaxSize() {
- return this.maxSize;
- }
-
- /**
- * Get the minimum number of connections in this pool.
- *
- * @return Minimum number of connections.
- */
- protected int getMinSize() {
- return this.minSize;
- }
-
- /**
- * Get the connection pool identifier.
- *
- * @return Connection pool identifier.
- */
- protected ConnectionPoolId getConnectionPoolId() {
- return this.connectionPoolId;
- }
-
- /**
- * Return the next connection round-robin.
- *
- * @return Connection context.
- */
- protected ConnectionContext getConnection() {
-
- this.lastActiveTime = Time.now();
-
- // Get a connection from the pool following round-robin
- ConnectionContext conn = null;
- List<ConnectionContext> tmpConnections = this.connections;
- int size = tmpConnections.size();
- int threadIndex = this.clientIndex.getAndIncrement();
- for (int i=0; i<size; i++) {
- int index = (threadIndex + i) % size;
- conn = tmpConnections.get(index);
- if (conn != null && conn.isUsable()) {
- return conn;
- }
- }
-
- // We return a connection even if it's active
- return conn;
- }
-
- /**
- * Add a connection to the current pool. It uses a Copy-On-Write approach.
- *
- * @param conn New connection to add to the pool.
- */
- public synchronized void addConnection(ConnectionContext conn) {
- List<ConnectionContext> tmpConnections = new ArrayList<>(this.connections);
- tmpConnections.add(conn);
- this.connections = tmpConnections;
-
- this.lastActiveTime = Time.now();
- }
-
- /**
- * Remove connections from the current pool.
- *
- * @param num Number of connections to remove.
- * @return Removed connections.
- */
- public synchronized List<ConnectionContext> removeConnections(int num) {
- List<ConnectionContext> removed = new LinkedList<>();
-
- // Remove and close the last connection
- List<ConnectionContext> tmpConnections = new ArrayList<>();
- for (int i=0; i<this.connections.size(); i++) {
- ConnectionContext conn = this.connections.get(i);
- if (i < this.minSize || i < this.connections.size() - num) {
- tmpConnections.add(conn);
- } else {
- removed.add(conn);
- }
- }
- this.connections = tmpConnections;
-
- return removed;
- }
-
- /**
- * Close the connection pool.
- */
- protected synchronized void close() {
- long timeSinceLastActive = TimeUnit.MILLISECONDS.toSeconds(
- Time.now() - getLastActiveTime());
- LOG.debug("Shutting down connection pool \"{}\" used {} seconds ago",
- this.connectionPoolId, timeSinceLastActive);
-
- for (ConnectionContext connection : this.connections) {
- connection.close();
- }
- this.connections.clear();
- }
-
- /**
- * Number of connections in the pool.
- *
- * @return Number of connections.
- */
- protected int getNumConnections() {
- return this.connections.size();
- }
-
- /**
- * Number of active connections in the pool.
- *
- * @return Number of active connections.
- */
- protected int getNumActiveConnections() {
- int ret = 0;
-
- List<ConnectionContext> tmpConnections = this.connections;
- for (ConnectionContext conn : tmpConnections) {
- if (conn.isActive()) {
- ret++;
- }
- }
- return ret;
- }
-
- /**
- * Get the last time the connection pool was used.
- *
- * @return Last time the connection pool was used.
- */
- protected long getLastActiveTime() {
- return this.lastActiveTime;
- }
-
- @Override
- public String toString() {
- return this.connectionPoolId.toString();
- }
-
- /**
- * JSON representation of the connection pool.
- *
- * @return String representation of the JSON.
- */
- public String getJSON() {
- final Map<String, String> info = new LinkedHashMap<>();
- info.put("active", Integer.toString(getNumActiveConnections()));
- info.put("total", Integer.toString(getNumConnections()));
- if (LOG.isDebugEnabled()) {
- List<ConnectionContext> tmpConnections = this.connections;
- for (int i=0; i<tmpConnections.size(); i++) {
- ConnectionContext connection = tmpConnections.get(i);
- info.put(i + " active", Boolean.toString(connection.isActive()));
- info.put(i + " closed", Boolean.toString(connection.isClosed()));
- }
- }
- return JSON.toString(info);
- }
-
- /**
- * Create a new proxy wrapper for a client NN connection.
- * @return Proxy for the target ClientProtocol that contains the user's
- * security context.
- * @throws IOException
- */
- public ConnectionContext newConnection() throws IOException {
- return newConnection(this.conf, this.namenodeAddress, this.ugi);
- }
-
- /**
- * Creates a proxy wrapper for a client NN connection. Each proxy contains
- * context for a single user/security context. To maximize throughput it is
- * recommended to use multiple connection per user+server, allowing multiple
- * writes and reads to be dispatched in parallel.
- *
- * @param conf Configuration for the connection.
- * @param nnAddress Address of server supporting the ClientProtocol.
- * @param ugi User context.
- * @return Proxy for the target ClientProtocol that contains the user's
- * security context.
- * @throws IOException If it cannot be created.
- */
- protected static ConnectionContext newConnection(Configuration conf,
- String nnAddress, UserGroupInformation ugi)
- throws IOException {
- RPC.setProtocolEngine(
- conf, ClientNamenodeProtocolPB.class, ProtobufRpcEngine.class);
-
- final RetryPolicy defaultPolicy = RetryUtils.getDefaultRetryPolicy(
- conf,
- HdfsClientConfigKeys.Retry.POLICY_ENABLED_KEY,
- HdfsClientConfigKeys.Retry.POLICY_ENABLED_DEFAULT,
- HdfsClientConfigKeys.Retry.POLICY_SPEC_KEY,
- HdfsClientConfigKeys.Retry.POLICY_SPEC_DEFAULT,
- HdfsConstants.SAFEMODE_EXCEPTION_CLASS_NAME);
-
- SocketFactory factory = SocketFactory.getDefault();
- if (UserGroupInformation.isSecurityEnabled()) {
- SaslRpcServer.init(conf);
- }
- InetSocketAddress socket = NetUtils.createSocketAddr(nnAddress);
- final long version = RPC.getProtocolVersion(ClientNamenodeProtocolPB.class);
- ClientNamenodeProtocolPB proxy = RPC.getProtocolProxy(
- ClientNamenodeProtocolPB.class, version, socket, ugi, conf,
- factory, RPC.getRpcTimeout(conf), defaultPolicy, null).getProxy();
- ClientProtocol client = new ClientNamenodeProtocolTranslatorPB(proxy);
- Text dtService = SecurityUtil.buildTokenService(socket);
-
- ProxyAndInfo<ClientProtocol> clientProxy =
- new ProxyAndInfo<ClientProtocol>(client, dtService, socket);
- ConnectionContext connection = new ConnectionContext(clientProxy);
- return connection;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java
deleted file mode 100644
index 6e1ee9a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-
-/**
- * Identifier for a connection for a user to a namenode.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class ConnectionPoolId implements Comparable<ConnectionPoolId> {
-
- /** Namenode identifier. */
- private final String nnId;
- /** Information about the user. */
- private final UserGroupInformation ugi;
-
- /**
- * New connection pool identifier.
- *
- * @param ugi Information of the user issuing the request.
- * @param nnId Namenode address with port.
- */
- public ConnectionPoolId(final UserGroupInformation ugi, final String nnId) {
- this.nnId = nnId;
- this.ugi = ugi;
- }
-
- @Override
- public int hashCode() {
- int hash = new HashCodeBuilder(17, 31)
- .append(this.nnId)
- .append(this.ugi.toString())
- .append(this.getTokenIds())
- .toHashCode();
- return hash;
- }
-
- @Override
- public boolean equals(Object o) {
- if (o instanceof ConnectionPoolId) {
- ConnectionPoolId other = (ConnectionPoolId) o;
- if (!this.nnId.equals(other.nnId)) {
- return false;
- }
- if (!this.ugi.toString().equals(other.ugi.toString())) {
- return false;
- }
- String thisTokens = this.getTokenIds().toString();
- String otherTokens = other.getTokenIds().toString();
- return thisTokens.equals(otherTokens);
- }
- return false;
- }
-
- @Override
- public String toString() {
- return this.ugi + " " + this.getTokenIds() + "->" + this.nnId;
- }
-
- @Override
- public int compareTo(ConnectionPoolId other) {
- int ret = this.nnId.compareTo(other.nnId);
- if (ret == 0) {
- ret = this.ugi.toString().compareTo(other.ugi.toString());
- }
- if (ret == 0) {
- String thisTokens = this.getTokenIds().toString();
- String otherTokens = other.getTokenIds().toString();
- ret = thisTokens.compareTo(otherTokens);
- }
- return ret;
- }
-
- @VisibleForTesting
- UserGroupInformation getUgi() {
- return this.ugi;
- }
-
- /**
- * Get the token identifiers for this connection.
- * @return List with the token identifiers.
- */
- private List<String> getTokenIds() {
- List<String> tokenIds = new ArrayList<>();
- Collection<Token<? extends TokenIdentifier>> tokens = this.ugi.getTokens();
- for (Token<? extends TokenIdentifier> token : tokens) {
- byte[] tokenIdBytes = token.getIdentifier();
- String tokenId = Arrays.toString(tokenIdBytes);
- tokenIds.add(tokenId);
- }
- Collections.sort(tokenIds);
- return tokenIds;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/DFSRouter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/DFSRouter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/DFSRouter.java
deleted file mode 100644
index a2ac258..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/DFSRouter.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import static org.apache.hadoop.util.ExitUtil.terminate;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.service.CompositeService.CompositeServiceShutdownHook;
-import org.apache.hadoop.util.ShutdownHookManager;
-import org.apache.hadoop.util.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Tool to start the {@link Router} for Router-based federation.
- */
-public final class DFSRouter {
-
- private static final Logger LOG = LoggerFactory.getLogger(DFSRouter.class);
-
-
- /** Usage string for help message. */
- private static final String USAGE = "Usage: hdfs dfsrouter";
-
- /** Priority of the Router shutdown hook. */
- public static final int SHUTDOWN_HOOK_PRIORITY = 30;
-
-
- private DFSRouter() {
- // This is just a class to trigger the Router
- }
-
- /**
- * Main run loop for the router.
- *
- * @param argv parameters.
- */
- public static void main(String[] argv) {
- if (DFSUtil.parseHelpArgument(argv, USAGE, System.out, true)) {
- System.exit(0);
- }
-
- try {
- StringUtils.startupShutdownMessage(Router.class, argv, LOG);
-
- Router router = new Router();
-
- ShutdownHookManager.get().addShutdownHook(
- new CompositeServiceShutdownHook(router), SHUTDOWN_HOOK_PRIORITY);
-
- Configuration conf = new HdfsConfiguration();
- router.init(conf);
- router.start();
- } catch (Throwable e) {
- LOG.error("Failed to start router", e);
- terminate(1, e);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ErasureCoding.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ErasureCoding.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ErasureCoding.java
deleted file mode 100644
index d2b2d50..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ErasureCoding.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import static org.apache.hadoop.hdfs.server.federation.router.RouterRpcServer.merge;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse;
-import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats;
-import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
-import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
-import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
-import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
-import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
-
-/**
- * Module that implements all the RPC calls in
- * {@link org.apache.hadoop.hdfs.protocol.ClientProtocol} related to
- * Erasure Coding in the {@link RouterRpcServer}.
- */
-public class ErasureCoding {
-
- /** RPC server to receive client calls. */
- private final RouterRpcServer rpcServer;
- /** RPC clients to connect to the Namenodes. */
- private final RouterRpcClient rpcClient;
- /** Interface to identify the active NN for a nameservice or blockpool ID. */
- private final ActiveNamenodeResolver namenodeResolver;
-
-
- public ErasureCoding(RouterRpcServer server) {
- this.rpcServer = server;
- this.rpcClient = this.rpcServer.getRPCClient();
- this.namenodeResolver = this.rpcClient.getNamenodeResolver();
- }
-
- public ErasureCodingPolicyInfo[] getErasureCodingPolicies()
- throws IOException {
- rpcServer.checkOperation(OperationCategory.READ);
-
- RemoteMethod method = new RemoteMethod("getErasureCodingPolicies");
- Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
- Map<FederationNamespaceInfo, ErasureCodingPolicyInfo[]> ret =
- rpcClient.invokeConcurrent(
- nss, method, true, false, ErasureCodingPolicyInfo[].class);
- return merge(ret, ErasureCodingPolicyInfo.class);
- }
-
- public Map<String, String> getErasureCodingCodecs() throws IOException {
- rpcServer.checkOperation(OperationCategory.READ);
-
- RemoteMethod method = new RemoteMethod("getErasureCodingCodecs");
- Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
- @SuppressWarnings("rawtypes")
- Map<FederationNamespaceInfo, Map> retCodecs =
- rpcClient.invokeConcurrent(
- nss, method, true, false, Map.class);
-
- Map<String, String> ret = new HashMap<>();
- Object obj = retCodecs;
- @SuppressWarnings("unchecked")
- Map<FederationNamespaceInfo, Map<String, String>> results =
- (Map<FederationNamespaceInfo, Map<String, String>>)obj;
- Collection<Map<String, String>> allCodecs = results.values();
- for (Map<String, String> codecs : allCodecs) {
- ret.putAll(codecs);
- }
-
- return ret;
- }
-
- public AddErasureCodingPolicyResponse[] addErasureCodingPolicies(
- ErasureCodingPolicy[] policies) throws IOException {
- rpcServer.checkOperation(OperationCategory.WRITE);
-
- RemoteMethod method = new RemoteMethod("addErasureCodingPolicies",
- new Class<?>[] {ErasureCodingPolicy[].class}, new Object[] {policies});
- Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
- Map<FederationNamespaceInfo, AddErasureCodingPolicyResponse[]> ret =
- rpcClient.invokeConcurrent(
- nss, method, true, false, AddErasureCodingPolicyResponse[].class);
-
- return merge(ret, AddErasureCodingPolicyResponse.class);
- }
-
- public void removeErasureCodingPolicy(String ecPolicyName)
- throws IOException {
- rpcServer.checkOperation(OperationCategory.WRITE);
-
- RemoteMethod method = new RemoteMethod("removeErasureCodingPolicy",
- new Class<?>[] {String.class}, ecPolicyName);
- Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
- rpcClient.invokeConcurrent(nss, method, true, false);
- }
-
- public void disableErasureCodingPolicy(String ecPolicyName)
- throws IOException {
- rpcServer.checkOperation(OperationCategory.WRITE);
-
- RemoteMethod method = new RemoteMethod("disableErasureCodingPolicy",
- new Class<?>[] {String.class}, ecPolicyName);
- Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
- rpcClient.invokeConcurrent(nss, method, true, false);
- }
-
- public void enableErasureCodingPolicy(String ecPolicyName)
- throws IOException {
- rpcServer.checkOperation(OperationCategory.WRITE);
-
- RemoteMethod method = new RemoteMethod("enableErasureCodingPolicy",
- new Class<?>[] {String.class}, ecPolicyName);
- Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
- rpcClient.invokeConcurrent(nss, method, true, false);
- }
-
- public ErasureCodingPolicy getErasureCodingPolicy(String src)
- throws IOException {
- rpcServer.checkOperation(OperationCategory.READ);
-
- final List<RemoteLocation> locations =
- rpcServer.getLocationsForPath(src, true);
- RemoteMethod remoteMethod = new RemoteMethod("getErasureCodingPolicy",
- new Class<?>[] {String.class}, new RemoteParam());
- ErasureCodingPolicy ret = rpcClient.invokeSequential(
- locations, remoteMethod, null, null);
- return ret;
- }
-
- public void setErasureCodingPolicy(String src, String ecPolicyName)
- throws IOException {
- rpcServer.checkOperation(OperationCategory.WRITE);
-
- final List<RemoteLocation> locations =
- rpcServer.getLocationsForPath(src, true);
- RemoteMethod remoteMethod = new RemoteMethod("setErasureCodingPolicy",
- new Class<?>[] {String.class, String.class},
- new RemoteParam(), ecPolicyName);
- rpcClient.invokeSequential(locations, remoteMethod, null, null);
- }
-
- public void unsetErasureCodingPolicy(String src) throws IOException {
- rpcServer.checkOperation(OperationCategory.WRITE);
-
- final List<RemoteLocation> locations =
- rpcServer.getLocationsForPath(src, true);
- RemoteMethod remoteMethod = new RemoteMethod("unsetErasureCodingPolicy",
- new Class<?>[] {String.class}, new RemoteParam());
- rpcClient.invokeSequential(locations, remoteMethod, null, null);
- }
-
- public ECBlockGroupStats getECBlockGroupStats() throws IOException {
- rpcServer.checkOperation(OperationCategory.READ);
-
- RemoteMethod method = new RemoteMethod("getECBlockGroupStats");
- Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
- Map<FederationNamespaceInfo, ECBlockGroupStats> allStats =
- rpcClient.invokeConcurrent(
- nss, method, true, false, ECBlockGroupStats.class);
-
- // Merge the stats from all the namespaces
- long lowRedundancyBlockGroups = 0;
- long corruptBlockGroups = 0;
- long missingBlockGroups = 0;
- long bytesInFutureBlockGroups = 0;
- long pendingDeletionBlocks = 0;
- for (ECBlockGroupStats stats : allStats.values()) {
- lowRedundancyBlockGroups += stats.getLowRedundancyBlockGroups();
- corruptBlockGroups += stats.getCorruptBlockGroups();
- missingBlockGroups += stats.getMissingBlockGroups();
- bytesInFutureBlockGroups += stats.getBytesInFutureBlockGroups();
- pendingDeletionBlocks += stats.getPendingDeletionBlocks();
- }
- return new ECBlockGroupStats(lowRedundancyBlockGroups, corruptBlockGroups,
- missingBlockGroups, bytesInFutureBlockGroups, pendingDeletionBlocks);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
deleted file mode 100644
index 3dfd998..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
+++ /dev/null
@@ -1,209 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.lang.reflect.Constructor;
-import java.net.URL;
-import java.net.URLConnection;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
-import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
-import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
-import org.apache.hadoop.util.VersionInfo;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Utilities for managing HDFS federation.
- */
-public final class FederationUtil {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(FederationUtil.class);
-
- private FederationUtil() {
- // Utility Class
- }
-
- /**
- * Get a JMX data from a web endpoint.
- *
- * @param beanQuery JMX bean.
- * @param webAddress Web address of the JMX endpoint.
- * @return JSON with the JMX data
- */
- public static JSONArray getJmx(String beanQuery, String webAddress) {
- JSONArray ret = null;
- BufferedReader reader = null;
- try {
- String host = webAddress;
- int port = -1;
- if (webAddress.indexOf(":") > 0) {
- String[] webAddressSplit = webAddress.split(":");
- host = webAddressSplit[0];
- port = Integer.parseInt(webAddressSplit[1]);
- }
- URL jmxURL = new URL("http", host, port, "/jmx?qry=" + beanQuery);
- URLConnection conn = jmxURL.openConnection();
- conn.setConnectTimeout(5 * 1000);
- conn.setReadTimeout(5 * 1000);
- InputStream in = conn.getInputStream();
- InputStreamReader isr = new InputStreamReader(in, "UTF-8");
- reader = new BufferedReader(isr);
-
- StringBuilder sb = new StringBuilder();
- String line = null;
- while ((line = reader.readLine()) != null) {
- sb.append(line);
- }
- String jmxOutput = sb.toString();
-
- // Parse JSON
- JSONObject json = new JSONObject(jmxOutput);
- ret = json.getJSONArray("beans");
- } catch (IOException e) {
- LOG.error("Cannot read JMX bean {} from server {}: {}",
- beanQuery, webAddress, e.getMessage());
- } catch (JSONException e) {
- LOG.error("Cannot parse JMX output for {} from server {}: {}",
- beanQuery, webAddress, e.getMessage());
- } catch (Exception e) {
- LOG.error("Cannot parse JMX output for {} from server {}: {}",
- beanQuery, webAddress, e);
- } finally {
- if (reader != null) {
- try {
- reader.close();
- } catch (IOException e) {
- LOG.error("Problem closing {}", webAddress, e);
- }
- }
- }
- return ret;
- }
-
- /**
- * Fetch the Hadoop version string for this jar.
- *
- * @return Hadoop version string, e.g., 3.0.1.
- */
- public static String getVersion() {
- return VersionInfo.getVersion();
- }
-
- /**
- * Fetch the build/compile information for this jar.
- *
- * @return String Compilation info.
- */
- public static String getCompileInfo() {
- return VersionInfo.getDate() + " by " + VersionInfo.getUser() + " from "
- + VersionInfo.getBranch();
- }
-
- /**
- * Create an instance of an interface with a constructor using a context.
- *
- * @param conf Configuration for the class names.
- * @param context Context object to pass to the instance.
- * @param contextClass Type of the context passed to the constructor.
- * @param clazz Class of the object to return.
- * @return New instance of the specified class that implements the desired
- * interface and a single parameter constructor containing a
- * StateStore reference.
- */
- private static <T, R> T newInstance(final Configuration conf,
- final R context, final Class<R> contextClass, final Class<T> clazz) {
- try {
- if (contextClass == null) {
- // Default constructor if no context
- Constructor<T> constructor = clazz.getConstructor();
- return constructor.newInstance();
- } else {
- // Constructor with context
- Constructor<T> constructor = clazz.getConstructor(
- Configuration.class, contextClass);
- return constructor.newInstance(conf, context);
- }
- } catch (ReflectiveOperationException e) {
- LOG.error("Could not instantiate: {}", clazz.getSimpleName(), e);
- return null;
- }
- }
-
- /**
- * Creates an instance of a FileSubclusterResolver from the configuration.
- *
- * @param conf Configuration that defines the file resolver class.
- * @param router Router service.
- * @return New file subcluster resolver.
- */
- public static FileSubclusterResolver newFileSubclusterResolver(
- Configuration conf, Router router) {
- Class<? extends FileSubclusterResolver> clazz = conf.getClass(
- DFSConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
- DFSConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS_DEFAULT,
- FileSubclusterResolver.class);
- return newInstance(conf, router, Router.class, clazz);
- }
-
- /**
- * Creates an instance of an ActiveNamenodeResolver from the configuration.
- *
- * @param conf Configuration that defines the namenode resolver class.
- * @param stateStore State store passed to class constructor.
- * @return New active namenode resolver.
- */
- public static ActiveNamenodeResolver newActiveNamenodeResolver(
- Configuration conf, StateStoreService stateStore) {
- Class<? extends ActiveNamenodeResolver> clazz = conf.getClass(
- DFSConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS,
- DFSConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS_DEFAULT,
- ActiveNamenodeResolver.class);
- return newInstance(conf, stateStore, StateStoreService.class, clazz);
- }
-
- /**
- * Check if the given path is the child of parent path.
- * @param path Path to be check.
- * @param parent Parent path.
- * @return True if parent path is parent entry for given path.
- */
- public static boolean isParentEntry(final String path, final String parent) {
- if (!path.startsWith(parent)) {
- return false;
- }
-
- if (path.equals(parent)) {
- return true;
- }
-
- return path.charAt(parent.length()) == Path.SEPARATOR_CHAR
- || parent.equals(Path.SEPARATOR);
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java
new file mode 100644
index 0000000..9667489
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java
@@ -0,0 +1,317 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records.impl.pb;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProto.DestOrder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProtoOrBuilder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoteLocationProto;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.QuotaUsageProto;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
+import org.apache.hadoop.hdfs.server.federation.router.RouterAdminServer;
+import org.apache.hadoop.hdfs.server.federation.router.RouterPermissionChecker;
+import org.apache.hadoop.hdfs.server.federation.router.RouterQuotaUsage;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.FederationProtocolPBTranslator;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the MountTable record.
+ */
+public class MountTablePBImpl extends MountTable implements PBRecord {
+
+ private FederationProtocolPBTranslator<MountTableRecordProto, Builder,
+ MountTableRecordProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<MountTableRecordProto, Builder,
+ MountTableRecordProtoOrBuilder>(MountTableRecordProto.class);
+
+ public MountTablePBImpl() {
+ }
+
+ public MountTablePBImpl(MountTableRecordProto proto) {
+ this.setProto(proto);
+ }
+
+ @Override
+ public MountTableRecordProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public String getSourcePath() {
+ MountTableRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
+ if (!proto.hasSrcPath()) {
+ return null;
+ }
+ return proto.getSrcPath();
+ }
+
+ @Override
+ public void setSourcePath(String path) {
+ Builder builder = this.translator.getBuilder();
+ if (path == null) {
+ builder.clearSrcPath();
+ } else {
+ builder.setSrcPath(path);
+ }
+ }
+
+ @Override
+ public List<RemoteLocation> getDestinations() {
+ MountTableRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
+ if (proto.getDestinationsCount() == 0) {
+ return null;
+ }
+
+ final List<RemoteLocation> ret = new LinkedList<>();
+ final List<RemoteLocationProto> destList = proto.getDestinationsList();
+ for (RemoteLocationProto dest : destList) {
+ String nsId = dest.getNameserviceId();
+ String path = dest.getPath();
+ RemoteLocation loc = new RemoteLocation(nsId, path);
+ ret.add(loc);
+ }
+ return ret;
+ }
+
+ @Override
+ public void setDestinations(final List<RemoteLocation> dests) {
+ Builder builder = this.translator.getBuilder();
+ builder.clearDestinations();
+ for (RemoteLocation dest : dests) {
+ RemoteLocationProto.Builder itemBuilder =
+ RemoteLocationProto.newBuilder();
+ String nsId = dest.getNameserviceId();
+ String path = dest.getDest();
+ itemBuilder.setNameserviceId(nsId);
+ itemBuilder.setPath(path);
+ RemoteLocationProto item = itemBuilder.build();
+ builder.addDestinations(item);
+ }
+ }
+
+ @Override
+ public boolean addDestination(String nsId, String path) {
+ // Check if the location is already there
+ List<RemoteLocation> dests = getDestinations();
+ for (RemoteLocation dest : dests) {
+ if (dest.getNameserviceId().equals(nsId) && dest.getDest().equals(path)) {
+ return false;
+ }
+ }
+
+ // Add it to the existing list
+ Builder builder = this.translator.getBuilder();
+ RemoteLocationProto.Builder itemBuilder =
+ RemoteLocationProto.newBuilder();
+ itemBuilder.setNameserviceId(nsId);
+ itemBuilder.setPath(path);
+ RemoteLocationProto item = itemBuilder.build();
+ builder.addDestinations(item);
+ return true;
+ }
+
+ @Override
+ public void setDateModified(long time) {
+ this.translator.getBuilder().setDateModified(time);
+ }
+
+ @Override
+ public long getDateModified() {
+ return this.translator.getProtoOrBuilder().getDateModified();
+ }
+
+ @Override
+ public void setDateCreated(long time) {
+ this.translator.getBuilder().setDateCreated(time);
+ }
+
+ @Override
+ public long getDateCreated() {
+ return this.translator.getProtoOrBuilder().getDateCreated();
+ }
+
+ @Override
+ public boolean isReadOnly() {
+ MountTableRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
+ if (!proto.hasReadOnly()) {
+ return false;
+ }
+ return proto.getReadOnly();
+ }
+
+ @Override
+ public void setReadOnly(boolean ro) {
+ this.translator.getBuilder().setReadOnly(ro);
+ }
+
+ @Override
+ public DestinationOrder getDestOrder() {
+ MountTableRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
+ return convert(proto.getDestOrder());
+ }
+
+ @Override
+ public void setDestOrder(DestinationOrder order) {
+ Builder builder = this.translator.getBuilder();
+ if (order == null) {
+ builder.clearDestOrder();
+ } else {
+ builder.setDestOrder(convert(order));
+ }
+ }
+
+ @Override
+ public String getOwnerName() {
+ MountTableRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
+ if (!proto.hasOwnerName()) {
+ return RouterAdminServer.getSuperUser();
+ }
+ return proto.getOwnerName();
+ }
+
+ @Override
+ public void setOwnerName(String owner) {
+ Builder builder = this.translator.getBuilder();
+ if (owner == null) {
+ builder.clearOwnerName();
+ } else {
+ builder.setOwnerName(owner);
+ }
+ }
+
+ @Override
+ public String getGroupName() {
+ MountTableRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
+ if (!proto.hasGroupName()) {
+ return RouterAdminServer.getSuperGroup();
+ }
+ return proto.getGroupName();
+ }
+
+ @Override
+ public void setGroupName(String group) {
+ Builder builder = this.translator.getBuilder();
+ if (group == null) {
+ builder.clearGroupName();
+ } else {
+ builder.setGroupName(group);
+ }
+ }
+
+ @Override
+ public FsPermission getMode() {
+ MountTableRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
+ short mode = RouterPermissionChecker.MOUNT_TABLE_PERMISSION_DEFAULT;
+ if (proto.hasMode()) {
+ mode = (short) proto.getMode();
+ }
+ return new FsPermission(mode);
+ }
+
+ @Override
+ public void setMode(FsPermission mode) {
+ Builder builder = this.translator.getBuilder();
+ if (mode == null) {
+ builder.clearMode();
+ } else {
+ builder.setMode(mode.toShort());
+ }
+ }
+
+ @Override
+ public RouterQuotaUsage getQuota() {
+ MountTableRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
+
+ long nsQuota = HdfsConstants.QUOTA_DONT_SET;
+ long nsCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
+ long ssQuota = HdfsConstants.QUOTA_DONT_SET;
+ long ssCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
+ if (proto.hasQuota()) {
+ QuotaUsageProto quotaProto = proto.getQuota();
+ nsQuota = quotaProto.getQuota();
+ nsCount = quotaProto.getFileAndDirectoryCount();
+ ssQuota = quotaProto.getSpaceQuota();
+ ssCount = quotaProto.getSpaceConsumed();
+ }
+
+ RouterQuotaUsage.Builder builder = new RouterQuotaUsage.Builder()
+ .quota(nsQuota).fileAndDirectoryCount(nsCount).spaceQuota(ssQuota)
+ .spaceConsumed(ssCount);
+ return builder.build();
+ }
+
+ @Override
+ public void setQuota(RouterQuotaUsage quota) {
+ Builder builder = this.translator.getBuilder();
+ if (quota == null) {
+ builder.clearQuota();
+ } else {
+ QuotaUsageProto quotaUsage = QuotaUsageProto.newBuilder()
+ .setFileAndDirectoryCount(quota.getFileAndDirectoryCount())
+ .setQuota(quota.getQuota()).setSpaceConsumed(quota.getSpaceConsumed())
+ .setSpaceQuota(quota.getSpaceQuota()).build();
+ builder.setQuota(quotaUsage);
+ }
+ }
+
+ private DestinationOrder convert(DestOrder order) {
+ switch (order) {
+ case LOCAL:
+ return DestinationOrder.LOCAL;
+ case RANDOM:
+ return DestinationOrder.RANDOM;
+ case HASH_ALL:
+ return DestinationOrder.HASH_ALL;
+ default:
+ return DestinationOrder.HASH;
+ }
+ }
+
+ private DestOrder convert(DestinationOrder order) {
+ switch (order) {
+ case LOCAL:
+ return DestOrder.LOCAL;
+ case RANDOM:
+ return DestOrder.RANDOM;
+ case HASH_ALL:
+ return DestOrder.HASH_ALL;
+ default:
+ return DestOrder.HASH;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/PBRecord.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/PBRecord.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/PBRecord.java
new file mode 100644
index 0000000..c369275
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/PBRecord.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records.impl.pb;
+
+import java.io.IOException;
+
+import com.google.protobuf.Message;
+
+/**
+ * A record implementation using Protobuf.
+ */
+public interface PBRecord {
+
+ /**
+ * Get the protocol for the record.
+ * @return The protocol for this record.
+ */
+ Message getProto();
+
+ /**
+ * Set the protocol for the record.
+ * @param proto Protocol for this record.
+ */
+ void setProto(Message proto);
+
+ /**
+ * Populate this record with serialized data.
+ * @param base64String Serialized data in base64.
+ * @throws IOException If it cannot read the data.
+ */
+ void readInstance(String base64String) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/RouterStatePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/RouterStatePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/RouterStatePBImpl.java
new file mode 100644
index 0000000..23a61f9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/RouterStatePBImpl.java
@@ -0,0 +1,202 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterRecordProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterRecordProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterRecordProtoOrBuilder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.StateStoreVersionRecordProto;
+import org.apache.hadoop.hdfs.server.federation.router.RouterServiceState;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.FederationProtocolPBTranslator;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+import org.apache.hadoop.hdfs.server.federation.store.records.StateStoreVersion;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the RouterState record.
+ */
+public class RouterStatePBImpl extends RouterState implements PBRecord {
+
+ private FederationProtocolPBTranslator<RouterRecordProto, Builder,
+ RouterRecordProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<RouterRecordProto, Builder,
+ RouterRecordProtoOrBuilder>(RouterRecordProto.class);
+
+ public RouterStatePBImpl() {
+ }
+
+ public RouterStatePBImpl(RouterRecordProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public RouterRecordProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public void setAddress(String address) {
+ RouterRecordProto.Builder builder = this.translator.getBuilder();
+ if (address == null) {
+ builder.clearAddress();
+ } else {
+ builder.setAddress(address);
+ }
+ }
+
+ @Override
+ public String getAddress() {
+ RouterRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
+ if (!proto.hasAddress()) {
+ return null;
+ }
+ return proto.getAddress();
+ }
+
+ @Override
+ public void setStateStoreVersion(StateStoreVersion version) {
+ RouterRecordProto.Builder builder = this.translator.getBuilder();
+ if (version instanceof StateStoreVersionPBImpl) {
+ StateStoreVersionPBImpl versionPB = (StateStoreVersionPBImpl)version;
+ StateStoreVersionRecordProto versionProto =
+ (StateStoreVersionRecordProto)versionPB.getProto();
+ builder.setStateStoreVersion(versionProto);
+ } else {
+ builder.clearStateStoreVersion();
+ }
+ }
+
+ @Override
+ public StateStoreVersion getStateStoreVersion() throws IOException {
+ RouterRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
+ if (!proto.hasStateStoreVersion()) {
+ return null;
+ }
+ StateStoreVersionRecordProto versionProto = proto.getStateStoreVersion();
+ StateStoreVersion version =
+ StateStoreSerializer.newRecord(StateStoreVersion.class);
+ if (version instanceof StateStoreVersionPBImpl) {
+ StateStoreVersionPBImpl versionPB = (StateStoreVersionPBImpl)version;
+ versionPB.setProto(versionProto);
+ return versionPB;
+ } else {
+ throw new IOException("Cannot get State Store version");
+ }
+ }
+
+ @Override
+ public RouterServiceState getStatus() {
+ RouterRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
+ if (!proto.hasStatus()) {
+ return null;
+ }
+ return RouterServiceState.valueOf(proto.getStatus());
+ }
+
+ @Override
+ public void setStatus(RouterServiceState newStatus) {
+ RouterRecordProto.Builder builder = this.translator.getBuilder();
+ if (newStatus == null) {
+ builder.clearStatus();
+ } else {
+ builder.setStatus(newStatus.toString());
+ }
+ }
+
+ @Override
+ public String getVersion() {
+ RouterRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
+ if (!proto.hasVersion()) {
+ return null;
+ }
+ return proto.getVersion();
+ }
+
+ @Override
+ public void setVersion(String version) {
+ RouterRecordProto.Builder builder = this.translator.getBuilder();
+ if (version == null) {
+ builder.clearVersion();
+ } else {
+ builder.setVersion(version);
+ }
+ }
+
+ @Override
+ public String getCompileInfo() {
+ RouterRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
+ if (!proto.hasCompileInfo()) {
+ return null;
+ }
+ return proto.getCompileInfo();
+ }
+
+ @Override
+ public void setCompileInfo(String info) {
+ RouterRecordProto.Builder builder = this.translator.getBuilder();
+ if (info == null) {
+ builder.clearCompileInfo();
+ } else {
+ builder.setCompileInfo(info);
+ }
+ }
+
+ @Override
+ public void setDateStarted(long dateStarted) {
+ this.translator.getBuilder().setDateStarted(dateStarted);
+ }
+
+ @Override
+ public long getDateStarted() {
+ return this.translator.getProtoOrBuilder().getDateStarted();
+ }
+
+ @Override
+ public void setDateModified(long time) {
+ this.translator.getBuilder().setDateModified(time);
+ }
+
+ @Override
+ public long getDateModified() {
+ return this.translator.getProtoOrBuilder().getDateModified();
+ }
+
+ @Override
+ public void setDateCreated(long time) {
+ this.translator.getBuilder().setDateCreated(time);
+ }
+
+ @Override
+ public long getDateCreated() {
+ return this.translator.getProtoOrBuilder().getDateCreated();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/StateStoreVersionPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/StateStoreVersionPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/StateStoreVersionPBImpl.java
new file mode 100644
index 0000000..7696136
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/StateStoreVersionPBImpl.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.StateStoreVersionRecordProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.StateStoreVersionRecordProto.Builder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.StateStoreVersionRecordProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.FederationProtocolPBTranslator;
+import org.apache.hadoop.hdfs.server.federation.store.records.StateStoreVersion;
+
+import com.google.protobuf.Message;
+
+/**
+ * Protobuf implementation of the StateStoreVersion record.
+ */
+public class StateStoreVersionPBImpl extends StateStoreVersion
+ implements PBRecord {
+
+ private FederationProtocolPBTranslator<StateStoreVersionRecordProto, Builder,
+ StateStoreVersionRecordProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<StateStoreVersionRecordProto,
+ Builder, StateStoreVersionRecordProtoOrBuilder>(
+ StateStoreVersionRecordProto.class);
+
+ public StateStoreVersionPBImpl() {
+ }
+
+ @Override
+ public StateStoreVersionRecordProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public long getMembershipVersion() {
+ return this.translator.getProtoOrBuilder().getMembershipVersion();
+ }
+
+ @Override
+ public void setMembershipVersion(long version) {
+ this.translator.getBuilder().setMembershipVersion(version);
+ }
+
+ @Override
+ public long getMountTableVersion() {
+ return this.translator.getProtoOrBuilder().getMountTableVersion();
+ }
+
+ @Override
+ public void setMountTableVersion(long version) {
+ this.translator.getBuilder().setMountTableVersion(version);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/package-info.java
new file mode 100644
index 0000000..b329732
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/package-info.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The protobuf implementations of state store data records defined in the
+ * org.apache.hadoop.hdfs.server.federation.store.records package. Each
+ * implementation wraps an associated protobuf proto definition.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+package org.apache.hadoop.hdfs.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/package-info.java
new file mode 100644
index 0000000..63b13af
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/package-info.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Contains the abstract definitions of the state store data records. The state
+ * store supports multiple multiple data records.
+ * <p>
+ * Data records inherit from a common class
+ * {@link org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord
+ * BaseRecord}. Data records are serialized when written to the data store using
+ * a modular serialization implementation. The default is profobuf
+ * serialization. Data is stored as rows of records of the same type with each
+ * data member in a record representing a column.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/utils/ConsistentHashRing.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/utils/ConsistentHashRing.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/utils/ConsistentHashRing.java
new file mode 100644
index 0000000..89273db
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/utils/ConsistentHashRing.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.utils;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.hadoop.io.MD5Hash;
+
+/**
+ * Consistent hash ring to distribute items across nodes (locations). If we add
+ * or remove nodes, it minimizes the item migration.
+ */
+public class ConsistentHashRing {
+ private static final String SEPERATOR = "/";
+ private static final String VIRTUAL_NODE_FORMAT = "%s" + SEPERATOR + "%d";
+
+ /** Hash ring. */
+ private SortedMap<String, String> ring = new TreeMap<String, String>();
+ /** Entry -> num virtual nodes on ring. */
+ private Map<String, Integer> entryToVirtualNodes =
+ new HashMap<String, Integer>();
+
+ /** Synchronization. */
+ private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ private final Lock readLock = readWriteLock.readLock();
+ private final Lock writeLock = readWriteLock.writeLock();
+
+ public ConsistentHashRing(Set<String> locations) {
+ for (String location : locations) {
+ addLocation(location);
+ }
+ }
+
+ /**
+ * Add entry to consistent hash ring.
+ *
+ * @param location Node to add to the ring.
+ */
+ public void addLocation(String location) {
+ addLocation(location, 100);
+ }
+
+ /**
+ * Add entry to consistent hash ring.
+ *
+ * @param location Node to add to the ring.
+ * @param numVirtualNodes Number of virtual nodes to add.
+ */
+ public void addLocation(String location, int numVirtualNodes) {
+ writeLock.lock();
+ try {
+ entryToVirtualNodes.put(location, numVirtualNodes);
+ for (int i = 0; i < numVirtualNodes; i++) {
+ String key = String.format(VIRTUAL_NODE_FORMAT, location, i);
+ String hash = getHash(key);
+ ring.put(hash, key);
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /**
+ * Remove specified entry from hash ring.
+ *
+ * @param location Node to remove from the ring.
+ */
+ public void removeLocation(String location) {
+ writeLock.lock();
+ try {
+ Integer numVirtualNodes = entryToVirtualNodes.remove(location);
+ for (int i = 0; i < numVirtualNodes; i++) {
+ String key = String.format(VIRTUAL_NODE_FORMAT, location, i);
+ String hash = getHash(key);
+ ring.remove(hash);
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /**
+ * Return location (owner) of specified item. Owner is the next
+ * entry on the hash ring (with a hash value > hash value of item).
+ * @param item Item to look for.
+ * @return The location of the item.
+ */
+ public String getLocation(String item) {
+ readLock.lock();
+ try {
+ if (ring.isEmpty()) {
+ return null;
+ }
+ String hash = getHash(item);
+ if (!ring.containsKey(hash)) {
+ SortedMap<String, String> tailMap = ring.tailMap(hash);
+ hash = tailMap.isEmpty() ? ring.firstKey() : tailMap.firstKey();
+ }
+ String virtualNode = ring.get(hash);
+ int index = virtualNode.lastIndexOf(SEPERATOR);
+ if (index >= 0) {
+ return virtualNode.substring(0, index);
+ } else {
+ return virtualNode;
+ }
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ public String getHash(String key) {
+ return MD5Hash.digest(key).toString();
+ }
+
+ /**
+ * Get the locations in the ring.
+ * @return Set of locations in the ring.
+ */
+ public Set<String> getLocations() {
+ return entryToVirtualNodes.keySet();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/utils/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/utils/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/utils/package-info.java
new file mode 100644
index 0000000..7149675
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/utils/package-info.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Assorted utility classes and helpers for HDFS Federation.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+
+package org.apache.hadoop.hdfs.server.federation.utils;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
new file mode 100644
index 0000000..f4adbad
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
@@ -0,0 +1,627 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.federation;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
+import org.apache.hadoop.hdfs.server.federation.router.RouterClient;
+import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.router.RouterQuotaUsage;
+import org.apache.hadoop.hdfs.server.federation.router.RouterStateManager;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class provides some Federation administrative access shell commands.
+ */
+@Private
+public class RouterAdmin extends Configured implements Tool {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RouterAdmin.class);
+
+ private RouterClient client;
+
+ public static void main(String[] argv) throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ RouterAdmin admin = new RouterAdmin(conf);
+
+ int res = ToolRunner.run(admin, argv);
+ System.exit(res);
+ }
+
+ public RouterAdmin(Configuration conf) {
+ super(conf);
+ }
+
+ /**
+ * Print the usage message.
+ */
+ public void printUsage() {
+ String usage = "Federation Admin Tools:\n"
+ + "\t[-add <source> <nameservice> <destination> "
+ + "[-readonly] [-order HASH|LOCAL|RANDOM|HASH_ALL] "
+ + "-owner <owner> -group <group> -mode <mode>]\n"
+ + "\t[-rm <source>]\n"
+ + "\t[-ls <path>]\n"
+ + "\t[-setQuota <path> -nsQuota <nsQuota> -ssQuota "
+ + "<quota in bytes or quota size string>]\n"
+ + "\t[-clrQuota <path>]\n"
+ + "\t[-safemode enter | leave | get]\n";
+
+ System.out.println(usage);
+ }
+
+ @Override
+ public int run(String[] argv) throws Exception {
+ if (argv.length < 1) {
+ System.err.println("Not enough parameters specificed");
+ printUsage();
+ return -1;
+ }
+
+ int exitCode = -1;
+ int i = 0;
+ String cmd = argv[i++];
+
+ // Verify that we have enough command line parameters
+ if ("-add".equals(cmd)) {
+ if (argv.length < 4) {
+ System.err.println("Not enough parameters specificed for cmd " + cmd);
+ printUsage();
+ return exitCode;
+ }
+ } else if ("-rm".equalsIgnoreCase(cmd)) {
+ if (argv.length < 2) {
+ System.err.println("Not enough parameters specificed for cmd " + cmd);
+ printUsage();
+ return exitCode;
+ }
+ } else if ("-setQuota".equalsIgnoreCase(cmd)) {
+ if (argv.length < 4) {
+ System.err.println("Not enough parameters specificed for cmd " + cmd);
+ printUsage();
+ return exitCode;
+ }
+ } else if ("-clrQuota".equalsIgnoreCase(cmd)) {
+ if (argv.length < 2) {
+ System.err.println("Not enough parameters specificed for cmd " + cmd);
+ printUsage();
+ return exitCode;
+ }
+ } else if ("-safemode".equalsIgnoreCase(cmd)) {
+ if (argv.length < 2) {
+ System.err.println("Not enough parameters specificed for cmd " + cmd);
+ printUsage();
+ return exitCode;
+ }
+ }
+
+ // Initialize RouterClient
+ try {
+ String address = getConf().getTrimmed(
+ RBFConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_KEY,
+ RBFConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_DEFAULT);
+ InetSocketAddress routerSocket = NetUtils.createSocketAddr(address);
+ client = new RouterClient(routerSocket, getConf());
+ } catch (RPC.VersionMismatch v) {
+ System.err.println(
+ "Version mismatch between client and server... command aborted");
+ return exitCode;
+ } catch (IOException e) {
+ System.err.println("Bad connection to Router... command aborted");
+ return exitCode;
+ }
+
+ Exception debugException = null;
+ exitCode = 0;
+ try {
+ if ("-add".equals(cmd)) {
+ if (addMount(argv, i)) {
+ System.out.println("Successfuly added mount point " + argv[i]);
+ }
+ } else if ("-rm".equals(cmd)) {
+ if (removeMount(argv[i])) {
+ System.out.println("Successfully removed mount point " + argv[i]);
+ }
+ } else if ("-ls".equals(cmd)) {
+ if (argv.length > 1) {
+ listMounts(argv[i]);
+ } else {
+ listMounts("/");
+ }
+ } else if ("-setQuota".equals(cmd)) {
+ if (setQuota(argv, i)) {
+ System.out.println(
+ "Successfully set quota for mount point " + argv[i]);
+ }
+ } else if ("-clrQuota".equals(cmd)) {
+ if (clrQuota(argv[i])) {
+ System.out.println(
+ "Successfully clear quota for mount point " + argv[i]);
+ }
+ } else if ("-safemode".equals(cmd)) {
+ manageSafeMode(argv[i]);
+ } else {
+ printUsage();
+ return exitCode;
+ }
+ } catch (IllegalArgumentException arge) {
+ debugException = arge;
+ exitCode = -1;
+ System.err.println(cmd.substring(1) + ": " + arge.getLocalizedMessage());
+ printUsage();
+ } catch (RemoteException e) {
+ // This is a error returned by the server.
+ // Print out the first line of the error message, ignore the stack trace.
+ exitCode = -1;
+ debugException = e;
+ try {
+ String[] content;
+ content = e.getLocalizedMessage().split("\n");
+ System.err.println(cmd.substring(1) + ": " + content[0]);
+ e.printStackTrace();
+ } catch (Exception ex) {
+ System.err.println(cmd.substring(1) + ": " + ex.getLocalizedMessage());
+ e.printStackTrace();
+ debugException = ex;
+ }
+ } catch (Exception e) {
+ exitCode = -1;
+ debugException = e;
+ System.err.println(cmd.substring(1) + ": " + e.getLocalizedMessage());
+ e.printStackTrace();
+ }
+ if (debugException != null) {
+ LOG.debug("Exception encountered", debugException);
+ }
+ return exitCode;
+ }
+
+ /**
+ * Add a mount table entry or update if it exists.
+ *
+ * @param parameters Parameters for the mount point.
+ * @param i Index in the parameters.
+ */
+ public boolean addMount(String[] parameters, int i) throws IOException {
+ // Mandatory parameters
+ String mount = parameters[i++];
+ String[] nss = parameters[i++].split(",");
+ String dest = parameters[i++];
+
+ // Optional parameters
+ boolean readOnly = false;
+ String owner = null;
+ String group = null;
+ FsPermission mode = null;
+ DestinationOrder order = DestinationOrder.HASH;
+ while (i < parameters.length) {
+ if (parameters[i].equals("-readonly")) {
+ readOnly = true;
+ } else if (parameters[i].equals("-order")) {
+ i++;
+ try {
+ order = DestinationOrder.valueOf(parameters[i]);
+ } catch(Exception e) {
+ System.err.println("Cannot parse order: " + parameters[i]);
+ }
+ } else if (parameters[i].equals("-owner")) {
+ i++;
+ owner = parameters[i];
+ } else if (parameters[i].equals("-group")) {
+ i++;
+ group = parameters[i];
+ } else if (parameters[i].equals("-mode")) {
+ i++;
+ short modeValue = Short.parseShort(parameters[i], 8);
+ mode = new FsPermission(modeValue);
+ }
+
+ i++;
+ }
+
+ return addMount(mount, nss, dest, readOnly, order,
+ new ACLEntity(owner, group, mode));
+ }
+
+ /**
+ * Add a mount table entry or update if it exists.
+ *
+ * @param mount Mount point.
+ * @param nss Namespaces where this is mounted to.
+ * @param dest Destination path.
+ * @param readonly If the mount point is read only.
+ * @param order Order of the destination locations.
+ * @param aclInfo the ACL info for mount point.
+ * @return If the mount point was added.
+ * @throws IOException Error adding the mount point.
+ */
+ public boolean addMount(String mount, String[] nss, String dest,
+ boolean readonly, DestinationOrder order, ACLEntity aclInfo)
+ throws IOException {
+ // Get the existing entry
+ MountTableManager mountTable = client.getMountTableManager();
+ GetMountTableEntriesRequest getRequest =
+ GetMountTableEntriesRequest.newInstance(mount);
+ GetMountTableEntriesResponse getResponse =
+ mountTable.getMountTableEntries(getRequest);
+ List<MountTable> results = getResponse.getEntries();
+ MountTable existingEntry = null;
+ for (MountTable result : results) {
+ if (mount.equals(result.getSourcePath())) {
+ existingEntry = result;
+ }
+ }
+
+ if (existingEntry == null) {
+ // Create and add the entry if it doesn't exist
+ Map<String, String> destMap = new LinkedHashMap<>();
+ for (String ns : nss) {
+ destMap.put(ns, dest);
+ }
+ MountTable newEntry = MountTable.newInstance(mount, destMap);
+ if (readonly) {
+ newEntry.setReadOnly(true);
+ }
+ if (order != null) {
+ newEntry.setDestOrder(order);
+ }
+
+ // Set ACL info for mount table entry
+ if (aclInfo.getOwner() != null) {
+ newEntry.setOwnerName(aclInfo.getOwner());
+ }
+
+ if (aclInfo.getGroup() != null) {
+ newEntry.setGroupName(aclInfo.getGroup());
+ }
+
+ if (aclInfo.getMode() != null) {
+ newEntry.setMode(aclInfo.getMode());
+ }
+
+ AddMountTableEntryRequest request =
+ AddMountTableEntryRequest.newInstance(newEntry);
+ AddMountTableEntryResponse addResponse =
+ mountTable.addMountTableEntry(request);
+ boolean added = addResponse.getStatus();
+ if (!added) {
+ System.err.println("Cannot add mount point " + mount);
+ }
+ return added;
+ } else {
+ // Update the existing entry if it exists
+ for (String nsId : nss) {
+ if (!existingEntry.addDestination(nsId, dest)) {
+ System.err.println("Cannot add destination at " + nsId + " " + dest);
+ }
+ }
+ if (readonly) {
+ existingEntry.setReadOnly(true);
+ }
+ if (order != null) {
+ existingEntry.setDestOrder(order);
+ }
+
+ // Update ACL info of mount table entry
+ if (aclInfo.getOwner() != null) {
+ existingEntry.setOwnerName(aclInfo.getOwner());
+ }
+
+ if (aclInfo.getGroup() != null) {
+ existingEntry.setGroupName(aclInfo.getGroup());
+ }
+
+ if (aclInfo.getMode() != null) {
+ existingEntry.setMode(aclInfo.getMode());
+ }
+
+ UpdateMountTableEntryRequest updateRequest =
+ UpdateMountTableEntryRequest.newInstance(existingEntry);
+ UpdateMountTableEntryResponse updateResponse =
+ mountTable.updateMountTableEntry(updateRequest);
+ boolean updated = updateResponse.getStatus();
+ if (!updated) {
+ System.err.println("Cannot update mount point " + mount);
+ }
+ return updated;
+ }
+ }
+
+ /**
+ * Remove mount point.
+ *
+ * @param path Path to remove.
+ * @throws IOException If it cannot be removed.
+ */
+ public boolean removeMount(String path) throws IOException {
+ MountTableManager mountTable = client.getMountTableManager();
+ RemoveMountTableEntryRequest request =
+ RemoveMountTableEntryRequest.newInstance(path);
+ RemoveMountTableEntryResponse response =
+ mountTable.removeMountTableEntry(request);
+ boolean removed = response.getStatus();
+ if (!removed) {
+ System.out.println("Cannot remove mount point " + path);
+ }
+ return removed;
+ }
+
+ /**
+ * List mount points.
+ *
+ * @param path Path to list.
+ * @throws IOException If it cannot be listed.
+ */
+ public void listMounts(String path) throws IOException {
+ MountTableManager mountTable = client.getMountTableManager();
+ GetMountTableEntriesRequest request =
+ GetMountTableEntriesRequest.newInstance(path);
+ GetMountTableEntriesResponse response =
+ mountTable.getMountTableEntries(request);
+ List<MountTable> entries = response.getEntries();
+ printMounts(entries);
+ }
+
+ private static void printMounts(List<MountTable> entries) {
+ System.out.println("Mount Table Entries:");
+ System.out.println(String.format(
+ "%-25s %-25s %-25s %-25s %-25s %-25s",
+ "Source", "Destinations", "Owner", "Group", "Mode", "Quota/Usage"));
+ for (MountTable entry : entries) {
+ StringBuilder destBuilder = new StringBuilder();
+ for (RemoteLocation location : entry.getDestinations()) {
+ if (destBuilder.length() > 0) {
+ destBuilder.append(",");
+ }
+ destBuilder.append(String.format("%s->%s", location.getNameserviceId(),
+ location.getDest()));
+ }
+ System.out.print(String.format("%-25s %-25s", entry.getSourcePath(),
+ destBuilder.toString()));
+
+ System.out.print(String.format(" %-25s %-25s %-25s",
+ entry.getOwnerName(), entry.getGroupName(), entry.getMode()));
+
+ System.out.println(String.format(" %-25s", entry.getQuota()));
+ }
+ }
+
+ /**
+ * Set quota for a mount table entry.
+ *
+ * @param parameters Parameters of the quota.
+ * @param i Index in the parameters.
+ */
+ private boolean setQuota(String[] parameters, int i) throws IOException {
+ long nsQuota = HdfsConstants.QUOTA_DONT_SET;
+ long ssQuota = HdfsConstants.QUOTA_DONT_SET;
+
+ String mount = parameters[i++];
+ while (i < parameters.length) {
+ if (parameters[i].equals("-nsQuota")) {
+ i++;
+ try {
+ nsQuota = Long.parseLong(parameters[i]);
+ } catch (Exception e) {
+ System.err.println("Cannot parse nsQuota: " + parameters[i]);
+ }
+ } else if (parameters[i].equals("-ssQuota")) {
+ i++;
+ try {
+ ssQuota = StringUtils.TraditionalBinaryPrefix
+ .string2long(parameters[i]);
+ } catch (Exception e) {
+ System.err.println("Cannot parse ssQuota: " + parameters[i]);
+ }
+ }
+
+ i++;
+ }
+
+ if (nsQuota <= 0 || ssQuota <= 0) {
+ System.err.println("Input quota value should be a positive number.");
+ return false;
+ }
+
+ return updateQuota(mount, nsQuota, ssQuota);
+ }
+
+ /**
+ * Clear quota of the mount point.
+ *
+ * @param mount Mount table to clear
+ * @return If the quota was cleared.
+ * @throws IOException Error clearing the mount point.
+ */
+ private boolean clrQuota(String mount) throws IOException {
+ return updateQuota(mount, HdfsConstants.QUOTA_DONT_SET,
+ HdfsConstants.QUOTA_DONT_SET);
+ }
+
+ /**
+ * Update quota of specified mount table.
+ *
+ * @param mount Specified mount table to update.
+ * @param nsQuota Namespace quota.
+ * @param ssQuota Storage space quota.
+ * @return If the quota was updated.
+ * @throws IOException Error updating quota.
+ */
+ private boolean updateQuota(String mount, long nsQuota, long ssQuota)
+ throws IOException {
+ // Get existing entry
+ MountTableManager mountTable = client.getMountTableManager();
+ GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest
+ .newInstance(mount);
+ GetMountTableEntriesResponse getResponse = mountTable
+ .getMountTableEntries(getRequest);
+ List<MountTable> results = getResponse.getEntries();
+ MountTable existingEntry = null;
+ for (MountTable result : results) {
+ if (mount.equals(result.getSourcePath())) {
+ existingEntry = result;
+ break;
+ }
+ }
+
+ if (existingEntry == null) {
+ return false;
+ } else {
+ long nsCount = existingEntry.getQuota().getFileAndDirectoryCount();
+ long ssCount = existingEntry.getQuota().getSpaceConsumed();
+ // If nsQuota or ssQuota was unset, reset corresponding usage
+ // value to zero.
+ if (nsQuota == HdfsConstants.QUOTA_DONT_SET) {
+ nsCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
+ }
+
+ if (nsQuota == HdfsConstants.QUOTA_DONT_SET) {
+ ssCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
+ }
+
+ RouterQuotaUsage updatedQuota = new RouterQuotaUsage.Builder()
+ .fileAndDirectoryCount(nsCount).quota(nsQuota)
+ .spaceConsumed(ssCount).spaceQuota(ssQuota).build();
+ existingEntry.setQuota(updatedQuota);
+ }
+
+ UpdateMountTableEntryRequest updateRequest =
+ UpdateMountTableEntryRequest.newInstance(existingEntry);
+ UpdateMountTableEntryResponse updateResponse = mountTable
+ .updateMountTableEntry(updateRequest);
+ return updateResponse.getStatus();
+ }
+
+ /**
+ * Manager the safe mode state.
+ * @param cmd Input command, enter or leave safe mode.
+ * @throws IOException
+ */
+ private void manageSafeMode(String cmd) throws IOException {
+ if (cmd.equals("enter")) {
+ if (enterSafeMode()) {
+ System.out.println("Successfully enter safe mode.");
+ }
+ } else if (cmd.equals("leave")) {
+ if (leaveSafeMode()) {
+ System.out.println("Successfully leave safe mode.");
+ }
+ } else if (cmd.equals("get")) {
+ boolean result = getSafeMode();
+ System.out.println("Safe Mode: " + result);
+ }
+ }
+
+ /**
+ * Request the Router entering safemode state.
+ * @return Return true if entering safemode successfully.
+ * @throws IOException
+ */
+ private boolean enterSafeMode() throws IOException {
+ RouterStateManager stateManager = client.getRouterStateManager();
+ EnterSafeModeResponse response = stateManager.enterSafeMode(
+ EnterSafeModeRequest.newInstance());
+ return response.getStatus();
+ }
+
+ /**
+ * Request the Router leaving safemode state.
+ * @return Return true if leaving safemode successfully.
+ * @throws IOException
+ */
+ private boolean leaveSafeMode() throws IOException {
+ RouterStateManager stateManager = client.getRouterStateManager();
+ LeaveSafeModeResponse response = stateManager.leaveSafeMode(
+ LeaveSafeModeRequest.newInstance());
+ return response.getStatus();
+ }
+
+ /**
+ * Verify if current Router state is safe mode state.
+ * @return True if the Router is in safe mode.
+ * @throws IOException
+ */
+ private boolean getSafeMode() throws IOException {
+ RouterStateManager stateManager = client.getRouterStateManager();
+ GetSafeModeResponse response = stateManager.getSafeMode(
+ GetSafeModeRequest.newInstance());
+ return response.isInSafeMode();
+ }
+
+ /**
+ * Inner class that stores ACL info of mount table.
+ */
+ static class ACLEntity {
+ private final String owner;
+ private final String group;
+ private final FsPermission mode;
+
+ ACLEntity(String owner, String group, FsPermission mode) {
+ this.owner = owner;
+ this.group = group;
+ this.mode = mode;
+ }
+
+ public String getOwner() {
+ return owner;
+ }
+
+ public String getGroup() {
+ return group;
+ }
+
+ public FsPermission getMode() {
+ return mode;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/package-info.java
new file mode 100644
index 0000000..466c3d3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/package-info.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * It includes the tools to manage the Router-based federation. Includes the
+ * utilities to add and remove mount table entries.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+package org.apache.hadoop.hdfs.tools.federation;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto
new file mode 100644
index 0000000..b0d6982
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.hadoop.hdfs.federation.protocol.proto";
+option java_outer_classname = "HdfsServerFederationProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+package hadoop.hdfs;
+
+import "hdfs.proto";
+
+/////////////////////////////////////////////////
+// Membership
+/////////////////////////////////////////////////
+
+message NamenodeMembershipStatsRecordProto {
+ optional uint64 totalSpace = 1;
+ optional uint64 availableSpace = 2;
+ optional uint64 providedSpace = 3;
+
+ optional uint64 numOfFiles = 10;
+ optional uint64 numOfBlocks = 11;
+ optional uint64 numOfBlocksMissing = 12;
+ optional uint64 numOfBlocksPendingReplication = 13;
+ optional uint64 numOfBlocksUnderReplicated = 14;
+ optional uint64 numOfBlocksPendingDeletion = 15;
+
+ optional uint32 numOfActiveDatanodes = 20;
+ optional uint32 numOfDeadDatanodes = 21;
+ optional uint32 numOfDecommissioningDatanodes = 22;
+ optional uint32 numOfDecomActiveDatanodes = 23;
+ optional uint32 numOfDecomDeadDatanodes = 24;
+}
+
+message NamenodeMembershipRecordProto {
+ optional uint64 dateCreated = 1;
+ optional uint64 dateModified = 2;
+ optional uint64 lastContact = 3;
+ optional string routerId = 4;
+ optional string nameserviceId = 5;
+ optional string namenodeId = 6;
+ optional string clusterId = 7;
+ optional string blockPoolId = 8;
+ optional string webAddress = 9;
+ optional string rpcAddress = 10;
+ optional string serviceAddress = 11;
+ optional string lifelineAddress = 12;
+ optional string state = 13;
+ optional bool isSafeMode = 14;
+
+ optional NamenodeMembershipStatsRecordProto stats = 15;
+}
+
+message FederationNamespaceInfoProto {
+ optional string blockPoolId = 1;
+ optional string clusterId = 2;
+ optional string nameserviceId = 3;
+}
+
+message GetNamenodeRegistrationsRequestProto {
+ optional NamenodeMembershipRecordProto membership = 1;
+}
+
+message GetNamenodeRegistrationsResponseProto {
+ repeated NamenodeMembershipRecordProto namenodeMemberships = 1;
+}
+
+message GetExpiredRegistrationsRequestProto {
+}
+
+message GetNamespaceInfoRequestProto {
+}
+
+message GetNamespaceInfoResponseProto {
+ repeated FederationNamespaceInfoProto namespaceInfos = 1;
+}
+
+message UpdateNamenodeRegistrationRequestProto {
+ optional string nameserviceId = 1;
+ optional string namenodeId = 2;
+ optional string state = 3;
+}
+
+message UpdateNamenodeRegistrationResponseProto {
+ optional bool status = 1;
+}
+
+message NamenodeHeartbeatRequestProto {
+ optional NamenodeMembershipRecordProto namenodeMembership = 1;
+}
+
+message NamenodeHeartbeatResponseProto {
+ optional bool status = 1;
+}
+
+
+/////////////////////////////////////////////////
+// Mount table
+/////////////////////////////////////////////////
+
+message RemoteLocationProto {
+ optional string nameserviceId = 1;
+ optional string path = 2;
+}
+
+message MountTableRecordProto {
+ optional string srcPath = 1;
+ repeated RemoteLocationProto destinations = 2;
+ optional uint64 dateCreated = 3;
+ optional uint64 dateModified = 4;
+ optional bool readOnly = 5 [default = false];
+
+ enum DestOrder {
+ HASH = 0;
+ LOCAL = 1;
+ RANDOM = 2;
+ HASH_ALL = 3;
+ }
+ optional DestOrder destOrder = 6 [default = HASH];
+
+ optional string ownerName = 10;
+ optional string groupName = 11;
+ optional int32 mode = 12;
+
+ optional QuotaUsageProto quota = 13;
+}
+
+message AddMountTableEntryRequestProto {
+ optional MountTableRecordProto entry = 1;
+}
+
+message AddMountTableEntryResponseProto {
+ optional bool status = 1;
+}
+
+message UpdateMountTableEntryRequestProto {
+ optional MountTableRecordProto entry = 1;
+}
+
+message UpdateMountTableEntryResponseProto {
+ optional bool status = 1;
+}
+
+message RemoveMountTableEntryRequestProto {
+ optional string srcPath = 1;
+}
+
+message RemoveMountTableEntryResponseProto{
+ optional bool status = 1;
+}
+
+message GetMountTableEntriesRequestProto {
+ optional string srcPath = 1;
+}
+
+message GetMountTableEntriesResponseProto {
+ repeated MountTableRecordProto entries = 1;
+ optional uint64 timestamp = 2;
+}
+
+
+/////////////////////////////////////////////////
+// Routers
+/////////////////////////////////////////////////
+
+message StateStoreVersionRecordProto {
+ optional uint64 membershipVersion = 1;
+ optional uint64 mountTableVersion = 2;
+}
+
+message RouterRecordProto {
+ optional uint64 dateCreated = 1;
+ optional uint64 dateModified = 2;
+ optional string address = 3;
+ optional string status = 4;
+ optional StateStoreVersionRecordProto stateStoreVersion = 5;
+ optional string version = 6;
+ optional string compileInfo = 7;
+ optional uint64 dateStarted = 8;
+}
+
+message GetRouterRegistrationRequestProto {
+ optional string routerId = 1;
+}
+
+message GetRouterRegistrationResponseProto {
+ optional RouterRecordProto router = 1;
+}
+
+message GetRouterRegistrationsRequestProto {
+}
+
+message GetRouterRegistrationsResponseProto {
+ optional uint64 timestamp = 1;
+ repeated RouterRecordProto routers = 2;
+}
+
+message RouterHeartbeatRequestProto {
+ optional RouterRecordProto router = 1;
+}
+
+message RouterHeartbeatResponseProto {
+ optional bool status = 1;
+}
+
+/////////////////////////////////////////////////
+// Route State
+/////////////////////////////////////////////////
+
+message EnterSafeModeRequestProto {
+}
+
+message EnterSafeModeResponseProto {
+ optional bool status = 1;
+}
+
+message LeaveSafeModeRequestProto {
+}
+
+message LeaveSafeModeResponseProto {
+ optional bool status = 1;
+}
+
+message GetSafeModeRequestProto {
+}
+
+message GetSafeModeResponseProto {
+ optional bool isInSafeMode = 1;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/RouterProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/RouterProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/RouterProtocol.proto
new file mode 100644
index 0000000..a4e4d65
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/RouterProtocol.proto
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.hadoop.hdfs.protocol.proto";
+option java_outer_classname = "RouterProtocolProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+package hadoop.hdfs.router;
+
+import "FederationProtocol.proto";
+
+service RouterAdminProtocolService {
+ /**
+ * Add a mount table entry.
+ */
+ rpc addMountTableEntry(AddMountTableEntryRequestProto) returns(AddMountTableEntryResponseProto);
+
+ /**
+ * Update an existing mount table entry without copying files.
+ */
+ rpc updateMountTableEntry(UpdateMountTableEntryRequestProto) returns(UpdateMountTableEntryResponseProto);
+
+ /**
+ * Remove a mount table entry.
+ */
+ rpc removeMountTableEntry(RemoveMountTableEntryRequestProto) returns(RemoveMountTableEntryResponseProto);
+
+ /**
+ * Get matching mount entries
+ */
+ rpc getMountTableEntries(GetMountTableEntriesRequestProto) returns(GetMountTableEntriesResponseProto);
+
+ /**
+ * Transform Router state to safe mode state.
+ */
+ rpc enterSafeMode(EnterSafeModeRequestProto) returns (EnterSafeModeResponseProto);
+
+ /**
+ * Transform Router state from safe mode to running state.
+ */
+ rpc leaveSafeMode(LeaveSafeModeRequestProto) returns (LeaveSafeModeResponseProto);
+
+ /**
+ * Verify if current Router state is safe mode state.
+ */
+ rpc getSafeMode(GetSafeModeRequestProto) returns (GetSafeModeResponseProto);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
new file mode 100644
index 0000000..92f899d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
@@ -0,0 +1,434 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Do not modify this file directly. Instead, copy entries that you -->
+<!-- wish to modify from this file into hdfs-site.xml and change them -->
+<!-- there. If hdfs-site.xml does not already exist, create it. -->
+
+<configuration>
+ <property>
+ <name>dfs.federation.router.default.nameserviceId</name>
+ <value></value>
+ <description>
+ Nameservice identifier of the default subcluster to monitor.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.rpc.enable</name>
+ <value>true</value>
+ <description>
+ If true, the RPC service to handle client requests in the router is
+ enabled.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.rpc-address</name>
+ <value>0.0.0.0:8888</value>
+ <description>
+ RPC address that handles all clients requests.
+ The value of this property will take the form of router-host1:rpc-port.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.rpc-bind-host</name>
+ <value></value>
+ <description>
+ The actual address the RPC server will bind to. If this optional address is
+ set, it overrides only the hostname portion of
+ dfs.federation.router.rpc-address. This is useful for making the name node
+ listen on all interfaces by setting it to 0.0.0.0.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.handler.count</name>
+ <value>10</value>
+ <description>
+ The number of server threads for the router to handle RPC requests from
+ clients.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.handler.queue.size</name>
+ <value>100</value>
+ <description>
+ The size of the queue for the number of handlers to handle RPC client requests.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.reader.count</name>
+ <value>1</value>
+ <description>
+ The number of readers for the router to handle RPC client requests.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.reader.queue.size</name>
+ <value>100</value>
+ <description>
+ The size of the queue for the number of readers for the router to handle RPC client requests.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.connection.pool-size</name>
+ <value>1</value>
+ <description>
+ Size of the pool of connections from the router to namenodes.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.connection.clean.ms</name>
+ <value>10000</value>
+ <description>
+ Time interval, in milliseconds, to check if the connection pool should
+ remove unused connections.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.connection.pool.clean.ms</name>
+ <value>60000</value>
+ <description>
+ Time interval, in milliseconds, to check if the connection manager should
+ remove unused connection pools.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.metrics.enable</name>
+ <value>true</value>
+ <description>
+ If the metrics in the router are enabled.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.metrics.class</name>
+ <value>org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCPerformanceMonitor</value>
+ <description>
+ Class to monitor the RPC system in the router. It must implement the
+ RouterRpcMonitor interface.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.admin.enable</name>
+ <value>true</value>
+ <description>
+ If true, the RPC admin service to handle client requests in the router is
+ enabled.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.admin-address</name>
+ <value>0.0.0.0:8111</value>
+ <description>
+ RPC address that handles the admin requests.
+ The value of this property will take the form of router-host1:rpc-port.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.admin-bind-host</name>
+ <value></value>
+ <description>
+ The actual address the RPC admin server will bind to. If this optional
+ address is set, it overrides only the hostname portion of
+ dfs.federation.router.admin-address. This is useful for making the name
+ node listen on all interfaces by setting it to 0.0.0.0.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.admin.handler.count</name>
+ <value>1</value>
+ <description>
+ The number of server threads for the router to handle RPC requests from
+ admin.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.http-address</name>
+ <value>0.0.0.0:50071</value>
+ <description>
+ HTTP address that handles the web requests to the Router.
+ The value of this property will take the form of router-host1:http-port.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.http-bind-host</name>
+ <value></value>
+ <description>
+ The actual address the HTTP server will bind to. If this optional
+ address is set, it overrides only the hostname portion of
+ dfs.federation.router.http-address. This is useful for making the name
+ node listen on all interfaces by setting it to 0.0.0.0.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.https-address</name>
+ <value>0.0.0.0:50072</value>
+ <description>
+ HTTPS address that handles the web requests to the Router.
+ The value of this property will take the form of router-host1:https-port.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.https-bind-host</name>
+ <value></value>
+ <description>
+ The actual address the HTTPS server will bind to. If this optional
+ address is set, it overrides only the hostname portion of
+ dfs.federation.router.https-address. This is useful for making the name
+ node listen on all interfaces by setting it to 0.0.0.0.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.http.enable</name>
+ <value>true</value>
+ <description>
+ If the HTTP service to handle client requests in the router is enabled.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.metrics.enable</name>
+ <value>true</value>
+ <description>
+ If the metrics service in the router is enabled.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.file.resolver.client.class</name>
+ <value>org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver</value>
+ <description>
+ Class to resolve files to subclusters.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.namenode.resolver.client.class</name>
+ <value>org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver</value>
+ <description>
+ Class to resolve the namenode for a subcluster.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.store.enable</name>
+ <value>true</value>
+ <description>
+ If true, the Router connects to the State Store.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.store.serializer</name>
+ <value>org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreSerializerPBImpl</value>
+ <description>
+ Class to serialize State Store records.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.store.driver.class</name>
+ <value>org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl</value>
+ <description>
+ Class to implement the State Store. There are three implementation classes currently
+ being supported:
+ org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl,
+ org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileSystemImpl and
+ org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl.
+ These implementation classes use the local file, filesystem and ZooKeeper as a backend respectively.
+ By default it uses the ZooKeeper as the default State Store.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.store.connection.test</name>
+ <value>60000</value>
+ <description>
+ How often to check for the connection to the State Store in milliseconds.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.cache.ttl</name>
+ <value>1m</value>
+ <description>
+ How often to refresh the State Store caches in milliseconds. This setting
+ supports multiple time unit suffixes as described in
+ dfs.heartbeat.interval. If no suffix is specified then milliseconds is
+ assumed.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.store.membership.expiration</name>
+ <value>300000</value>
+ <description>
+ Expiration time in milliseconds for a membership record.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.heartbeat.enable</name>
+ <value>true</value>
+ <description>
+ If true, the Router heartbeats into the State Store.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.heartbeat.interval</name>
+ <value>5000</value>
+ <description>
+ How often the Router should heartbeat into the State Store in milliseconds.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.heartbeat-state.interval</name>
+ <value>5s</value>
+ <description>
+ How often the Router should heartbeat its state into the State Store in
+ milliseconds. This setting supports multiple time unit suffixes as
+ described in dfs.federation.router.quota-cache.update.interval.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.store.router.expiration</name>
+ <value>5m</value>
+ <description>
+ Expiration time in milliseconds for a router state record. This setting
+ supports multiple time unit suffixes as described in
+ dfs.federation.router.quota-cache.update.interval.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.safemode.enable</name>
+ <value>true</value>
+ <description>
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.safemode.extension</name>
+ <value>30s</value>
+ <description>
+ Time after startup that the Router is in safe mode. This setting
+ supports multiple time unit suffixes as described in
+ dfs.heartbeat.interval. If no suffix is specified then milliseconds is
+ assumed.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.safemode.expiration</name>
+ <value>3m</value>
+ <description>
+ Time without being able to reach the State Store to enter safe mode. This
+ setting supports multiple time unit suffixes as described in
+ dfs.heartbeat.interval. If no suffix is specified then milliseconds is
+ assumed.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.monitor.namenode</name>
+ <value></value>
+ <description>
+ The identifier of the namenodes to monitor and heartbeat.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.monitor.localnamenode.enable</name>
+ <value>true</value>
+ <description>
+ If true, the Router should monitor the namenode in the local machine.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.mount-table.max-cache-size</name>
+ <value>10000</value>
+ <description>
+ Maximum number of mount table cache entries to have.
+ By default, remove cache entries if we have more than 10k.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.quota.enable</name>
+ <value>false</value>
+ <description>
+ Set to true to enable quota system in Router.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.quota-cache.update.interval</name>
+ <value>60s</value>
+ <description>
+ Interval time for updating quota usage cache in Router.
+ This property is used only if the value of
+ dfs.federation.router.quota.enable is true.
+ This setting supports multiple time unit suffixes as described
+ in dfs.heartbeat.interval. If no suffix is specified then milliseconds
+ is assumed.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.client.thread-size</name>
+ <value>32</value>
+ <description>
+ Max threads size for the RouterClient to execute concurrent
+ requests.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.federation.router.client.retry.max.attempts</name>
+ <value>3</value>
+ <description>
+ Max retry attempts for the RouterClient talking to the Router.
+ </description>
+ </property>
+
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/proto-web.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/proto-web.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/proto-web.xml
new file mode 100644
index 0000000..83b36b2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/proto-web.xml
@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<web-app version="2.4" xmlns="http://java.sun.com/xml/ns/j2ee">
+</web-app>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FileSubclusterResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FileSubclusterResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FileSubclusterResolver.java
new file mode 100644
index 0000000..af9f493
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FileSubclusterResolver.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Interface to map a file path in the global name space to a specific
+ * subcluster and path in an HDFS name space.
+ * <p>
+ * Each path in the global/federated namespace may map to 1-N different HDFS
+ * locations. Each location specifies a single nameservice and a single HDFS
+ * path. The behavior is similar to MergeFS and Nfly and allows the merger
+ * of multiple HDFS locations into a single path. See HADOOP-8298 and
+ * HADOOP-12077
+ * <p>
+ * For example, a directory listing will fetch listings for each destination
+ * path and combine them into a single set of results.
+ * <p>
+ * When multiple destinations are available for a path, the destinations are
+ * prioritized in a consistent manner. This allows the proxy server to
+ * guess the best/most likely destination and attempt it first.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface FileSubclusterResolver {
+
+ /**
+ * Get the destinations for a global path. Results are from the mount table
+ * cache. If multiple destinations are available, the first result is the
+ * highest priority destination.
+ *
+ * @param path Global path.
+ * @return Location in a destination namespace or null if it does not exist.
+ * @throws IOException Throws exception if the data is not available.
+ */
+ PathLocation getDestinationForPath(String path) throws IOException;
+
+ /**
+ * Get a list of mount points for a path. Results are from the mount table
+ * cache.
+ *
+ * @return List of mount points present at this path or zero-length list if
+ * none are found.
+ * @throws IOException Throws exception if the data is not available.
+ */
+ List<String> getMountPoints(String path) throws IOException;
+
+ /**
+ * Get the default namespace for the cluster.
+ *
+ * @return Default namespace identifier.
+ */
+ String getDefaultNamespace();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java
new file mode 100644
index 0000000..b87eeec
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java
@@ -0,0 +1,317 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.ACTIVE;
+import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.EXPIRED;
+import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.UNAVAILABLE;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.store.MembershipStore;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreCache;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreUnavailableException;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateNamenodeRegistrationRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipStats;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Implements a cached lookup of the most recently active namenode for a
+ * particular nameservice. Relies on the {@link StateStoreService} to
+ * discover available nameservices and namenodes.
+ */
+public class MembershipNamenodeResolver
+ implements ActiveNamenodeResolver, StateStoreCache {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(MembershipNamenodeResolver.class);
+
+ /** Reference to the State Store. */
+ private final StateStoreService stateStore;
+ /** Membership State Store interface. */
+ private MembershipStore membershipInterface;
+
+ /** Parent router ID. */
+ private String routerId;
+
+ /** Cached lookup of NN for nameservice. Invalidated on cache refresh. */
+ private Map<String, List<? extends FederationNamenodeContext>> cacheNS;
+ /** Cached lookup of NN for block pool. Invalidated on cache refresh. */
+ private Map<String, List<? extends FederationNamenodeContext>> cacheBP;
+
+
+ public MembershipNamenodeResolver(
+ Configuration conf, StateStoreService store) throws IOException {
+ this.stateStore = store;
+
+ this.cacheNS = new ConcurrentHashMap<>();
+ this.cacheBP = new ConcurrentHashMap<>();
+
+ if (this.stateStore != null) {
+ // Request cache updates from the state store
+ this.stateStore.registerCacheExternal(this);
+ }
+ }
+
+ private synchronized MembershipStore getMembershipStore() throws IOException {
+ if (this.membershipInterface == null) {
+ this.membershipInterface = this.stateStore.getRegisteredRecordStore(
+ MembershipStore.class);
+ if (this.membershipInterface == null) {
+ throw new IOException("State Store does not have an interface for " +
+ MembershipStore.class.getSimpleName());
+ }
+ }
+ return this.membershipInterface;
+ }
+
+ @Override
+ public boolean loadCache(boolean force) {
+ // Our cache depends on the store, update it first
+ try {
+ MembershipStore membership = getMembershipStore();
+ membership.loadCache(force);
+ } catch (IOException e) {
+ LOG.error("Cannot update membership from the State Store", e);
+ }
+
+ // Force refresh of active NN cache
+ cacheBP.clear();
+ cacheNS.clear();
+ return true;
+ }
+
+ @Override
+ public void updateActiveNamenode(
+ final String nsId, final InetSocketAddress address) throws IOException {
+
+ // Called when we have an RPC miss and successful hit on an alternate NN.
+ // Temporarily update our cache, it will be overwritten on the next update.
+ try {
+ MembershipState partial = MembershipState.newInstance();
+ String rpcAddress = address.getHostName() + ":" + address.getPort();
+ partial.setRpcAddress(rpcAddress);
+ partial.setNameserviceId(nsId);
+
+ GetNamenodeRegistrationsRequest request =
+ GetNamenodeRegistrationsRequest.newInstance(partial);
+
+ MembershipStore membership = getMembershipStore();
+ GetNamenodeRegistrationsResponse response =
+ membership.getNamenodeRegistrations(request);
+ List<MembershipState> records = response.getNamenodeMemberships();
+
+ if (records != null && records.size() == 1) {
+ MembershipState record = records.get(0);
+ UpdateNamenodeRegistrationRequest updateRequest =
+ UpdateNamenodeRegistrationRequest.newInstance(
+ record.getNameserviceId(), record.getNamenodeId(), ACTIVE);
+ membership.updateNamenodeRegistration(updateRequest);
+ }
+ } catch (StateStoreUnavailableException e) {
+ LOG.error("Cannot update {} as active, State Store unavailable", address);
+ }
+ }
+
+ @Override
+ public List<? extends FederationNamenodeContext> getNamenodesForNameserviceId(
+ final String nsId) throws IOException {
+
+ List<? extends FederationNamenodeContext> ret = cacheNS.get(nsId);
+ if (ret == null) {
+ try {
+ MembershipState partial = MembershipState.newInstance();
+ partial.setNameserviceId(nsId);
+ GetNamenodeRegistrationsRequest request =
+ GetNamenodeRegistrationsRequest.newInstance(partial);
+
+ final List<MembershipState> result =
+ getRecentRegistrationForQuery(request, true, false);
+ if (result == null || result.isEmpty()) {
+ LOG.error("Cannot locate eligible NNs for {}", nsId);
+ return null;
+ } else {
+ cacheNS.put(nsId, result);
+ ret = result;
+ }
+ } catch (StateStoreUnavailableException e) {
+ LOG.error("Cannot get active NN for {}, State Store unavailable", nsId);
+ }
+ }
+ if (ret == null) {
+ return null;
+ }
+ return Collections.unmodifiableList(ret);
+ }
+
+ @Override
+ public List<? extends FederationNamenodeContext> getNamenodesForBlockPoolId(
+ final String bpId) throws IOException {
+
+ List<? extends FederationNamenodeContext> ret = cacheBP.get(bpId);
+ if (ret == null) {
+ try {
+ MembershipState partial = MembershipState.newInstance();
+ partial.setBlockPoolId(bpId);
+ GetNamenodeRegistrationsRequest request =
+ GetNamenodeRegistrationsRequest.newInstance(partial);
+
+ final List<MembershipState> result =
+ getRecentRegistrationForQuery(request, true, false);
+ if (result == null || result.isEmpty()) {
+ LOG.error("Cannot locate eligible NNs for {}", bpId);
+ } else {
+ cacheBP.put(bpId, result);
+ ret = result;
+ }
+ } catch (StateStoreUnavailableException e) {
+ LOG.error("Cannot get active NN for {}, State Store unavailable", bpId);
+ return null;
+ }
+ }
+ if (ret == null) {
+ return null;
+ }
+ return Collections.unmodifiableList(ret);
+ }
+
+ @Override
+ public boolean registerNamenode(NamenodeStatusReport report)
+ throws IOException {
+
+ if (this.routerId == null) {
+ LOG.warn("Cannot register namenode, router ID is not known {}", report);
+ return false;
+ }
+
+ MembershipState record = MembershipState.newInstance(
+ routerId, report.getNameserviceId(), report.getNamenodeId(),
+ report.getClusterId(), report.getBlockPoolId(), report.getRpcAddress(),
+ report.getServiceAddress(), report.getLifelineAddress(),
+ report.getWebAddress(), report.getState(), report.getSafemode());
+
+ if (report.statsValid()) {
+ MembershipStats stats = MembershipStats.newInstance();
+ stats.setNumOfFiles(report.getNumFiles());
+ stats.setNumOfBlocks(report.getNumBlocks());
+ stats.setNumOfBlocksMissing(report.getNumBlocksMissing());
+ stats.setNumOfBlocksPendingReplication(
+ report.getNumOfBlocksPendingReplication());
+ stats.setNumOfBlocksUnderReplicated(
+ report.getNumOfBlocksUnderReplicated());
+ stats.setNumOfBlocksPendingDeletion(
+ report.getNumOfBlocksPendingDeletion());
+ stats.setAvailableSpace(report.getAvailableSpace());
+ stats.setTotalSpace(report.getTotalSpace());
+ stats.setProvidedSpace(report.getProvidedSpace());
+ stats.setNumOfDecommissioningDatanodes(
+ report.getNumDecommissioningDatanodes());
+ stats.setNumOfActiveDatanodes(report.getNumLiveDatanodes());
+ stats.setNumOfDeadDatanodes(report.getNumDeadDatanodes());
+ stats.setNumOfDecomActiveDatanodes(report.getNumDecomLiveDatanodes());
+ stats.setNumOfDecomDeadDatanodes(report.getNumDecomDeadDatanodes());
+ record.setStats(stats);
+ }
+
+ if (report.getState() != UNAVAILABLE) {
+ // Set/update our last contact time
+ record.setLastContact(Time.now());
+ }
+
+ NamenodeHeartbeatRequest request = NamenodeHeartbeatRequest.newInstance();
+ request.setNamenodeMembership(record);
+ return getMembershipStore().namenodeHeartbeat(request).getResult();
+ }
+
+ @Override
+ public Set<FederationNamespaceInfo> getNamespaces() throws IOException {
+ GetNamespaceInfoRequest request = GetNamespaceInfoRequest.newInstance();
+ GetNamespaceInfoResponse response =
+ getMembershipStore().getNamespaceInfo(request);
+ return response.getNamespaceInfo();
+ }
+
+ /**
+ * Picks the most relevant record registration that matches the query. Return
+ * registrations matching the query in this preference: 1) Most recently
+ * updated ACTIVE registration 2) Most recently updated STANDBY registration
+ * (if showStandby) 3) Most recently updated UNAVAILABLE registration (if
+ * showUnavailable). EXPIRED registrations are ignored.
+ *
+ * @param request The select query for NN registrations.
+ * @param addUnavailable include UNAVAILABLE registrations.
+ * @param addExpired include EXPIRED registrations.
+ * @return List of memberships or null if no registrations that
+ * both match the query AND the selected states.
+ * @throws IOException
+ */
+ private List<MembershipState> getRecentRegistrationForQuery(
+ GetNamenodeRegistrationsRequest request, boolean addUnavailable,
+ boolean addExpired) throws IOException {
+
+ // Retrieve a list of all registrations that match this query.
+ // This may include all NN records for a namespace/blockpool, including
+ // duplicate records for the same NN from different routers.
+ MembershipStore membershipStore = getMembershipStore();
+ GetNamenodeRegistrationsResponse response =
+ membershipStore.getNamenodeRegistrations(request);
+
+ List<MembershipState> memberships = response.getNamenodeMemberships();
+ if (!addExpired || !addUnavailable) {
+ Iterator<MembershipState> iterator = memberships.iterator();
+ while (iterator.hasNext()) {
+ MembershipState membership = iterator.next();
+ if (membership.getState() == EXPIRED && !addExpired) {
+ iterator.remove();
+ } else if (membership.getState() == UNAVAILABLE && !addUnavailable) {
+ iterator.remove();
+ }
+ }
+ }
+
+ List<MembershipState> priorityList = new ArrayList<>();
+ priorityList.addAll(memberships);
+ Collections.sort(priorityList, new NamenodePriorityComparator());
+
+ LOG.debug("Selected most recent NN {} for query", priorityList);
+ return priorityList;
+ }
+
+ @Override
+ public void setRouterId(String router) {
+ this.routerId = router;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableManager.java
new file mode 100644
index 0000000..c2e4a5b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableManager.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse;
+
+/**
+ * Manage a mount table.
+ */
+public interface MountTableManager {
+
+ /**
+ * Add an entry to the mount table.
+ *
+ * @param request Fully populated request object.
+ * @return True if the mount table entry was successfully committed to the
+ * data store.
+ * @throws IOException Throws exception if the data store is not initialized.
+ */
+ AddMountTableEntryResponse addMountTableEntry(
+ AddMountTableEntryRequest request) throws IOException;
+
+ /**
+ * Updates an existing entry in the mount table.
+ *
+ * @param request Fully populated request object.
+ * @return True if the mount table entry was successfully committed to the
+ * data store.
+ * @throws IOException Throws exception if the data store is not initialized.
+ */
+ UpdateMountTableEntryResponse updateMountTableEntry(
+ UpdateMountTableEntryRequest request) throws IOException;
+
+ /**
+ * Remove an entry from the mount table.
+ *
+ * @param request Fully populated request object.
+ * @return True the mount table entry was removed from the data store.
+ * @throws IOException Throws exception if the data store is not initialized.
+ */
+ RemoveMountTableEntryResponse removeMountTableEntry(
+ RemoveMountTableEntryRequest request) throws IOException;
+
+ /**
+ * List all mount table entries present at or below the path. Fetches from the
+ * state store.
+ *
+ * @param request Fully populated request object.
+ *
+ * @return List of all mount table entries under the path. Zero-length list if
+ * none are found.
+ * @throws IOException Throws exception if the data store cannot be queried.
+ */
+ GetMountTableEntriesResponse getMountTableEntries(
+ GetMountTableEntriesRequest request) throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
new file mode 100644
index 0000000..9713138
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
@@ -0,0 +1,597 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE_DEFAULT;
+import static org.apache.hadoop.hdfs.server.federation.router.FederationUtil.isParentEntry;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DFSUtilClient;
+import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
+import org.apache.hadoop.hdfs.server.federation.router.Router;
+import org.apache.hadoop.hdfs.server.federation.store.MountTableStore;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreCache;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreUnavailableException;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+
+/**
+ * Mount table to map between global paths and remote locations. This allows the
+ * {@link org.apache.hadoop.hdfs.server.federation.router.Router Router} to map
+ * the global HDFS view to the remote namespaces. This is similar to
+ * {@link org.apache.hadoop.fs.viewfs.ViewFs ViewFs}.
+ * This is implemented as a tree.
+ */
+public class MountTableResolver
+ implements FileSubclusterResolver, StateStoreCache {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(MountTableResolver.class);
+
+ /** Reference to Router. */
+ private final Router router;
+ /** Reference to the State Store. */
+ private final StateStoreService stateStore;
+ /** Interface to the mount table store. */
+ private MountTableStore mountTableStore;
+
+ /** If the tree has been initialized. */
+ private boolean init = false;
+ /** Path -> Remote HDFS location. */
+ private final TreeMap<String, MountTable> tree = new TreeMap<>();
+ /** Path -> Remote location. */
+ private final Cache<String, PathLocation> locationCache;
+
+ /** Default nameservice when no mount matches the math. */
+ private String defaultNameService = "";
+
+ /** Synchronization for both the tree and the cache. */
+ private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ private final Lock readLock = readWriteLock.readLock();
+ private final Lock writeLock = readWriteLock.writeLock();
+
+
+ @VisibleForTesting
+ public MountTableResolver(Configuration conf) {
+ this(conf, (StateStoreService)null);
+ }
+
+ public MountTableResolver(Configuration conf, Router routerService) {
+ this(conf, routerService, null);
+ }
+
+ public MountTableResolver(Configuration conf, StateStoreService store) {
+ this(conf, null, store);
+ }
+
+ public MountTableResolver(Configuration conf, Router routerService,
+ StateStoreService store) {
+ this.router = routerService;
+ if (store != null) {
+ this.stateStore = store;
+ } else if (this.router != null) {
+ this.stateStore = this.router.getStateStore();
+ } else {
+ this.stateStore = null;
+ }
+
+ int maxCacheSize = conf.getInt(
+ FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE,
+ FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE_DEFAULT);
+ this.locationCache = CacheBuilder.newBuilder()
+ .maximumSize(maxCacheSize)
+ .build();
+
+ registerCacheExternal();
+ initDefaultNameService(conf);
+ }
+
+ /**
+ * Request cache updates from the State Store for this resolver.
+ */
+ private void registerCacheExternal() {
+ if (this.stateStore != null) {
+ this.stateStore.registerCacheExternal(this);
+ }
+ }
+
+ /**
+ * Nameservice for APIs that cannot be resolved to a specific one.
+ *
+ * @param conf Configuration for this resolver.
+ */
+ private void initDefaultNameService(Configuration conf) {
+ try {
+ this.defaultNameService = conf.get(
+ DFS_ROUTER_DEFAULT_NAMESERVICE,
+ DFSUtil.getNamenodeNameServiceId(conf));
+ } catch (HadoopIllegalArgumentException e) {
+ LOG.error("Cannot find default name service, setting it to the first");
+ Collection<String> nsIds = DFSUtilClient.getNameServiceIds(conf);
+ this.defaultNameService = nsIds.iterator().next();
+ LOG.info("Default name service: {}", this.defaultNameService);
+ }
+ }
+
+ /**
+ * Get a reference for the Router for this resolver.
+ *
+ * @return Router for this resolver.
+ */
+ protected Router getRouter() {
+ return this.router;
+ }
+
+ /**
+ * Get the mount table store for this resolver.
+ *
+ * @return Mount table store.
+ * @throws IOException If it cannot connect to the State Store.
+ */
+ protected MountTableStore getMountTableStore() throws IOException {
+ if (this.mountTableStore == null) {
+ this.mountTableStore = this.stateStore.getRegisteredRecordStore(
+ MountTableStore.class);
+ if (this.mountTableStore == null) {
+ throw new IOException("State Store does not have an interface for " +
+ MountTableStore.class);
+ }
+ }
+ return this.mountTableStore;
+ }
+
+ /**
+ * Add a mount entry to the table.
+ *
+ * @param entry The mount table record to add from the state store.
+ */
+ public void addEntry(final MountTable entry) {
+ writeLock.lock();
+ try {
+ String srcPath = entry.getSourcePath();
+ this.tree.put(srcPath, entry);
+ invalidateLocationCache(srcPath);
+ } finally {
+ writeLock.unlock();
+ }
+ this.init = true;
+ }
+
+ /**
+ * Remove a mount table entry.
+ *
+ * @param srcPath Source path for the entry to remove.
+ */
+ public void removeEntry(final String srcPath) {
+ writeLock.lock();
+ try {
+ this.tree.remove(srcPath);
+ invalidateLocationCache(srcPath);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /**
+ * Invalidates all cache entries below this path. It requires the write lock.
+ *
+ * @param path Source path.
+ */
+ private void invalidateLocationCache(final String path) {
+ LOG.debug("Invalidating {} from {}", path, locationCache);
+ if (locationCache.size() == 0) {
+ return;
+ }
+
+ // Go through the entries and remove the ones from the path to invalidate
+ ConcurrentMap<String, PathLocation> map = locationCache.asMap();
+ Set<Entry<String, PathLocation>> entries = map.entrySet();
+ Iterator<Entry<String, PathLocation>> it = entries.iterator();
+ while (it.hasNext()) {
+ Entry<String, PathLocation> entry = it.next();
+ PathLocation loc = entry.getValue();
+ String src = loc.getSourcePath();
+ if (src != null) {
+ if (isParentEntry(src, path)) {
+ LOG.debug("Removing {}", src);
+ it.remove();
+ }
+ } else {
+ String dest = loc.getDefaultLocation().getDest();
+ if (dest.startsWith(path)) {
+ LOG.debug("Removing default cache {}", dest);
+ it.remove();
+ }
+ }
+ }
+
+ LOG.debug("Location cache after invalidation: {}", locationCache);
+ }
+
+ /**
+ * Updates the mount path tree with a new set of mount table entries. It also
+ * updates the needed caches.
+ *
+ * @param entries Full set of mount table entries to update.
+ */
+ @VisibleForTesting
+ public void refreshEntries(final Collection<MountTable> entries) {
+ // The tree read/write must be atomic
+ writeLock.lock();
+ try {
+ // New entries
+ Map<String, MountTable> newEntries = new ConcurrentHashMap<>();
+ for (MountTable entry : entries) {
+ String srcPath = entry.getSourcePath();
+ newEntries.put(srcPath, entry);
+ }
+
+ // Old entries (reversed to sort from the leaves to the root)
+ Set<String> oldEntries = new TreeSet<>(Collections.reverseOrder());
+ for (MountTable entry : getTreeValues("/")) {
+ String srcPath = entry.getSourcePath();
+ oldEntries.add(srcPath);
+ }
+
+ // Entries that need to be removed
+ for (String srcPath : oldEntries) {
+ if (!newEntries.containsKey(srcPath)) {
+ this.tree.remove(srcPath);
+ invalidateLocationCache(srcPath);
+ LOG.info("Removed stale mount point {} from resolver", srcPath);
+ }
+ }
+
+ // Entries that need to be added
+ for (MountTable entry : entries) {
+ String srcPath = entry.getSourcePath();
+ if (!oldEntries.contains(srcPath)) {
+ // Add node, it does not exist
+ this.tree.put(srcPath, entry);
+ invalidateLocationCache(srcPath);
+ LOG.info("Added new mount point {} to resolver", srcPath);
+ } else {
+ // Node exists, check for updates
+ MountTable existingEntry = this.tree.get(srcPath);
+ if (existingEntry != null && !existingEntry.equals(entry)) {
+ LOG.info("Entry has changed from \"{}\" to \"{}\"",
+ existingEntry, entry);
+ this.tree.put(srcPath, entry);
+ invalidateLocationCache(srcPath);
+ LOG.info("Updated mount point {} in resolver", srcPath);
+ }
+ }
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ this.init = true;
+ }
+
+ /**
+ * Replaces the current in-memory cached of the mount table with a new
+ * version fetched from the data store.
+ */
+ @Override
+ public boolean loadCache(boolean force) {
+ try {
+ // Our cache depends on the store, update it first
+ MountTableStore mountTable = this.getMountTableStore();
+ mountTable.loadCache(force);
+
+ GetMountTableEntriesRequest request =
+ GetMountTableEntriesRequest.newInstance("/");
+ GetMountTableEntriesResponse response =
+ mountTable.getMountTableEntries(request);
+ List<MountTable> records = response.getEntries();
+ refreshEntries(records);
+ } catch (IOException e) {
+ LOG.error("Cannot fetch mount table entries from State Store", e);
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Clears all data.
+ */
+ public void clear() {
+ LOG.info("Clearing all mount location caches");
+ writeLock.lock();
+ try {
+ this.locationCache.invalidateAll();
+ this.tree.clear();
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public PathLocation getDestinationForPath(final String path)
+ throws IOException {
+ verifyMountTable();
+ readLock.lock();
+ try {
+ Callable<? extends PathLocation> meh = new Callable<PathLocation>() {
+ @Override
+ public PathLocation call() throws Exception {
+ return lookupLocation(path);
+ }
+ };
+ return this.locationCache.get(path, meh);
+ } catch (ExecutionException e) {
+ throw new IOException(e);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ /**
+ * Build the path location to insert into the cache atomically. It must hold
+ * the read lock.
+ * @param path Path to check/insert.
+ * @return New remote location.
+ */
+ public PathLocation lookupLocation(final String path) {
+ PathLocation ret = null;
+ MountTable entry = findDeepest(path);
+ if (entry != null) {
+ ret = buildLocation(path, entry);
+ } else {
+ // Not found, use default location
+ RemoteLocation remoteLocation =
+ new RemoteLocation(defaultNameService, path);
+ List<RemoteLocation> locations =
+ Collections.singletonList(remoteLocation);
+ ret = new PathLocation(null, locations);
+ }
+ return ret;
+ }
+
+ /**
+ * Get the mount table entry for a path.
+ *
+ * @param path Path to look for.
+ * @return Mount table entry the path belongs.
+ * @throws IOException If the State Store could not be reached.
+ */
+ public MountTable getMountPoint(final String path) throws IOException {
+ verifyMountTable();
+ return findDeepest(path);
+ }
+
+ @Override
+ public List<String> getMountPoints(final String path) throws IOException {
+ verifyMountTable();
+
+ Set<String> children = new TreeSet<>();
+ readLock.lock();
+ try {
+ String from = path;
+ String to = path + Character.MAX_VALUE;
+ SortedMap<String, MountTable> subMap = this.tree.subMap(from, to);
+
+ boolean exists = false;
+ for (String subPath : subMap.keySet()) {
+ String child = subPath;
+
+ // Special case for /
+ if (!path.equals(Path.SEPARATOR)) {
+ // Get the children
+ int ini = path.length();
+ child = subPath.substring(ini);
+ }
+
+ if (child.isEmpty()) {
+ // This is a mount point but without children
+ exists = true;
+ } else if (child.startsWith(Path.SEPARATOR)) {
+ // This is a mount point with children
+ exists = true;
+ child = child.substring(1);
+
+ // We only return immediate children
+ int fin = child.indexOf(Path.SEPARATOR);
+ if (fin > -1) {
+ child = child.substring(0, fin);
+ }
+ if (!child.isEmpty()) {
+ children.add(child);
+ }
+ }
+ }
+ if (!exists) {
+ return null;
+ }
+ return new LinkedList<>(children);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ /**
+ * Get all the mount records at or beneath a given path.
+ * @param path Path to get the mount points from.
+ * @return List of mount table records under the path or null if the path is
+ * not found.
+ * @throws IOException If it's not connected to the State Store.
+ */
+ public List<MountTable> getMounts(final String path) throws IOException {
+ verifyMountTable();
+
+ return getTreeValues(path, false);
+ }
+
+ /**
+ * Check if the Mount Table is ready to be used.
+ * @throws StateStoreUnavailableException If it cannot connect to the store.
+ */
+ private void verifyMountTable() throws StateStoreUnavailableException {
+ if (!this.init) {
+ throw new StateStoreUnavailableException("Mount Table not initialized");
+ }
+ }
+
+ @Override
+ public String toString() {
+ readLock.lock();
+ try {
+ return this.tree.toString();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ /**
+ * Build a location for this result beneath the discovered mount point.
+ *
+ * @param path Path to build for.
+ * @param entry Mount table entry.
+ * @return PathLocation containing the namespace, local path.
+ */
+ private static PathLocation buildLocation(
+ final String path, final MountTable entry) {
+
+ String srcPath = entry.getSourcePath();
+ if (!path.startsWith(srcPath)) {
+ LOG.error("Cannot build location, {} not a child of {}", path, srcPath);
+ return null;
+ }
+ String remainingPath = path.substring(srcPath.length());
+ if (remainingPath.startsWith(Path.SEPARATOR)) {
+ remainingPath = remainingPath.substring(1);
+ }
+
+ List<RemoteLocation> locations = new LinkedList<>();
+ for (RemoteLocation oneDst : entry.getDestinations()) {
+ String nsId = oneDst.getNameserviceId();
+ String dest = oneDst.getDest();
+ String newPath = dest;
+ if (!newPath.endsWith(Path.SEPARATOR) && !remainingPath.isEmpty()) {
+ newPath += Path.SEPARATOR;
+ }
+ newPath += remainingPath;
+ RemoteLocation remoteLocation = new RemoteLocation(nsId, newPath);
+ locations.add(remoteLocation);
+ }
+ DestinationOrder order = entry.getDestOrder();
+ return new PathLocation(srcPath, locations, order);
+ }
+
+ @Override
+ public String getDefaultNamespace() {
+ return this.defaultNameService;
+ }
+
+ /**
+ * Find the deepest mount point for a path.
+ * @param path Path to look for.
+ * @return Mount table entry.
+ */
+ private MountTable findDeepest(final String path) {
+ readLock.lock();
+ try {
+ Entry<String, MountTable> entry = this.tree.floorEntry(path);
+ while (entry != null && !isParentEntry(path, entry.getKey())) {
+ entry = this.tree.lowerEntry(entry.getKey());
+ }
+ if (entry == null) {
+ return null;
+ }
+ return entry.getValue();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ /**
+ * Get the mount table entries under a path.
+ * @param path Path to search from.
+ * @return Mount Table entries.
+ */
+ private List<MountTable> getTreeValues(final String path) {
+ return getTreeValues(path, false);
+ }
+
+ /**
+ * Get the mount table entries under a path.
+ * @param path Path to search from.
+ * @param reverse If the order should be reversed.
+ * @return Mount Table entries.
+ */
+ private List<MountTable> getTreeValues(final String path, boolean reverse) {
+ LinkedList<MountTable> ret = new LinkedList<>();
+ readLock.lock();
+ try {
+ String from = path;
+ String to = path + Character.MAX_VALUE;
+ SortedMap<String, MountTable> subMap = this.tree.subMap(from, to);
+ for (MountTable entry : subMap.values()) {
+ if (!reverse) {
+ ret.add(entry);
+ } else {
+ ret.addFirst(entry);
+ }
+ }
+ } finally {
+ readLock.unlock();
+ }
+ return ret;
+ }
+
+ /**
+ * Get the size of the cache.
+ * @return Size of the cache.
+ */
+ protected long getCacheSize() {
+ return this.locationCache.size();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MultipleDestinationMountTableResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MultipleDestinationMountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MultipleDestinationMountTableResolver.java
new file mode 100644
index 0000000..8dd73ec
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MultipleDestinationMountTableResolver.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import java.io.IOException;
+import java.util.EnumMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
+import org.apache.hadoop.hdfs.server.federation.resolver.order.HashFirstResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.order.HashResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.order.LocalResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.order.OrderedResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.order.RandomResolver;
+import org.apache.hadoop.hdfs.server.federation.router.Router;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Mount table resolver that supports multiple locations for each mount entry.
+ * The returned location contains prioritized remote paths from highest priority
+ * to the lowest priority. Multiple locations for a mount point are optional.
+ * When multiple locations are specified, both will be checked for the presence
+ * of a file and the nameservice for a new file/dir is chosen based on the
+ * results of a consistent hashing algorithm.
+ * <p>
+ * Does the Mount table entry for this path have multiple destinations?
+ * <ul>
+ * <li>No -> Return the location
+ * <li>Yes -> Return all locations, prioritizing the best guess from the
+ * consistent hashing algorithm.
+ * </ul>
+ * <p>
+ * It has multiple options to order the locations: HASH (default), LOCAL,
+ * RANDOM, and HASH_ALL.
+ * <p>
+ * The consistent hashing result is dependent on the number and combination of
+ * nameservices that are registered for particular mount point. The order of
+ * nameservices/locations in the mount table is not prioritized. Each consistent
+ * hash calculation considers only the set of unique nameservices present for
+ * the mount table location.
+ */
+public class MultipleDestinationMountTableResolver extends MountTableResolver {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(MultipleDestinationMountTableResolver.class);
+
+
+ /** Resolvers that use a particular order for multiple destinations. */
+ private EnumMap<DestinationOrder, OrderedResolver> orderedResolvers =
+ new EnumMap<>(DestinationOrder.class);
+
+
+ public MultipleDestinationMountTableResolver(
+ Configuration conf, Router router) {
+ super(conf, router);
+
+ // Initialize the ordered resolvers
+ addResolver(DestinationOrder.HASH, new HashFirstResolver());
+ addResolver(DestinationOrder.LOCAL, new LocalResolver(conf, router));
+ addResolver(DestinationOrder.RANDOM, new RandomResolver());
+ addResolver(DestinationOrder.HASH_ALL, new HashResolver());
+ }
+
+ @Override
+ public PathLocation getDestinationForPath(String path) throws IOException {
+ PathLocation mountTableResult = super.getDestinationForPath(path);
+ if (mountTableResult == null) {
+ LOG.error("The {} cannot find a location for {}",
+ super.getClass().getSimpleName(), path);
+ } else if (mountTableResult.hasMultipleDestinations()) {
+ DestinationOrder order = mountTableResult.getDestinationOrder();
+ OrderedResolver orderedResolver = orderedResolvers.get(order);
+ if (orderedResolver == null) {
+ LOG.error("Cannot find resolver for order {}", order);
+ } else {
+ String firstNamespace =
+ orderedResolver.getFirstNamespace(path, mountTableResult);
+
+ // Change the order of the name spaces according to the policy
+ if (firstNamespace != null) {
+ // This is the entity in the tree, we need to create our own copy
+ mountTableResult = new PathLocation(mountTableResult, firstNamespace);
+ LOG.debug("Ordered locations following {} are {}",
+ order, mountTableResult);
+ } else {
+ LOG.error("Cannot get main namespace for path {} with order {}",
+ path, order);
+ }
+ }
+ }
+ return mountTableResult;
+ }
+
+ @VisibleForTesting
+ public void addResolver(DestinationOrder order, OrderedResolver resolver) {
+ orderedResolvers.put(order, resolver);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodePriorityComparator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodePriorityComparator.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodePriorityComparator.java
new file mode 100644
index 0000000..fe82f29
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodePriorityComparator.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import java.util.Comparator;
+
+/**
+ * Compares NNs in the same namespace and prioritizes by their status. The
+ * priorities are:
+ * <ul>
+ * <li>ACTIVE
+ * <li>STANDBY
+ * <li>UNAVAILABLE
+ * </ul>
+ * When two NNs have the same state, the last modification date is the tie
+ * breaker, newest has priority. Expired NNs are excluded.
+ */
+public class NamenodePriorityComparator
+ implements Comparator<FederationNamenodeContext> {
+
+ @Override
+ public int compare(FederationNamenodeContext o1,
+ FederationNamenodeContext o2) {
+ FederationNamenodeServiceState state1 = o1.getState();
+ FederationNamenodeServiceState state2 = o2.getState();
+
+ if (state1 == state2) {
+ // Both have the same state, use mode dates
+ return compareModDates(o1, o2);
+ } else {
+ // Enum is ordered by priority
+ return state1.compareTo(state2);
+ }
+ }
+
+ /**
+ * Compare the modification dates.
+ *
+ * @param o1 Context 1.
+ * @param o2 Context 2.
+ * @return Comparison between dates.
+ */
+ private int compareModDates(FederationNamenodeContext o1,
+ FederationNamenodeContext o2) {
+ // Reverse sort, lowest position is highest priority.
+ return (int) (o2.getDateModified() - o1.getDateModified());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java
new file mode 100644
index 0000000..c3c6fa8
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java
@@ -0,0 +1,406 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
+
+/**
+ * Status of the namenode.
+ */
+public class NamenodeStatusReport {
+
+ /** Namenode information. */
+ private String nameserviceId = "";
+ private String namenodeId = "";
+ private String clusterId = "";
+ private String blockPoolId = "";
+ private String rpcAddress = "";
+ private String serviceAddress = "";
+ private String lifelineAddress = "";
+ private String webAddress = "";
+
+ /** Namenode state. */
+ private HAServiceState status = HAServiceState.STANDBY;
+ private boolean safeMode = false;
+
+ /** Datanodes stats. */
+ private int liveDatanodes = -1;
+ private int deadDatanodes = -1;
+ /** Decommissioning datanodes. */
+ private int decomDatanodes = -1;
+ /** Live decommissioned datanodes. */
+ private int liveDecomDatanodes = -1;
+ /** Dead decommissioned datanodes. */
+ private int deadDecomDatanodes = -1;
+
+ /** Space stats. */
+ private long availableSpace = -1;
+ private long numOfFiles = -1;
+ private long numOfBlocks = -1;
+ private long numOfBlocksMissing = -1;
+ private long numOfBlocksPendingReplication = -1;
+ private long numOfBlocksUnderReplicated = -1;
+ private long numOfBlocksPendingDeletion = -1;
+ private long totalSpace = -1;
+ private long providedSpace = -1;
+
+ /** If the fields are valid. */
+ private boolean registrationValid = false;
+ private boolean statsValid = false;
+ private boolean haStateValid = false;
+
+ public NamenodeStatusReport(String ns, String nn, String rpc, String service,
+ String lifeline, String web) {
+ this.nameserviceId = ns;
+ this.namenodeId = nn;
+ this.rpcAddress = rpc;
+ this.serviceAddress = service;
+ this.lifelineAddress = lifeline;
+ this.webAddress = web;
+ }
+
+ /**
+ * If the statistics are valid.
+ *
+ * @return If the statistics are valid.
+ */
+ public boolean statsValid() {
+ return this.statsValid;
+ }
+
+ /**
+ * If the registration is valid.
+ *
+ * @return If the registration is valid.
+ */
+ public boolean registrationValid() {
+ return this.registrationValid;
+ }
+
+ /**
+ * If the HA state is valid.
+ *
+ * @return If the HA state is valid.
+ */
+ public boolean haStateValid() {
+ return this.haStateValid;
+ }
+
+ /**
+ * Get the state of the Namenode being monitored.
+ *
+ * @return State of the Namenode.
+ */
+ public FederationNamenodeServiceState getState() {
+ if (!registrationValid) {
+ return FederationNamenodeServiceState.UNAVAILABLE;
+ } else if (haStateValid) {
+ return FederationNamenodeServiceState.getState(status);
+ } else {
+ return FederationNamenodeServiceState.ACTIVE;
+ }
+ }
+
+ /**
+ * Get the name service identifier.
+ *
+ * @return The name service identifier.
+ */
+ public String getNameserviceId() {
+ return this.nameserviceId;
+ }
+
+ /**
+ * Get the namenode identifier.
+ *
+ * @return The namenode identifier.
+ */
+ public String getNamenodeId() {
+ return this.namenodeId;
+ }
+
+ /**
+ * Get the cluster identifier.
+ *
+ * @return The cluster identifier.
+ */
+ public String getClusterId() {
+ return this.clusterId;
+ }
+
+ /**
+ * Get the block pool identifier.
+ *
+ * @return The block pool identifier.
+ */
+ public String getBlockPoolId() {
+ return this.blockPoolId;
+ }
+
+ /**
+ * Get the RPC address.
+ *
+ * @return The RPC address.
+ */
+ public String getRpcAddress() {
+ return this.rpcAddress;
+ }
+
+ /**
+ * Get the Service RPC address.
+ *
+ * @return The Service RPC address.
+ */
+ public String getServiceAddress() {
+ return this.serviceAddress;
+ }
+
+ /**
+ * Get the Lifeline RPC address.
+ *
+ * @return The Lifeline RPC address.
+ */
+ public String getLifelineAddress() {
+ return this.lifelineAddress;
+ }
+
+ /**
+ * Get the web address.
+ *
+ * @return The web address.
+ */
+ public String getWebAddress() {
+ return this.webAddress;
+ }
+
+ /**
+ * Get the HA service state.
+ *
+ * @return The HA service state.
+ */
+ public void setHAServiceState(HAServiceState state) {
+ this.status = state;
+ this.haStateValid = true;
+ }
+
+ /**
+ * Set the namespace information.
+ *
+ * @param info Namespace information.
+ */
+ public void setNamespaceInfo(NamespaceInfo info) {
+ this.clusterId = info.getClusterID();
+ this.blockPoolId = info.getBlockPoolID();
+ this.registrationValid = true;
+ }
+
+ public void setSafeMode(boolean safemode) {
+ this.safeMode = safemode;
+ }
+
+ public boolean getSafemode() {
+ return this.safeMode;
+ }
+
+ /**
+ * Set the datanode information.
+ *
+ * @param numLive Number of live nodes.
+ * @param numDead Number of dead nodes.
+ * @param numDecom Number of decommissioning nodes.
+ * @param numLiveDecom Number of decommissioned live nodes.
+ * @param numDeadDecom Number of decommissioned dead nodes.
+ */
+ public void setDatanodeInfo(int numLive, int numDead, int numDecom,
+ int numLiveDecom, int numDeadDecom) {
+ this.liveDatanodes = numLive;
+ this.deadDatanodes = numDead;
+ this.decomDatanodes = numDecom;
+ this.liveDecomDatanodes = numLiveDecom;
+ this.deadDecomDatanodes = numDeadDecom;
+ this.statsValid = true;
+ }
+
+ /**
+ * Get the number of live blocks.
+ *
+ * @return The number of dead nodes.
+ */
+ public int getNumLiveDatanodes() {
+ return this.liveDatanodes;
+ }
+
+ /**
+ * Get the number of dead blocks.
+ *
+ * @return The number of dead nodes.
+ */
+ public int getNumDeadDatanodes() {
+ return this.deadDatanodes;
+ }
+
+ /**
+ * Get the number of decommissionining nodes.
+ *
+ * @return The number of decommissionining nodes.
+ */
+ public int getNumDecommissioningDatanodes() {
+ return this.decomDatanodes;
+ }
+
+ /**
+ * Get the number of live decommissioned nodes.
+ *
+ * @return The number of live decommissioned nodes.
+ */
+ public int getNumDecomLiveDatanodes() {
+ return this.liveDecomDatanodes;
+ }
+
+ /**
+ * Get the number of dead decommissioned nodes.
+ *
+ * @return The number of dead decommissioned nodes.
+ */
+ public int getNumDecomDeadDatanodes() {
+ return this.deadDecomDatanodes;
+ }
+
+ /**
+ * Set the filesystem information.
+ *
+ * @param available Available capacity.
+ * @param total Total capacity.
+ * @param numFiles Number of files.
+ * @param numBlocks Total number of blocks.
+ * @param numBlocksMissing Number of missing blocks.
+ * @param numBlocksPendingReplication Number of blocks pending replication.
+ * @param numBlocksUnderReplicated Number of blocks under replication.
+ * @param numBlocksPendingDeletion Number of blocks pending deletion.
+ */
+ public void setNamesystemInfo(long available, long total,
+ long numFiles, long numBlocks, long numBlocksMissing,
+ long numBlocksPendingReplication, long numBlocksUnderReplicated,
+ long numBlocksPendingDeletion, long providedSpace) {
+ this.totalSpace = total;
+ this.availableSpace = available;
+ this.numOfBlocks = numBlocks;
+ this.numOfBlocksMissing = numBlocksMissing;
+ this.numOfBlocksPendingReplication = numBlocksPendingReplication;
+ this.numOfBlocksUnderReplicated = numBlocksUnderReplicated;
+ this.numOfBlocksPendingDeletion = numBlocksPendingDeletion;
+ this.numOfFiles = numFiles;
+ this.statsValid = true;
+ this.providedSpace = providedSpace;
+ }
+
+ /**
+ * Get the number of blocks.
+ *
+ * @return The number of blocks.
+ */
+ public long getNumBlocks() {
+ return this.numOfBlocks;
+ }
+
+ /**
+ * Get the number of files.
+ *
+ * @return The number of files.
+ */
+ public long getNumFiles() {
+ return this.numOfFiles;
+ }
+
+ /**
+ * Get the total space.
+ *
+ * @return The total space.
+ */
+ public long getTotalSpace() {
+ return this.totalSpace;
+ }
+
+ /**
+ * Get the available space.
+ *
+ * @return The available space.
+ */
+ public long getAvailableSpace() {
+ return this.availableSpace;
+ }
+
+ /**
+ * Get the space occupied by provided storage.
+ *
+ * @return the provided capacity.
+ */
+ public long getProvidedSpace() {
+ return this.providedSpace;
+ }
+ /**
+ * Get the number of missing blocks.
+ *
+ * @return Number of missing blocks.
+ */
+ public long getNumBlocksMissing() {
+ return this.numOfBlocksMissing;
+ }
+
+ /**
+ * Get the number of pending replication blocks.
+ *
+ * @return Number of pending replication blocks.
+ */
+ public long getNumOfBlocksPendingReplication() {
+ return this.numOfBlocksPendingReplication;
+ }
+
+ /**
+ * Get the number of under replicated blocks.
+ *
+ * @return Number of under replicated blocks.
+ */
+ public long getNumOfBlocksUnderReplicated() {
+ return this.numOfBlocksUnderReplicated;
+ }
+
+ /**
+ * Get the number of pending deletion blocks.
+ *
+ * @return Number of pending deletion blocks.
+ */
+ public long getNumOfBlocksPendingDeletion() {
+ return this.numOfBlocksPendingDeletion;
+ }
+
+ /**
+ * Set the validity of registration.
+ * @param isValid The desired value to be set.
+ */
+ public void setRegistrationValid(boolean isValid) {
+ this.registrationValid = isValid;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("%s-%s:%s",
+ nameserviceId, namenodeId, serviceAddress);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/PathLocation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/PathLocation.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/PathLocation.java
new file mode 100644
index 0000000..945d81d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/PathLocation.java
@@ -0,0 +1,212 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A map of the properties and target destinations (name space + path) for
+ * a path in the global/federated name space.
+ * This data is generated from the @see MountTable records.
+ */
+public class PathLocation {
+
+ private static final Logger LOG = LoggerFactory.getLogger(PathLocation.class);
+
+
+ /** Source path in global namespace. */
+ private final String sourcePath;
+
+ /** Remote paths in the target name spaces. */
+ private final List<RemoteLocation> destinations;
+ /** Order for the destinations. */
+ private final DestinationOrder destOrder;
+
+
+ /**
+ * Create a new PathLocation.
+ *
+ * @param source Source path in the global name space.
+ * @param dest Destinations of the mount table entry.
+ * @param order Order of the locations.
+ */
+ public PathLocation(
+ String source, List<RemoteLocation> dest, DestinationOrder order) {
+ this.sourcePath = source;
+ this.destinations = Collections.unmodifiableList(dest);
+ this.destOrder = order;
+ }
+
+ /**
+ * Create a new PathLocation with default HASH order.
+ *
+ * @param source Source path in the global name space.
+ * @param dest Destinations of the mount table entry.
+ */
+ public PathLocation(String source, List<RemoteLocation> dest) {
+ this(source, dest, DestinationOrder.HASH);
+ }
+
+ /**
+ * Create a path location from another path.
+ *
+ * @param other Other path location to copy from.
+ */
+ public PathLocation(final PathLocation other) {
+ this.sourcePath = other.sourcePath;
+ this.destinations = Collections.unmodifiableList(other.destinations);
+ this.destOrder = other.destOrder;
+ }
+
+ /**
+ * Create a path location from another path with the destinations sorted.
+ *
+ * @param other Other path location to copy from.
+ * @param firstNsId Identifier of the namespace to place first.
+ */
+ public PathLocation(PathLocation other, String firstNsId) {
+ this.sourcePath = other.sourcePath;
+ this.destOrder = other.destOrder;
+ this.destinations = orderedNamespaces(other.destinations, firstNsId);
+ }
+
+ /**
+ * Prioritize a location/destination by its name space/nameserviceId.
+ * This destination might be used by other threads, so the source is not
+ * modifiable.
+ *
+ * @param original List of destinations to order.
+ * @param nsId The name space/nameserviceID to prioritize.
+ * @return Prioritized list of detinations that cannot be modified.
+ */
+ private static List<RemoteLocation> orderedNamespaces(
+ final List<RemoteLocation> original, final String nsId) {
+ if (original.size() <= 1) {
+ return original;
+ }
+
+ LinkedList<RemoteLocation> newDestinations = new LinkedList<>();
+ boolean found = false;
+ for (RemoteLocation dest : original) {
+ if (dest.getNameserviceId().equals(nsId)) {
+ found = true;
+ newDestinations.addFirst(dest);
+ } else {
+ newDestinations.add(dest);
+ }
+ }
+
+ if (!found) {
+ LOG.debug("Cannot find location with namespace {} in {}",
+ nsId, original);
+ }
+ return Collections.unmodifiableList(newDestinations);
+ }
+
+ /**
+ * Get the source path in the global namespace for this path location.
+ *
+ * @return The path in the global namespace.
+ */
+ public String getSourcePath() {
+ return this.sourcePath;
+ }
+
+ /**
+ * Get the subclusters defined for the destinations.
+ *
+ * @return Set containing the subclusters.
+ */
+ public Set<String> getNamespaces() {
+ Set<String> namespaces = new HashSet<>();
+ List<RemoteLocation> locations = this.getDestinations();
+ for (RemoteLocation location : locations) {
+ String nsId = location.getNameserviceId();
+ namespaces.add(nsId);
+ }
+ return namespaces;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ for (RemoteLocation destination : this.destinations) {
+ String nsId = destination.getNameserviceId();
+ String path = destination.getDest();
+ if (sb.length() > 0) {
+ sb.append(",");
+ }
+ sb.append(nsId + "->" + path);
+ }
+ if (this.destinations.size() > 1) {
+ sb.append(" [");
+ sb.append(this.destOrder.toString());
+ sb.append("]");
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Check if this location supports multiple clusters/paths.
+ *
+ * @return If it has multiple destinations.
+ */
+ public boolean hasMultipleDestinations() {
+ return this.destinations.size() > 1;
+ }
+
+ /**
+ * Get the list of locations found in the mount table.
+ * The first result is the highest priority path.
+ *
+ * @return List of remote locations.
+ */
+ public List<RemoteLocation> getDestinations() {
+ return Collections.unmodifiableList(this.destinations);
+ }
+
+ /**
+ * Get the order for the destinations.
+ *
+ * @return Order for the destinations.
+ */
+ public DestinationOrder getDestinationOrder() {
+ return this.destOrder;
+ }
+
+ /**
+ * Get the default or highest priority location.
+ *
+ * @return The default location.
+ */
+ public RemoteLocation getDefaultLocation() {
+ if (destinations.isEmpty() || destinations.get(0).getDest() == null) {
+ throw new UnsupportedOperationException(
+ "Unsupported path " + sourcePath + " please check mount table");
+ }
+ return destinations.get(0);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java
new file mode 100644
index 0000000..6aa12ce
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RemoteLocation.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import org.apache.hadoop.hdfs.server.federation.router.RemoteLocationContext;
+
+/**
+ * A single in a remote namespace consisting of a nameservice ID
+ * and a HDFS path.
+ */
+public class RemoteLocation extends RemoteLocationContext {
+
+ /** Identifier of the remote namespace for this location. */
+ private final String nameserviceId;
+ /** Identifier of the namenode in the namespace for this location. */
+ private final String namenodeId;
+ /** Path in the remote location. */
+ private final String path;
+
+ /**
+ * Create a new remote location.
+ *
+ * @param nsId
+ * @param pPath
+ */
+ public RemoteLocation(String nsId, String pPath) {
+ this(nsId, null, pPath);
+ }
+
+ /**
+ * Create a new remote location pointing to a particular namenode in the
+ * namespace.
+ *
+ * @param nsId Destination namespace.
+ * @param pPath Path in the destination namespace.
+ */
+ public RemoteLocation(String nsId, String nnId, String pPath) {
+ this.nameserviceId = nsId;
+ this.namenodeId = nnId;
+ this.path = pPath;
+ }
+
+ @Override
+ public String getNameserviceId() {
+ String ret = this.nameserviceId;
+ if (this.namenodeId != null) {
+ ret += "-" + this.namenodeId;
+ }
+ return ret;
+ }
+
+ @Override
+ public String getDest() {
+ return this.path;
+ }
+
+ @Override
+ public String toString() {
+ return getNameserviceId() + "->" + this.path;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/DestinationOrder.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/DestinationOrder.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/DestinationOrder.java
new file mode 100644
index 0000000..03e68e5
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/DestinationOrder.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver.order;
+
+/**
+ * Order of the destinations when we have multiple of them. When the resolver
+ * of files to subclusters (FileSubclusterResolver) has multiple destinations,
+ * this determines which location should be checked first.
+ */
+public enum DestinationOrder {
+ HASH, // Follow consistent hashing in the first folder level
+ LOCAL, // Local first
+ RANDOM, // Random order
+ HASH_ALL // Follow consistent hashing
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/HashFirstResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/HashFirstResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/HashFirstResolver.java
new file mode 100644
index 0000000..831b082
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/HashFirstResolver.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver.order;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
+
+/**
+ * Variation of HashResolver that only uses the first level of the path.
+ */
+public class HashFirstResolver extends HashResolver {
+
+ @Override
+ public String getFirstNamespace(final String path, final PathLocation loc) {
+ String srcPath = loc.getSourcePath();
+ String trimmedPath = trimPathToChild(path, srcPath);
+ LOG.debug("Only using the first part of the path: {} -> {}",
+ path, trimmedPath);
+ return super.getFirstNamespace(trimmedPath, loc);
+ }
+
+ /**
+ * Hash only up to the immediate child of the mount point. This prevents the
+ * need to create/maintain subtrees under each multi-destination mount point.
+ * Each child of a multi-destination mount is mapped to only one hash
+ * location.
+ * <p>
+ * Trims a path to at most the immediate child of a parent path. For example:
+ * <ul>
+ * <li>path = /a/b/c, parent = /a will be trimmed to /a/b.
+ * <li>path = /a/b, parent = /a/b will be trimmed to /a/b
+ * </ul>
+ *
+ * @param path The path to trim.
+ * @param parent The parent used to find the immediate child.
+ * @return Trimmed path.
+ */
+ private static String trimPathToChild(String path, String parent) {
+ // Path is invalid or equal to the parent
+ if (path.length() <= parent.length()) {
+ return parent;
+ }
+ String remainder = path.substring(parent.length());
+ String[] components =
+ remainder.replaceFirst("^/", "").split(Path.SEPARATOR);
+ if (components.length > 0 && components[0].length() > 0) {
+ if (parent.endsWith(Path.SEPARATOR)) {
+ return parent + components[0];
+ } else {
+ return parent + Path.SEPARATOR + components[0];
+ }
+ } else {
+ return parent;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/HashResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/HashResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/HashResolver.java
new file mode 100644
index 0000000..4034a46
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/HashResolver.java
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver.order;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
+import org.apache.hadoop.hdfs.server.federation.utils.ConsistentHashRing;
+import org.apache.hadoop.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Order the destinations based on consistent hashing.
+ */
+public class HashResolver implements OrderedResolver {
+
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(HashResolver.class);
+
+
+ /** Namespace set hash -> Locator. */
+ private final Map<Integer, ConsistentHashRing> hashResolverMap;
+
+ /** Patterns for temporary files. */
+ private static final String HEX_PATTERN = "\\p{XDigit}";
+ private static final String UUID_PATTERN = HEX_PATTERN + "{8}-" +
+ HEX_PATTERN + "{4}-" + HEX_PATTERN + "{4}-" + HEX_PATTERN + "{4}-" +
+ HEX_PATTERN + "{12}";
+ private static final String ATTEMPT_PATTERN =
+ "attempt_\\d+_\\d{4}_._\\d{6}_\\d{2}";
+ private static final String[] TEMP_FILE_PATTERNS = {
+ "(.+)\\.COPYING$",
+ "(.+)\\._COPYING_.*$",
+ "(.+)\\.tmp$",
+ "_temp/(.+)$",
+ "_temporary/(.+)\\." + UUID_PATTERN + "$",
+ "(.*)_temporary/\\d/_temporary/" + ATTEMPT_PATTERN + "/(.+)$" };
+ /** Pattern for temporary files (or of the individual patterns). */
+ private static final Pattern TEMP_FILE_PATTERN =
+ Pattern.compile(StringUtils.join("|", TEMP_FILE_PATTERNS));
+
+
+ public HashResolver() {
+ this.hashResolverMap = new ConcurrentHashMap<>();
+ }
+
+ /**
+ * Use the result from consistent hashing locator to prioritize the locations
+ * for a path.
+ *
+ * @param path Path to check.
+ * @param loc Federated location with multiple destinations.
+ * @return First namespace based on hash.
+ */
+ @Override
+ public String getFirstNamespace(final String path, final PathLocation loc) {
+ String finalPath = extractTempFileName(path);
+ Set<String> namespaces = loc.getNamespaces();
+ ConsistentHashRing locator = getHashResolver(namespaces);
+ String hashedSubcluster = locator.getLocation(finalPath);
+ if (hashedSubcluster == null) {
+ String srcPath = loc.getSourcePath();
+ LOG.error("Cannot find subcluster for {} ({} -> {})",
+ srcPath, path, finalPath);
+ }
+ LOG.debug("Namespace for {} ({}) is {}", path, finalPath, hashedSubcluster);
+ return hashedSubcluster;
+ }
+
+ /**
+ * Get the cached (if available) or generate a new hash resolver for this
+ * particular set of unique namespace identifiers.
+ *
+ * @param namespaces A set of unique namespace identifiers.
+ * @return A hash resolver configured to consistently resolve paths to
+ * namespaces using the provided set of namespace identifiers.
+ */
+ private ConsistentHashRing getHashResolver(final Set<String> namespaces) {
+ int hash = namespaces.hashCode();
+ ConsistentHashRing resolver = this.hashResolverMap.get(hash);
+ if (resolver == null) {
+ resolver = new ConsistentHashRing(namespaces);
+ this.hashResolverMap.put(hash, resolver);
+ }
+ return resolver;
+ }
+
+ /**
+ * Some files use a temporary naming pattern. Extract the final name from the
+ * temporary name. For example, files *._COPYING_ will be renamed, so we
+ * remove that chunk.
+ *
+ * @param input Input string.
+ * @return Final file name.
+ */
+ @VisibleForTesting
+ public static String extractTempFileName(final String input) {
+ StringBuilder sb = new StringBuilder();
+ Matcher matcher = TEMP_FILE_PATTERN.matcher(input);
+ if (matcher.find()) {
+ for (int i=1; i <= matcher.groupCount(); i++) {
+ String match = matcher.group(i);
+ if (match != null) {
+ sb.append(match);
+ }
+ }
+ }
+ if (sb.length() > 0) {
+ String ret = sb.toString();
+ LOG.debug("Extracted {} from {}", ret, input);
+ return ret;
+ }
+ return input;
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/39] hadoop git commit: HDFS-13215. RBF: Move Router to its own
module. Contributed by Wei Yan
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
new file mode 100644
index 0000000..9788683
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
@@ -0,0 +1,1005 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_BIND_HOST_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMENODES;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.addDirectory;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.waitNamenodeRegistered;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_ADMIN_BIND_HOST_KEY;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_CACHE_TIME_TO_LIVE_MS;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HANDLER_COUNT_KEY;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HEARTBEAT_INTERVAL_MS;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HTTPS_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HTTP_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HTTP_BIND_HOST_KEY;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_MONITOR_NAMENODE;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_RPC_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_RPC_BIND_HOST_KEY;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSCluster.NameNodeInfo;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology.NNConf;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology.NSConf;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport;
+import org.apache.hadoop.hdfs.server.federation.router.Router;
+import org.apache.hadoop.hdfs.server.federation.router.RouterClient;
+import org.apache.hadoop.hdfs.server.namenode.FSImage;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.Service.STATE;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test utility to mimic a federated HDFS cluster with multiple routers.
+ */
+public class RouterDFSCluster {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RouterDFSCluster.class);
+
+ public static final String TEST_STRING = "teststring";
+ public static final String TEST_DIR = "testdir";
+ public static final String TEST_FILE = "testfile";
+
+
+ /** Nameservices in the federated cluster. */
+ private List<String> nameservices;
+ /** Namenodes in the federated cluster. */
+ private List<NamenodeContext> namenodes;
+ /** Routers in the federated cluster. */
+ private List<RouterContext> routers;
+ /** If the Namenodes are in high availability.*/
+ private boolean highAvailability;
+ /** Number of datanodes per nameservice. */
+ private int numDatanodesPerNameservice = 2;
+
+ /** Mini cluster. */
+ private MiniDFSCluster cluster;
+
+ protected static final long DEFAULT_HEARTBEAT_INTERVAL_MS =
+ TimeUnit.SECONDS.toMillis(5);
+ protected static final long DEFAULT_CACHE_INTERVAL_MS =
+ TimeUnit.SECONDS.toMillis(5);
+ /** Heartbeat interval in milliseconds. */
+ private long heartbeatInterval;
+ /** Cache flush interval in milliseconds. */
+ private long cacheFlushInterval;
+
+ /** Router configuration overrides. */
+ private Configuration routerOverrides;
+ /** Namenode configuration overrides. */
+ private Configuration namenodeOverrides;
+
+
+ /**
+ * Router context.
+ */
+ public class RouterContext {
+ private Router router;
+ private FileContext fileContext;
+ private String nameserviceId;
+ private String namenodeId;
+ private int rpcPort;
+ private int httpPort;
+ private DFSClient client;
+ private Configuration conf;
+ private RouterClient adminClient;
+ private URI fileSystemUri;
+
+ public RouterContext(Configuration conf, String nsId, String nnId) {
+ this.conf = conf;
+ this.nameserviceId = nsId;
+ this.namenodeId = nnId;
+
+ this.router = new Router();
+ this.router.init(conf);
+ }
+
+ public Router getRouter() {
+ return this.router;
+ }
+
+ public String getNameserviceId() {
+ return this.nameserviceId;
+ }
+
+ public String getNamenodeId() {
+ return this.namenodeId;
+ }
+
+ public int getRpcPort() {
+ return this.rpcPort;
+ }
+
+ public int getHttpPort() {
+ return this.httpPort;
+ }
+
+ public FileContext getFileContext() {
+ return this.fileContext;
+ }
+
+ public void initRouter() throws URISyntaxException {
+ // Store the bound points for the router interfaces
+ InetSocketAddress rpcAddress = router.getRpcServerAddress();
+ if (rpcAddress != null) {
+ this.rpcPort = rpcAddress.getPort();
+ this.fileSystemUri =
+ URI.create("hdfs://" + NetUtils.getHostPortString(rpcAddress));
+ // Override the default FS to point to the router RPC
+ DistributedFileSystem.setDefaultUri(conf, fileSystemUri);
+ try {
+ this.fileContext = FileContext.getFileContext(conf);
+ } catch (UnsupportedFileSystemException e) {
+ this.fileContext = null;
+ }
+ }
+ InetSocketAddress httpAddress = router.getHttpServerAddress();
+ if (httpAddress != null) {
+ this.httpPort = httpAddress.getPort();
+ }
+ }
+
+ public FileSystem getFileSystem() throws IOException {
+ return DistributedFileSystem.get(conf);
+ }
+
+ public DFSClient getClient(UserGroupInformation user)
+ throws IOException, URISyntaxException, InterruptedException {
+
+ LOG.info("Connecting to router at {}", fileSystemUri);
+ return user.doAs(new PrivilegedExceptionAction<DFSClient>() {
+ @Override
+ public DFSClient run() throws IOException {
+ return new DFSClient(fileSystemUri, conf);
+ }
+ });
+ }
+
+ public RouterClient getAdminClient() throws IOException {
+ if (adminClient == null) {
+ InetSocketAddress routerSocket = router.getAdminServerAddress();
+ LOG.info("Connecting to router admin at {}", routerSocket);
+ adminClient = new RouterClient(routerSocket, conf);
+ }
+ return adminClient;
+ }
+
+ public DFSClient getClient() throws IOException, URISyntaxException {
+ if (client == null) {
+ LOG.info("Connecting to router at {}", fileSystemUri);
+ client = new DFSClient(fileSystemUri, conf);
+ }
+ return client;
+ }
+ }
+
+ /**
+ * Namenode context in the federated cluster.
+ */
+ public class NamenodeContext {
+ private Configuration conf;
+ private NameNode namenode;
+ private String nameserviceId;
+ private String namenodeId;
+ private FileContext fileContext;
+ private int rpcPort;
+ private int servicePort;
+ private int lifelinePort;
+ private int httpPort;
+ private URI fileSystemUri;
+ private int index;
+ private DFSClient client;
+
+ public NamenodeContext(
+ Configuration conf, String nsId, String nnId, int index) {
+ this.conf = conf;
+ this.nameserviceId = nsId;
+ this.namenodeId = nnId;
+ this.index = index;
+ }
+
+ public NameNode getNamenode() {
+ return this.namenode;
+ }
+
+ public String getNameserviceId() {
+ return this.nameserviceId;
+ }
+
+ public String getNamenodeId() {
+ return this.namenodeId;
+ }
+
+ public FileContext getFileContext() {
+ return this.fileContext;
+ }
+
+ public void setNamenode(NameNode nn) throws URISyntaxException {
+ this.namenode = nn;
+
+ // Store the bound ports and override the default FS with the local NN RPC
+ this.rpcPort = nn.getNameNodeAddress().getPort();
+ this.servicePort = nn.getServiceRpcAddress().getPort();
+ this.lifelinePort = nn.getServiceRpcAddress().getPort();
+ this.httpPort = nn.getHttpAddress().getPort();
+ this.fileSystemUri = new URI("hdfs://" + namenode.getHostAndPort());
+ DistributedFileSystem.setDefaultUri(this.conf, this.fileSystemUri);
+
+ try {
+ this.fileContext = FileContext.getFileContext(this.conf);
+ } catch (UnsupportedFileSystemException e) {
+ this.fileContext = null;
+ }
+ }
+
+ public String getRpcAddress() {
+ return namenode.getNameNodeAddress().getHostName() + ":" + rpcPort;
+ }
+
+ public String getServiceAddress() {
+ return namenode.getServiceRpcAddress().getHostName() + ":" + servicePort;
+ }
+
+ public String getLifelineAddress() {
+ return namenode.getServiceRpcAddress().getHostName() + ":" + lifelinePort;
+ }
+
+ public String getHttpAddress() {
+ return namenode.getHttpAddress().getHostName() + ":" + httpPort;
+ }
+
+ public FileSystem getFileSystem() throws IOException {
+ return DistributedFileSystem.get(conf);
+ }
+
+ public void resetClient() {
+ client = null;
+ }
+
+ public DFSClient getClient(UserGroupInformation user)
+ throws IOException, URISyntaxException, InterruptedException {
+
+ LOG.info("Connecting to namenode at {}", fileSystemUri);
+ return user.doAs(new PrivilegedExceptionAction<DFSClient>() {
+ @Override
+ public DFSClient run() throws IOException {
+ return new DFSClient(fileSystemUri, conf);
+ }
+ });
+ }
+
+ public DFSClient getClient() throws IOException, URISyntaxException {
+ if (client == null) {
+ LOG.info("Connecting to namenode at {}", fileSystemUri);
+ client = new DFSClient(fileSystemUri, conf);
+ }
+ return client;
+ }
+
+ public String getConfSuffix() {
+ String suffix = nameserviceId;
+ if (highAvailability) {
+ suffix += "." + namenodeId;
+ }
+ return suffix;
+ }
+ }
+
+ public RouterDFSCluster(boolean ha, int numNameservices, int numNamenodes,
+ long heartbeatInterval, long cacheFlushInterval) {
+ this.highAvailability = ha;
+ this.heartbeatInterval = heartbeatInterval;
+ this.cacheFlushInterval = cacheFlushInterval;
+ configureNameservices(numNameservices, numNamenodes);
+ }
+
+ public RouterDFSCluster(boolean ha, int numNameservices) {
+ this(ha, numNameservices, 2,
+ DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_CACHE_INTERVAL_MS);
+ }
+
+ public RouterDFSCluster(boolean ha, int numNameservices, int numNamenodes) {
+ this(ha, numNameservices, numNamenodes,
+ DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_CACHE_INTERVAL_MS);
+ }
+
+ /**
+ * Add configuration settings to override default Router settings.
+ *
+ * @param conf Router configuration overrides.
+ */
+ public void addRouterOverrides(Configuration conf) {
+ if (this.routerOverrides == null) {
+ this.routerOverrides = conf;
+ } else {
+ this.routerOverrides.addResource(conf);
+ }
+ }
+
+ /**
+ * Add configuration settings to override default Namenode settings.
+ *
+ * @param conf Namenode configuration overrides.
+ */
+ public void addNamenodeOverrides(Configuration conf) {
+ if (this.namenodeOverrides == null) {
+ this.namenodeOverrides = conf;
+ } else {
+ this.namenodeOverrides.addResource(conf);
+ }
+ }
+
+ /**
+ * Generate the configuration for a client.
+ *
+ * @param nsId Nameservice identifier.
+ * @return New namenode configuration.
+ */
+ public Configuration generateNamenodeConfiguration(String nsId) {
+ Configuration conf = new HdfsConfiguration();
+
+ conf.set(DFS_NAMESERVICES, getNameservicesKey());
+ conf.set(FS_DEFAULT_NAME_KEY, "hdfs://" + nsId);
+
+ for (String ns : nameservices) {
+ if (highAvailability) {
+ conf.set(
+ DFS_HA_NAMENODES_KEY_PREFIX + "." + ns,
+ NAMENODES[0] + "," + NAMENODES[1]);
+ }
+
+ for (NamenodeContext context : getNamenodes(ns)) {
+ String suffix = context.getConfSuffix();
+
+ conf.set(DFS_NAMENODE_RPC_ADDRESS_KEY + "." + suffix,
+ "127.0.0.1:" + context.rpcPort);
+ conf.set(DFS_NAMENODE_HTTP_ADDRESS_KEY + "." + suffix,
+ "127.0.0.1:" + context.httpPort);
+ conf.set(DFS_NAMENODE_RPC_BIND_HOST_KEY + "." + suffix,
+ "0.0.0.0");
+
+ // If the service port is enabled by default, we need to set them up
+ boolean servicePortEnabled = false;
+ if (servicePortEnabled) {
+ conf.set(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY + "." + suffix,
+ "127.0.0.1:" + context.servicePort);
+ conf.set(DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY + "." + suffix,
+ "0.0.0.0");
+ }
+ }
+ }
+
+ if (this.namenodeOverrides != null) {
+ conf.addResource(this.namenodeOverrides);
+ }
+ return conf;
+ }
+
+ /**
+ * Generate the configuration for a client.
+ *
+ * @return New configuration for a client.
+ */
+ public Configuration generateClientConfiguration() {
+ Configuration conf = new HdfsConfiguration(false);
+ String ns0 = getNameservices().get(0);
+ conf.addResource(generateNamenodeConfiguration(ns0));
+ return conf;
+ }
+
+ /**
+ * Generate the configuration for a Router.
+ *
+ * @param nsId Nameservice identifier.
+ * @param nnId Namenode identifier.
+ * @return New configuration for a Router.
+ */
+ public Configuration generateRouterConfiguration(String nsId, String nnId) {
+
+ Configuration conf = new HdfsConfiguration(false);
+ conf.addResource(generateNamenodeConfiguration(nsId));
+
+ conf.setInt(DFS_ROUTER_HANDLER_COUNT_KEY, 10);
+ conf.set(DFS_ROUTER_RPC_ADDRESS_KEY, "127.0.0.1:0");
+ conf.set(DFS_ROUTER_RPC_BIND_HOST_KEY, "0.0.0.0");
+
+ conf.set(DFS_ROUTER_ADMIN_ADDRESS_KEY, "127.0.0.1:0");
+ conf.set(DFS_ROUTER_ADMIN_BIND_HOST_KEY, "0.0.0.0");
+
+ conf.set(DFS_ROUTER_HTTP_ADDRESS_KEY, "127.0.0.1:0");
+ conf.set(DFS_ROUTER_HTTPS_ADDRESS_KEY, "127.0.0.1:0");
+ conf.set(DFS_ROUTER_HTTP_BIND_HOST_KEY, "0.0.0.0");
+
+ conf.set(DFS_ROUTER_DEFAULT_NAMESERVICE, nameservices.get(0));
+ conf.setLong(DFS_ROUTER_HEARTBEAT_INTERVAL_MS, heartbeatInterval);
+ conf.setLong(DFS_ROUTER_CACHE_TIME_TO_LIVE_MS, cacheFlushInterval);
+
+ // Use mock resolver classes
+ conf.setClass(FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS,
+ MockResolver.class, ActiveNamenodeResolver.class);
+ conf.setClass(FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
+ MockResolver.class, FileSubclusterResolver.class);
+
+ // Set the nameservice ID for the default NN monitor
+ conf.set(DFS_NAMESERVICE_ID, nsId);
+ if (nnId != null) {
+ conf.set(DFS_HA_NAMENODE_ID_KEY, nnId);
+ }
+
+ // Namenodes to monitor
+ StringBuilder sb = new StringBuilder();
+ for (String ns : this.nameservices) {
+ for (NamenodeContext context : getNamenodes(ns)) {
+ String suffix = context.getConfSuffix();
+ if (sb.length() != 0) {
+ sb.append(",");
+ }
+ sb.append(suffix);
+ }
+ }
+ conf.set(DFS_ROUTER_MONITOR_NAMENODE, sb.toString());
+
+ // Add custom overrides if available
+ if (this.routerOverrides != null) {
+ for (Entry<String, String> entry : this.routerOverrides) {
+ String confKey = entry.getKey();
+ String confValue = entry.getValue();
+ conf.set(confKey, confValue);
+ }
+ }
+ return conf;
+ }
+
+ public void configureNameservices(int numNameservices, int numNamenodes) {
+ this.nameservices = new ArrayList<>();
+ this.namenodes = new ArrayList<>();
+
+ NamenodeContext context = null;
+ int nnIndex = 0;
+ for (int i=0; i<numNameservices; i++) {
+ String ns = "ns" + i;
+ this.nameservices.add("ns" + i);
+
+ Configuration nnConf = generateNamenodeConfiguration(ns);
+ if (!highAvailability) {
+ context = new NamenodeContext(nnConf, ns, null, nnIndex++);
+ this.namenodes.add(context);
+ } else {
+ for (int j=0; j<numNamenodes; j++) {
+ context = new NamenodeContext(nnConf, ns, NAMENODES[j], nnIndex++);
+ this.namenodes.add(context);
+ }
+ }
+ }
+ }
+
+ public void setNumDatanodesPerNameservice(int num) {
+ this.numDatanodesPerNameservice = num;
+ }
+
+ public String getNameservicesKey() {
+ StringBuilder sb = new StringBuilder();
+ for (String nsId : this.nameservices) {
+ if (sb.length() > 0) {
+ sb.append(",");
+ }
+ sb.append(nsId);
+ }
+ return sb.toString();
+ }
+
+ public String getRandomNameservice() {
+ Random r = new Random();
+ int randIndex = r.nextInt(nameservices.size());
+ return nameservices.get(randIndex);
+ }
+
+ public List<String> getNameservices() {
+ return nameservices;
+ }
+
+ public List<NamenodeContext> getNamenodes(String nameservice) {
+ List<NamenodeContext> nns = new ArrayList<>();
+ for (NamenodeContext c : namenodes) {
+ if (c.nameserviceId.equals(nameservice)) {
+ nns.add(c);
+ }
+ }
+ return nns;
+ }
+
+ public NamenodeContext getRandomNamenode() {
+ Random rand = new Random();
+ int i = rand.nextInt(this.namenodes.size());
+ return this.namenodes.get(i);
+ }
+
+ public List<NamenodeContext> getNamenodes() {
+ return this.namenodes;
+ }
+
+ public boolean isHighAvailability() {
+ return highAvailability;
+ }
+
+ public NamenodeContext getNamenode(String nameservice, String namenode) {
+ for (NamenodeContext c : this.namenodes) {
+ if (c.nameserviceId.equals(nameservice)) {
+ if (namenode == null || namenode.isEmpty() ||
+ c.namenodeId == null || c.namenodeId.isEmpty()) {
+ return c;
+ } else if (c.namenodeId.equals(namenode)) {
+ return c;
+ }
+ }
+ }
+ return null;
+ }
+
+ public List<RouterContext> getRouters(String nameservice) {
+ List<RouterContext> nns = new ArrayList<>();
+ for (RouterContext c : routers) {
+ if (c.nameserviceId.equals(nameservice)) {
+ nns.add(c);
+ }
+ }
+ return nns;
+ }
+
+ public RouterContext getRouterContext(String nsId, String nnId) {
+ for (RouterContext c : routers) {
+ if (nnId == null) {
+ return c;
+ }
+ if (c.namenodeId.equals(nnId) &&
+ c.nameserviceId.equals(nsId)) {
+ return c;
+ }
+ }
+ return null;
+ }
+
+ public RouterContext getRandomRouter() {
+ Random rand = new Random();
+ return routers.get(rand.nextInt(routers.size()));
+ }
+
+ public List<RouterContext> getRouters() {
+ return routers;
+ }
+
+ public RouterContext buildRouter(String nsId, String nnId)
+ throws URISyntaxException, IOException {
+ Configuration config = generateRouterConfiguration(nsId, nnId);
+ RouterContext rc = new RouterContext(config, nsId, nnId);
+ return rc;
+ }
+
+ public void startCluster() {
+ startCluster(null);
+ }
+
+ public void startCluster(Configuration overrideConf) {
+ try {
+ MiniDFSNNTopology topology = new MiniDFSNNTopology();
+ for (String ns : nameservices) {
+ NSConf conf = new MiniDFSNNTopology.NSConf(ns);
+ if (highAvailability) {
+ for (int i=0; i<namenodes.size()/nameservices.size(); i++) {
+ NNConf nnConf = new MiniDFSNNTopology.NNConf("nn" + i);
+ conf.addNN(nnConf);
+ }
+ } else {
+ NNConf nnConf = new MiniDFSNNTopology.NNConf(null);
+ conf.addNN(nnConf);
+ }
+ topology.addNameservice(conf);
+ }
+ topology.setFederation(true);
+
+ // Start mini DFS cluster
+ String ns0 = nameservices.get(0);
+ Configuration nnConf = generateNamenodeConfiguration(ns0);
+ if (overrideConf != null) {
+ nnConf.addResource(overrideConf);
+ }
+ cluster = new MiniDFSCluster.Builder(nnConf)
+ .numDataNodes(nameservices.size() * numDatanodesPerNameservice)
+ .nnTopology(topology)
+ .build();
+ cluster.waitActive();
+
+ // Store NN pointers
+ for (int i = 0; i < namenodes.size(); i++) {
+ NameNode nn = cluster.getNameNode(i);
+ namenodes.get(i).setNamenode(nn);
+ }
+
+ } catch (Exception e) {
+ LOG.error("Cannot start Router DFS cluster: {}", e.getMessage(), e);
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ public void startRouters()
+ throws InterruptedException, URISyntaxException, IOException {
+
+ // Create one router per nameservice
+ this.routers = new ArrayList<>();
+ for (String ns : this.nameservices) {
+ for (NamenodeContext context : getNamenodes(ns)) {
+ RouterContext router = buildRouter(ns, context.namenodeId);
+ this.routers.add(router);
+ }
+ }
+
+ // Start all routers
+ for (RouterContext router : this.routers) {
+ router.router.start();
+ }
+
+ // Wait until all routers are active and record their ports
+ for (RouterContext router : this.routers) {
+ waitActive(router);
+ router.initRouter();
+ }
+ }
+
+ public void waitActive(NamenodeContext nn) throws IOException {
+ cluster.waitActive(nn.index);
+ }
+
+ public void waitActive(RouterContext router)
+ throws InterruptedException {
+ for (int loopCount = 0; loopCount < 20; loopCount++) {
+ // Validate connection of routers to NNs
+ if (router.router.getServiceState() == STATE.STARTED) {
+ return;
+ }
+ Thread.sleep(1000);
+ }
+ fail("Timeout waiting for " + router.router + " to activate");
+ }
+
+ public void registerNamenodes() throws IOException {
+ for (RouterContext r : this.routers) {
+ ActiveNamenodeResolver resolver = r.router.getNamenodeResolver();
+ for (NamenodeContext nn : this.namenodes) {
+ // Generate a report
+ NamenodeStatusReport report = new NamenodeStatusReport(
+ nn.nameserviceId, nn.namenodeId,
+ nn.getRpcAddress(), nn.getServiceAddress(),
+ nn.getLifelineAddress(), nn.getHttpAddress());
+ FSImage fsImage = nn.namenode.getNamesystem().getFSImage();
+ NamespaceInfo nsInfo = fsImage.getStorage().getNamespaceInfo();
+ report.setNamespaceInfo(nsInfo);
+
+ // Determine HA state from nn public state string
+ String nnState = nn.namenode.getState();
+ HAServiceState haState = HAServiceState.ACTIVE;
+ for (HAServiceState state : HAServiceState.values()) {
+ if (nnState.equalsIgnoreCase(state.name())) {
+ haState = state;
+ break;
+ }
+ }
+ report.setHAServiceState(haState);
+
+ // Register with the resolver
+ resolver.registerNamenode(report);
+ }
+ }
+ }
+
+ public void waitNamenodeRegistration() throws Exception {
+ for (RouterContext r : this.routers) {
+ Router router = r.router;
+ for (NamenodeContext nn : this.namenodes) {
+ ActiveNamenodeResolver nnResolver = router.getNamenodeResolver();
+ waitNamenodeRegistered(
+ nnResolver, nn.nameserviceId, nn.namenodeId, null);
+ }
+ }
+ }
+
+ public void waitRouterRegistrationQuorum(RouterContext router,
+ FederationNamenodeServiceState state, String nsId, String nnId)
+ throws Exception {
+ LOG.info("Waiting for NN {} {} to transition to {}", nsId, nnId, state);
+ ActiveNamenodeResolver nnResolver = router.router.getNamenodeResolver();
+ waitNamenodeRegistered(nnResolver, nsId, nnId, state);
+ }
+
+ /**
+ * Get the federated path for a nameservice.
+ * @param nsId Nameservice identifier.
+ * @return Path in the Router.
+ */
+ public String getFederatedPathForNS(String nsId) {
+ return "/" + nsId;
+ }
+
+ /**
+ * Get the namenode path for a nameservice.
+ * @param nsId Nameservice identifier.
+ * @return Path in the Namenode.
+ */
+ public String getNamenodePathForNS(String nsId) {
+ return "/target-" + nsId;
+ }
+
+ /**
+ * Get the federated test directory for a nameservice.
+ * @param nsId Nameservice identifier.
+ * @return Example:
+ * <ul>
+ * <li>/ns0/testdir which maps to ns0->/target-ns0/testdir
+ * </ul>
+ */
+ public String getFederatedTestDirectoryForNS(String nsId) {
+ return getFederatedPathForNS(nsId) + "/" + TEST_DIR;
+ }
+
+ /**
+ * Get the namenode test directory for a nameservice.
+ * @param nsId Nameservice identifier.
+ * @return example:
+ * <ul>
+ * <li>/target-ns0/testdir
+ * </ul>
+ */
+ public String getNamenodeTestDirectoryForNS(String nsId) {
+ return getNamenodePathForNS(nsId) + "/" + TEST_DIR;
+ }
+
+ /**
+ * Get the federated test file for a nameservice.
+ * @param nsId Nameservice identifier.
+ * @return example:
+ * <ul>
+ * <li>/ns0/testfile which maps to ns0->/target-ns0/testfile
+ * </ul>
+ */
+ public String getFederatedTestFileForNS(String nsId) {
+ return getFederatedPathForNS(nsId) + "/" + TEST_FILE;
+ }
+
+ /**
+ * Get the namenode test file for a nameservice.
+ * @param nsId Nameservice identifier.
+ * @return example:
+ * <ul>
+ * <li>/target-ns0/testfile
+ * </ul>
+ */
+ public String getNamenodeTestFileForNS(String nsId) {
+ return getNamenodePathForNS(nsId) + "/" + TEST_FILE;
+ }
+
+ /**
+ * Switch a namenode in a nameservice to be the active.
+ * @param nsId Nameservice identifier.
+ * @param nnId Namenode identifier.
+ */
+ public void switchToActive(String nsId, String nnId) {
+ try {
+ int total = cluster.getNumNameNodes();
+ NameNodeInfo[] nns = cluster.getNameNodeInfos();
+ for (int i = 0; i < total; i++) {
+ NameNodeInfo nn = nns[i];
+ if (nn.getNameserviceId().equals(nsId) &&
+ nn.getNamenodeId().equals(nnId)) {
+ cluster.transitionToActive(i);
+ }
+ }
+ } catch (Throwable e) {
+ LOG.error("Cannot transition to active", e);
+ }
+ }
+
+ /**
+ * Switch a namenode in a nameservice to be in standby.
+ * @param nsId Nameservice identifier.
+ * @param nnId Namenode identifier.
+ */
+ public void switchToStandby(String nsId, String nnId) {
+ try {
+ int total = cluster.getNumNameNodes();
+ NameNodeInfo[] nns = cluster.getNameNodeInfos();
+ for (int i = 0; i < total; i++) {
+ NameNodeInfo nn = nns[i];
+ if (nn.getNameserviceId().equals(nsId) &&
+ nn.getNamenodeId().equals(nnId)) {
+ cluster.transitionToStandby(i);
+ }
+ }
+ } catch (Throwable e) {
+ LOG.error("Cannot transition to standby", e);
+ }
+ }
+
+ /**
+ * Stop the federated HDFS cluster.
+ */
+ public void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ if (routers != null) {
+ for (RouterContext context : routers) {
+ stopRouter(context);
+ }
+ }
+ }
+
+ /**
+ * Stop a router.
+ * @param router Router context.
+ */
+ public void stopRouter(RouterContext router) {
+ try {
+ router.router.shutDown();
+
+ int loopCount = 0;
+ while (router.router.getServiceState() != STATE.STOPPED) {
+ loopCount++;
+ Thread.sleep(1000);
+ if (loopCount > 20) {
+ LOG.error("Cannot shutdown router {}", router.rpcPort);
+ break;
+ }
+ }
+ } catch (InterruptedException e) {
+ }
+ }
+
+ /////////////////////////////////////////////////////////////////////////////
+ // Namespace Test Fixtures
+ /////////////////////////////////////////////////////////////////////////////
+
+ /**
+ * Creates test directories via the namenode.
+ * 1) /target-ns0/testfile
+ * 2) /target-ns1/testfile
+ * @throws IOException
+ */
+ public void createTestDirectoriesNamenode() throws IOException {
+ // Add a test dir to each NS and verify
+ for (String ns : getNameservices()) {
+ NamenodeContext context = getNamenode(ns, null);
+ if (!createTestDirectoriesNamenode(context)) {
+ throw new IOException("Cannot create test directory for ns " + ns);
+ }
+ }
+ }
+
+ public boolean createTestDirectoriesNamenode(NamenodeContext nn)
+ throws IOException {
+ FileSystem fs = nn.getFileSystem();
+ String testDir = getNamenodeTestDirectoryForNS(nn.nameserviceId);
+ return addDirectory(fs, testDir);
+ }
+
+ public void deleteAllFiles() throws IOException {
+ // Delete all files via the NNs and verify
+ for (NamenodeContext context : getNamenodes()) {
+ FileSystem fs = context.getFileSystem();
+ FileStatus[] status = fs.listStatus(new Path("/"));
+ for (int i = 0; i <status.length; i++) {
+ Path p = status[i].getPath();
+ fs.delete(p, true);
+ }
+ status = fs.listStatus(new Path("/"));
+ assertEquals(status.length, 0);
+ }
+ }
+
+ /////////////////////////////////////////////////////////////////////////////
+ // MockRouterResolver Test Fixtures
+ /////////////////////////////////////////////////////////////////////////////
+
+ /**
+ * <ul>
+ * <li>/ -> [ns0->/].
+ * <li>/nso -> ns0->/target-ns0.
+ * <li>/ns1 -> ns1->/target-ns1.
+ * </ul>
+ */
+ public void installMockLocations() {
+ for (RouterContext r : routers) {
+ MockResolver resolver =
+ (MockResolver) r.router.getSubclusterResolver();
+ // create table entries
+ for (String nsId : nameservices) {
+ // Direct path
+ String routerPath = getFederatedPathForNS(nsId);
+ String nnPath = getNamenodePathForNS(nsId);
+ resolver.addLocation(routerPath, nsId, nnPath);
+ }
+
+ // Root path points to both first nameservice
+ String ns0 = nameservices.get(0);
+ resolver.addLocation("/", ns0, "/");
+ }
+ }
+
+ public MiniDFSCluster getCluster() {
+ return cluster;
+ }
+
+ /**
+ * Wait until the federated cluster is up and ready.
+ * @throws IOException If we cannot wait for the cluster to be up.
+ */
+ public void waitClusterUp() throws IOException {
+ cluster.waitClusterUp();
+ registerNamenodes();
+ try {
+ waitNamenodeRegistration();
+ } catch (Exception e) {
+ throw new IOException("Cannot wait for the namenodes", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/StateStoreDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/StateStoreDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/StateStoreDFSCluster.java
new file mode 100644
index 0000000..aa1906f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/StateStoreDFSCluster.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation;
+
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.createMockRegistrationForNamenode;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.getStateStoreConfiguration;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.synchronizeRecords;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
+import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+
+/**
+ * Test utility to mimic a federated HDFS cluster with a router and a state
+ * store.
+ */
+public class StateStoreDFSCluster extends RouterDFSCluster {
+
+ private static final Class<?> DEFAULT_FILE_RESOLVER =
+ MountTableResolver.class;
+ private static final Class<?> DEFAULT_NAMENODE_RESOLVER =
+ MembershipNamenodeResolver.class;
+
+ public StateStoreDFSCluster(boolean ha, int numNameservices, int numNamenodes,
+ long heartbeatInterval, long cacheFlushInterval)
+ throws IOException, InterruptedException {
+ this(ha, numNameservices, numNamenodes, heartbeatInterval,
+ cacheFlushInterval, DEFAULT_FILE_RESOLVER);
+ }
+
+ public StateStoreDFSCluster(boolean ha, int numNameservices, int numNamenodes,
+ long heartbeatInterval, long cacheFlushInterval, Class<?> fileResolver)
+ throws IOException, InterruptedException {
+ super(ha, numNameservices, numNamenodes, heartbeatInterval,
+ cacheFlushInterval);
+
+ // Attach state store and resolvers to router
+ Configuration stateStoreConfig = getStateStoreConfiguration();
+ // Use state store backed resolvers
+ stateStoreConfig.setClass(
+ RBFConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS,
+ DEFAULT_NAMENODE_RESOLVER, ActiveNamenodeResolver.class);
+ stateStoreConfig.setClass(
+ RBFConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
+ fileResolver, FileSubclusterResolver.class);
+ this.addRouterOverrides(stateStoreConfig);
+ }
+
+ public StateStoreDFSCluster(boolean ha, int numNameservices,
+ Class<?> fileResolver) throws IOException, InterruptedException {
+ this(ha, numNameservices, 2,
+ DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_CACHE_INTERVAL_MS, fileResolver);
+ }
+
+ public StateStoreDFSCluster(boolean ha, int numNameservices)
+ throws IOException, InterruptedException {
+ this(ha, numNameservices, 2,
+ DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_CACHE_INTERVAL_MS);
+ }
+
+ public StateStoreDFSCluster(boolean ha, int numNameservices,
+ int numNamnodes) throws IOException, InterruptedException {
+ this(ha, numNameservices, numNamnodes,
+ DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_CACHE_INTERVAL_MS);
+ }
+
+ /////////////////////////////////////////////////////////////////////////////
+ // State Store Test Fixtures
+ /////////////////////////////////////////////////////////////////////////////
+
+ /**
+ * Adds test fixtures for NN registation for each NN nameservice -> NS
+ * namenode -> NN rpcAddress -> 0.0.0.0:0 webAddress -> 0.0.0.0:0 state ->
+ * STANDBY safeMode -> false blockPool -> test.
+ *
+ * @param stateStore State Store.
+ * @throws IOException If it cannot register.
+ */
+ public void createTestRegistration(StateStoreService stateStore)
+ throws IOException {
+ List<MembershipState> entries = new ArrayList<MembershipState>();
+ for (NamenodeContext nn : this.getNamenodes()) {
+ MembershipState entry = createMockRegistrationForNamenode(
+ nn.getNameserviceId(), nn.getNamenodeId(),
+ FederationNamenodeServiceState.STANDBY);
+ entries.add(entry);
+ }
+ synchronizeRecords(
+ stateStore, entries, MembershipState.class);
+ }
+
+ public void createTestMountTable(StateStoreService stateStore)
+ throws IOException {
+ List<MountTable> mounts = generateMockMountTable();
+ synchronizeRecords(stateStore, mounts, MountTable.class);
+ stateStore.refreshCaches();
+ }
+
+ public List<MountTable> generateMockMountTable() throws IOException {
+ // create table entries
+ List<MountTable> entries = new ArrayList<>();
+ for (String ns : this.getNameservices()) {
+ Map<String, String> destMap = new HashMap<>();
+ destMap.put(ns, getNamenodePathForNS(ns));
+
+ // Direct path
+ String fedPath = getFederatedPathForNS(ns);
+ MountTable entry = MountTable.newInstance(fedPath, destMap);
+ entries.add(entry);
+ }
+
+ // Root path goes to nameservice 1
+ Map<String, String> destMap = new HashMap<>();
+ String ns0 = this.getNameservices().get(0);
+ destMap.put(ns0, "/");
+ MountTable entry = MountTable.newInstance("/", destMap);
+ entries.add(entry);
+ return entries;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestFederationMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestFederationMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestFederationMetrics.java
new file mode 100644
index 0000000..94799f3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestFederationMetrics.java
@@ -0,0 +1,293 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.metrics;
+
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.getBean;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+
+import javax.management.MalformedObjectNameException;
+
+import org.apache.commons.collections.ListUtils;
+import org.apache.hadoop.hdfs.server.federation.router.Router;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipStats;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+import org.apache.hadoop.hdfs.server.federation.store.records.StateStoreVersion;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.junit.Test;
+
+/**
+ * Test the JMX interface for the {@link Router}.
+ */
+public class TestFederationMetrics extends TestMetricsBase {
+
+ public static final String FEDERATION_BEAN =
+ "Hadoop:service=Router,name=FederationState";
+ public static final String STATE_STORE_BEAN =
+ "Hadoop:service=Router,name=StateStore";
+ public static final String RPC_BEAN =
+ "Hadoop:service=Router,name=FederationRPC";
+
+ @Test
+ public void testClusterStatsJMX()
+ throws MalformedObjectNameException, IOException {
+
+ FederationMBean bean = getBean(FEDERATION_BEAN, FederationMBean.class);
+ validateClusterStatsBean(bean);
+ }
+
+ @Test
+ public void testClusterStatsDataSource() throws IOException {
+ FederationMetrics metrics = getRouter().getMetrics();
+ validateClusterStatsBean(metrics);
+ }
+
+ @Test
+ public void testMountTableStatsDataSource()
+ throws IOException, JSONException {
+
+ FederationMetrics metrics = getRouter().getMetrics();
+ String jsonString = metrics.getMountTable();
+ JSONArray jsonArray = new JSONArray(jsonString);
+ assertEquals(jsonArray.length(), getMockMountTable().size());
+
+ int match = 0;
+ for (int i = 0; i < jsonArray.length(); i++) {
+ JSONObject json = jsonArray.getJSONObject(i);
+ String src = json.getString("sourcePath");
+
+ for (MountTable entry : getMockMountTable()) {
+ if (entry.getSourcePath().equals(src)) {
+ assertEquals(entry.getDefaultLocation().getNameserviceId(),
+ json.getString("nameserviceId"));
+ assertEquals(entry.getDefaultLocation().getDest(),
+ json.getString("path"));
+ assertEquals(entry.getOwnerName(), json.getString("ownerName"));
+ assertEquals(entry.getGroupName(), json.getString("groupName"));
+ assertEquals(entry.getMode().toString(), json.getString("mode"));
+ assertEquals(entry.getQuota().toString(), json.getString("quota"));
+ assertNotNullAndNotEmpty(json.getString("dateCreated"));
+ assertNotNullAndNotEmpty(json.getString("dateModified"));
+ match++;
+ }
+ }
+ }
+ assertEquals(match, getMockMountTable().size());
+ }
+
+ private MembershipState findMockNamenode(String nsId, String nnId) {
+
+ @SuppressWarnings("unchecked")
+ List<MembershipState> namenodes =
+ ListUtils.union(getActiveMemberships(), getStandbyMemberships());
+ for (MembershipState nn : namenodes) {
+ if (nn.getNamenodeId().equals(nnId)
+ && nn.getNameserviceId().equals(nsId)) {
+ return nn;
+ }
+ }
+ return null;
+ }
+
+ @Test
+ public void testNamenodeStatsDataSource() throws IOException, JSONException {
+
+ FederationMetrics metrics = getRouter().getMetrics();
+ String jsonString = metrics.getNamenodes();
+ JSONObject jsonObject = new JSONObject(jsonString);
+ Iterator<?> keys = jsonObject.keys();
+ int nnsFound = 0;
+ while (keys.hasNext()) {
+ // Validate each entry against our mocks
+ JSONObject json = jsonObject.getJSONObject((String) keys.next());
+ String nameserviceId = json.getString("nameserviceId");
+ String namenodeId = json.getString("namenodeId");
+
+ MembershipState mockEntry =
+ this.findMockNamenode(nameserviceId, namenodeId);
+ assertNotNull(mockEntry);
+
+ assertEquals(json.getString("state"), mockEntry.getState().toString());
+ MembershipStats stats = mockEntry.getStats();
+ assertEquals(json.getLong("numOfActiveDatanodes"),
+ stats.getNumOfActiveDatanodes());
+ assertEquals(json.getLong("numOfDeadDatanodes"),
+ stats.getNumOfDeadDatanodes());
+ assertEquals(json.getLong("numOfDecommissioningDatanodes"),
+ stats.getNumOfDecommissioningDatanodes());
+ assertEquals(json.getLong("numOfDecomActiveDatanodes"),
+ stats.getNumOfDecomActiveDatanodes());
+ assertEquals(json.getLong("numOfDecomDeadDatanodes"),
+ stats.getNumOfDecomDeadDatanodes());
+ assertEquals(json.getLong("numOfBlocks"), stats.getNumOfBlocks());
+ assertEquals(json.getString("rpcAddress"), mockEntry.getRpcAddress());
+ assertEquals(json.getString("webAddress"), mockEntry.getWebAddress());
+ nnsFound++;
+ }
+ // Validate all memberships are present
+ assertEquals(getActiveMemberships().size() + getStandbyMemberships().size(),
+ nnsFound);
+ }
+
+ @Test
+ public void testNameserviceStatsDataSource()
+ throws IOException, JSONException {
+
+ FederationMetrics metrics = getRouter().getMetrics();
+ String jsonString = metrics.getNameservices();
+ JSONObject jsonObject = new JSONObject(jsonString);
+ Iterator<?> keys = jsonObject.keys();
+ int nameservicesFound = 0;
+ while (keys.hasNext()) {
+ JSONObject json = jsonObject.getJSONObject((String) keys.next());
+ String nameserviceId = json.getString("nameserviceId");
+ String namenodeId = json.getString("namenodeId");
+
+ MembershipState mockEntry =
+ this.findMockNamenode(nameserviceId, namenodeId);
+ assertNotNull(mockEntry);
+
+ // NS should report the active NN
+ assertEquals(mockEntry.getState().toString(), json.getString("state"));
+ assertEquals("ACTIVE", json.getString("state"));
+
+ // Stats in the NS should reflect the stats for the most active NN
+ MembershipStats stats = mockEntry.getStats();
+ assertEquals(stats.getNumOfFiles(), json.getLong("numOfFiles"));
+ assertEquals(stats.getTotalSpace(), json.getLong("totalSpace"));
+ assertEquals(stats.getAvailableSpace(),
+ json.getLong("availableSpace"));
+ assertEquals(stats.getNumOfBlocksMissing(),
+ json.getLong("numOfBlocksMissing"));
+ assertEquals(stats.getNumOfActiveDatanodes(),
+ json.getLong("numOfActiveDatanodes"));
+ assertEquals(stats.getNumOfDeadDatanodes(),
+ json.getLong("numOfDeadDatanodes"));
+ assertEquals(stats.getNumOfDecommissioningDatanodes(),
+ json.getLong("numOfDecommissioningDatanodes"));
+ assertEquals(stats.getNumOfDecomActiveDatanodes(),
+ json.getLong("numOfDecomActiveDatanodes"));
+ assertEquals(stats.getNumOfDecomDeadDatanodes(),
+ json.getLong("numOfDecomDeadDatanodes"));
+ assertEquals(stats.getProvidedSpace(),
+ json.getLong("providedSpace"));
+ nameservicesFound++;
+ }
+ assertEquals(getNameservices().size(), nameservicesFound);
+ }
+
+ @Test
+ public void testRouterStatsDataSource() throws IOException, JSONException {
+
+ FederationMetrics metrics = getRouter().getMetrics();
+ String jsonString = metrics.getRouters();
+ JSONObject jsonObject = new JSONObject(jsonString);
+ Iterator<?> keys = jsonObject.keys();
+ int routersFound = 0;
+ while (keys.hasNext()) {
+ JSONObject json = jsonObject.getJSONObject((String) keys.next());
+ String address = json.getString("address");
+ assertNotNullAndNotEmpty(address);
+ RouterState router = findMockRouter(address);
+ assertNotNull(router);
+
+ assertEquals(router.getStatus().toString(), json.getString("status"));
+ assertEquals(router.getCompileInfo(), json.getString("compileInfo"));
+ assertEquals(router.getVersion(), json.getString("version"));
+ assertEquals(router.getDateStarted(), json.getLong("dateStarted"));
+ assertEquals(router.getDateCreated(), json.getLong("dateCreated"));
+ assertEquals(router.getDateModified(), json.getLong("dateModified"));
+
+ StateStoreVersion version = router.getStateStoreVersion();
+ assertEquals(
+ FederationMetrics.getDateString(version.getMembershipVersion()),
+ json.get("lastMembershipUpdate"));
+ assertEquals(
+ FederationMetrics.getDateString(version.getMountTableVersion()),
+ json.get("lastMountTableUpdate"));
+ assertEquals(version.getMembershipVersion(),
+ json.get("membershipVersion"));
+ assertEquals(version.getMountTableVersion(),
+ json.get("mountTableVersion"));
+ routersFound++;
+ }
+
+ assertEquals(getMockRouters().size(), routersFound);
+ }
+
+ private void assertNotNullAndNotEmpty(String field) {
+ assertNotNull(field);
+ assertTrue(field.length() > 0);
+ }
+
+ private RouterState findMockRouter(String routerId) {
+ for (RouterState router : getMockRouters()) {
+ if (router.getAddress().equals(routerId)) {
+ return router;
+ }
+ }
+ return null;
+ }
+
+ private void validateClusterStatsBean(FederationMBean bean)
+ throws IOException {
+
+ // Determine aggregates
+ long numBlocks = 0;
+ long numLive = 0;
+ long numDead = 0;
+ long numDecom = 0;
+ long numDecomLive = 0;
+ long numDecomDead = 0;
+ long numFiles = 0;
+ for (MembershipState mock : getActiveMemberships()) {
+ MembershipStats stats = mock.getStats();
+ numBlocks += stats.getNumOfBlocks();
+ numLive += stats.getNumOfActiveDatanodes();
+ numDead += stats.getNumOfDeadDatanodes();
+ numDecom += stats.getNumOfDecommissioningDatanodes();
+ numDecomLive += stats.getNumOfDecomActiveDatanodes();
+ numDecomDead += stats.getNumOfDecomDeadDatanodes();
+ }
+
+ assertEquals(numBlocks, bean.getNumBlocks());
+ assertEquals(numLive, bean.getNumLiveNodes());
+ assertEquals(numDead, bean.getNumDeadNodes());
+ assertEquals(numDecom, bean.getNumDecommissioningNodes());
+ assertEquals(numDecomLive, bean.getNumDecomLiveNodes());
+ assertEquals(numDecomDead, bean.getNumDecomDeadNodes());
+ assertEquals(numFiles, bean.getNumFiles());
+ assertEquals(getActiveMemberships().size() + getStandbyMemberships().size(),
+ bean.getNumNamenodes());
+ assertEquals(getNameservices().size(), bean.getNumNameservices());
+ assertTrue(bean.getVersion().length() > 0);
+ assertTrue(bean.getCompiledDate().length() > 0);
+ assertTrue(bean.getCompileInfo().length() > 0);
+ assertTrue(bean.getRouterStarted().length() > 0);
+ assertTrue(bean.getHostAndPort().length() > 0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestMetricsBase.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestMetricsBase.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestMetricsBase.java
new file mode 100644
index 0000000..2169b21
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestMetricsBase.java
@@ -0,0 +1,196 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.metrics;
+
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMENODES;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMESERVICES;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.clearAllRecords;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.createMockMountTable;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.createMockRegistrationForNamenode;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.synchronizeRecords;
+import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.waitStateStore;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.router.Router;
+import org.apache.hadoop.hdfs.server.federation.router.RouterServiceState;
+import org.apache.hadoop.hdfs.server.federation.store.MembershipStore;
+import org.apache.hadoop.hdfs.server.federation.store.RouterStore;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetRouterRegistrationResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RouterHeartbeatRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
+import org.apache.hadoop.hdfs.server.federation.store.records.StateStoreVersion;
+import org.apache.hadoop.util.Time;
+import org.junit.After;
+import org.junit.Before;
+
+/**
+ * Test the basic metrics functionality.
+ */
+public class TestMetricsBase {
+
+ private StateStoreService stateStore;
+ private MembershipStore membershipStore;
+ private RouterStore routerStore;
+ private Router router;
+ private Configuration routerConfig;
+
+ private List<MembershipState> activeMemberships;
+ private List<MembershipState> standbyMemberships;
+ private List<MountTable> mockMountTable;
+ private List<RouterState> mockRouters;
+ private List<String> nameservices;
+
+ @Before
+ public void setupBase() throws Exception {
+
+ if (router == null) {
+ routerConfig = new RouterConfigBuilder()
+ .stateStore()
+ .metrics()
+ .http()
+ .build();
+ router = new Router();
+ router.init(routerConfig);
+ router.setRouterId("routerId");
+ router.start();
+ stateStore = router.getStateStore();
+
+ membershipStore =
+ stateStore.getRegisteredRecordStore(MembershipStore.class);
+ routerStore = stateStore.getRegisteredRecordStore(RouterStore.class);
+
+ // Read all data and load all caches
+ waitStateStore(stateStore, 10000);
+ createFixtures();
+ stateStore.refreshCaches(true);
+ Thread.sleep(1000);
+ }
+ }
+
+ @After
+ public void tearDownBase() throws IOException {
+ if (router != null) {
+ router.stop();
+ router.close();
+ router = null;
+ }
+ }
+
+ private void createFixtures() throws IOException {
+ // Clear all records
+ clearAllRecords(stateStore);
+
+ nameservices = new ArrayList<>();
+ nameservices.add(NAMESERVICES[0]);
+ nameservices.add(NAMESERVICES[1]);
+
+ // 2 NNs per NS
+ activeMemberships = new ArrayList<>();
+ standbyMemberships = new ArrayList<>();
+
+ for (String nameservice : nameservices) {
+ MembershipState namenode1 = createMockRegistrationForNamenode(
+ nameservice, NAMENODES[0], FederationNamenodeServiceState.ACTIVE);
+ NamenodeHeartbeatRequest request1 =
+ NamenodeHeartbeatRequest.newInstance(namenode1);
+ assertTrue(membershipStore.namenodeHeartbeat(request1).getResult());
+ activeMemberships.add(namenode1);
+
+ MembershipState namenode2 = createMockRegistrationForNamenode(
+ nameservice, NAMENODES[1], FederationNamenodeServiceState.STANDBY);
+ NamenodeHeartbeatRequest request2 =
+ NamenodeHeartbeatRequest.newInstance(namenode2);
+ assertTrue(membershipStore.namenodeHeartbeat(request2).getResult());
+ standbyMemberships.add(namenode2);
+ }
+
+ // Add 2 mount table memberships
+ mockMountTable = createMockMountTable(nameservices);
+ synchronizeRecords(stateStore, mockMountTable, MountTable.class);
+
+ // Add 2 router memberships in addition to the running router.
+ long t1 = Time.now();
+ mockRouters = new ArrayList<>();
+ RouterState router1 = RouterState.newInstance(
+ "router1", t1, RouterServiceState.RUNNING);
+ router1.setStateStoreVersion(StateStoreVersion.newInstance(
+ t1 - 1000, t1 - 2000));
+ RouterHeartbeatRequest heartbeatRequest =
+ RouterHeartbeatRequest.newInstance(router1);
+ assertTrue(routerStore.routerHeartbeat(heartbeatRequest).getStatus());
+
+ GetRouterRegistrationRequest getRequest =
+ GetRouterRegistrationRequest.newInstance("router1");
+ GetRouterRegistrationResponse getResponse =
+ routerStore.getRouterRegistration(getRequest);
+ RouterState routerState1 = getResponse.getRouter();
+ mockRouters.add(routerState1);
+
+ long t2 = Time.now();
+ RouterState router2 = RouterState.newInstance(
+ "router2", t2, RouterServiceState.RUNNING);
+ router2.setStateStoreVersion(StateStoreVersion.newInstance(
+ t2 - 6000, t2 - 7000));
+ heartbeatRequest.setRouter(router2);
+ assertTrue(routerStore.routerHeartbeat(heartbeatRequest).getStatus());
+ getRequest.setRouterId("router2");
+ getResponse = routerStore.getRouterRegistration(getRequest);
+ RouterState routerState2 = getResponse.getRouter();
+ mockRouters.add(routerState2);
+ }
+
+ protected Router getRouter() {
+ return router;
+ }
+
+ protected List<MountTable> getMockMountTable() {
+ return mockMountTable;
+ }
+
+ protected List<MembershipState> getActiveMemberships() {
+ return activeMemberships;
+ }
+
+ protected List<MembershipState> getStandbyMemberships() {
+ return standbyMemberships;
+ }
+
+ protected List<String> getNameservices() {
+ return nameservices;
+ }
+
+ protected List<RouterState> getMockRouters() {
+ return mockRouters;
+ }
+
+ protected StateStoreService getStateStore() {
+ return stateStore;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e2b5fa4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
new file mode 100644
index 0000000..cb3b472
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
@@ -0,0 +1,528 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.resolver;
+
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.router.Router;
+import org.apache.hadoop.hdfs.server.federation.store.MountTableStore;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test the {@link MountTableStore} from the {@link Router}.
+ */
+public class TestMountTableResolver {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestMountTableResolver.class);
+
+ private static final int TEST_MAX_CACHE_SIZE = 10;
+
+ private MountTableResolver mountTable;
+
+ private Map<String, String> getMountTableEntry(
+ String subcluster, String path) {
+ Map<String, String> ret = new HashMap<>();
+ ret.put(subcluster, path);
+ return ret;
+ }
+
+ /**
+ * Setup the mount table.
+ * / -> 1:/
+ * __tmp -> 2:/tmp
+ * __user -> 3:/user
+ * ____a -> 2:/user/test
+ * ______demo
+ * ________test
+ * __________a -> 1:/user/test
+ * __________b -> 3:/user/test
+ * ____b
+ * ______file1.txt -> 4:/user/file1.txt
+ * __usr
+ * ____bin -> 2:/bin
+ * __readonly -> 2:/tmp
+ *
+ * @throws IOException If it cannot set the mount table.
+ */
+ private void setupMountTable() throws IOException {
+ Configuration conf = new Configuration();
+ conf.setInt(
+ FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE, TEST_MAX_CACHE_SIZE);
+ conf.setStrings(DFS_ROUTER_DEFAULT_NAMESERVICE, "0");
+ mountTable = new MountTableResolver(conf);
+
+ // Root mount point
+ Map<String, String> map = getMountTableEntry("1", "/");
+ mountTable.addEntry(MountTable.newInstance("/", map));
+
+ // /tmp
+ map = getMountTableEntry("2", "/");
+ mountTable.addEntry(MountTable.newInstance("/tmp", map));
+
+ // /user
+ map = getMountTableEntry("3", "/user");
+ mountTable.addEntry(MountTable.newInstance("/user", map));
+
+ // /usr/bin
+ map = getMountTableEntry("2", "/bin");
+ mountTable.addEntry(MountTable.newInstance("/usr/bin", map));
+
+ // /user/a
+ map = getMountTableEntry("2", "/user/test");
+ mountTable.addEntry(MountTable.newInstance("/user/a", map));
+
+ // /user/b/file1.txt
+ map = getMountTableEntry("4", "/user/file1.txt");
+ mountTable.addEntry(MountTable.newInstance("/user/b/file1.txt", map));
+
+ // /user/a/demo/test/a
+ map = getMountTableEntry("1", "/user/test");
+ mountTable.addEntry(MountTable.newInstance("/user/a/demo/test/a", map));
+
+ // /user/a/demo/test/b
+ map = getMountTableEntry("3", "/user/test");
+ mountTable.addEntry(MountTable.newInstance("/user/a/demo/test/b", map));
+
+ // /readonly
+ map = getMountTableEntry("2", "/tmp");
+ MountTable readOnlyEntry = MountTable.newInstance("/readonly", map);
+ readOnlyEntry.setReadOnly(true);
+ mountTable.addEntry(readOnlyEntry);
+ }
+
+ @Before
+ public void setup() throws IOException {
+ setupMountTable();
+ }
+
+ @Test
+ public void testDestination() throws IOException {
+
+ // Check files
+ assertEquals("1->/tesfile1.txt",
+ mountTable.getDestinationForPath("/tesfile1.txt").toString());
+
+ assertEquals("3->/user/testfile2.txt",
+ mountTable.getDestinationForPath("/user/testfile2.txt").toString());
+
+ assertEquals("2->/user/test/testfile3.txt",
+ mountTable.getDestinationForPath("/user/a/testfile3.txt").toString());
+
+ assertEquals("3->/user/b/testfile4.txt",
+ mountTable.getDestinationForPath("/user/b/testfile4.txt").toString());
+
+ assertEquals("1->/share/file5.txt",
+ mountTable.getDestinationForPath("/share/file5.txt").toString());
+
+ assertEquals("2->/bin/file7.txt",
+ mountTable.getDestinationForPath("/usr/bin/file7.txt").toString());
+
+ assertEquals("1->/usr/file8.txt",
+ mountTable.getDestinationForPath("/usr/file8.txt").toString());
+
+ assertEquals("2->/user/test/demo/file9.txt",
+ mountTable.getDestinationForPath("/user/a/demo/file9.txt").toString());
+
+ // Check folders
+ assertEquals("3->/user/testfolder",
+ mountTable.getDestinationForPath("/user/testfolder").toString());
+
+ assertEquals("2->/user/test/b",
+ mountTable.getDestinationForPath("/user/a/b").toString());
+
+ assertEquals("3->/user/test/a",
+ mountTable.getDestinationForPath("/user/test/a").toString());
+
+ assertEquals("2->/tmp/tesfile1.txt",
+ mountTable.getDestinationForPath("/readonly/tesfile1.txt").toString());
+
+ }
+
+ private void compareLists(List<String> list1, String[] list2) {
+ assertEquals(list1.size(), list2.length);
+ for (String item : list2) {
+ assertTrue(list1.contains(item));
+ }
+ }
+
+ @Test
+ public void testGetMountPoint() throws IOException {
+ // Check get the mount table entry for a path
+ MountTable mtEntry;
+ mtEntry = mountTable.getMountPoint("/");
+ assertTrue(mtEntry.getSourcePath().equals("/"));
+
+ mtEntry = mountTable.getMountPoint("/user");
+ assertTrue(mtEntry.getSourcePath().equals("/user"));
+
+ mtEntry = mountTable.getMountPoint("/user/a");
+ assertTrue(mtEntry.getSourcePath().equals("/user/a"));
+
+ mtEntry = mountTable.getMountPoint("/user/a/");
+ assertTrue(mtEntry.getSourcePath().equals("/user/a"));
+
+ mtEntry = mountTable.getMountPoint("/user/a/11");
+ assertTrue(mtEntry.getSourcePath().equals("/user/a"));
+
+ mtEntry = mountTable.getMountPoint("/user/a1");
+ assertTrue(mtEntry.getSourcePath().equals("/user"));
+ }
+
+ @Test
+ public void testGetMountPoints() throws IOException {
+
+ // Check getting all mount points (virtual and real) beneath a path
+ List<String> mounts = mountTable.getMountPoints("/");
+ assertEquals(4, mounts.size());
+ compareLists(mounts, new String[] {"tmp", "user", "usr", "readonly"});
+
+ mounts = mountTable.getMountPoints("/user");
+ assertEquals(2, mounts.size());
+ compareLists(mounts, new String[] {"a", "b"});
+
+ mounts = mountTable.getMountPoints("/user/a");
+ assertEquals(1, mounts.size());
+ compareLists(mounts, new String[] {"demo"});
+
+ mounts = mountTable.getMountPoints("/user/a/demo");
+ assertEquals(1, mounts.size());
+ compareLists(mounts, new String[] {"test"});
+
+ mounts = mountTable.getMountPoints("/user/a/demo/test");
+ assertEquals(2, mounts.size());
+ compareLists(mounts, new String[] {"a", "b"});
+
+ mounts = mountTable.getMountPoints("/tmp");
+ assertEquals(0, mounts.size());
+
+ mounts = mountTable.getMountPoints("/t");
+ assertNull(mounts);
+
+ mounts = mountTable.getMountPoints("/unknownpath");
+ assertNull(mounts);
+ }
+
+ private void compareRecords(List<MountTable> list1, String[] list2) {
+ assertEquals(list1.size(), list2.length);
+ for (String item : list2) {
+ for (MountTable record : list1) {
+ if (record.getSourcePath().equals(item)) {
+ return;
+ }
+ }
+ }
+ fail();
+ }
+
+ @Test
+ public void testGetMounts() throws IOException {
+
+ // Check listing the mount table records at or beneath a path
+ List<MountTable> records = mountTable.getMounts("/");
+ assertEquals(9, records.size());
+ compareRecords(records, new String[] {"/", "/tmp", "/user", "/usr/bin",
+ "user/a", "/user/a/demo/a", "/user/a/demo/b", "/user/b/file1.txt",
+ "readonly"});
+
+ records = mountTable.getMounts("/user");
+ assertEquals(5, records.size());
+ compareRecords(records, new String[] {"/user", "/user/a/demo/a",
+ "/user/a/demo/b", "user/a", "/user/b/file1.txt"});
+
+ records = mountTable.getMounts("/user/a");
+ assertEquals(3, records.size());
+ compareRecords(records,
+ new String[] {"/user/a/demo/a", "/user/a/demo/b", "/user/a"});
+
+ records = mountTable.getMounts("/tmp");
+ assertEquals(1, records.size());
+ compareRecords(records, new String[] {"/tmp"});
+
+ records = mountTable.getMounts("/readonly");
+ assertEquals(1, records.size());
+ compareRecords(records, new String[] {"/readonly"});
+ assertTrue(records.get(0).isReadOnly());
+ }
+
+ @Test
+ public void testRemoveSubTree()
+ throws UnsupportedOperationException, IOException {
+
+ // 3 mount points are present /tmp, /user, /usr
+ compareLists(mountTable.getMountPoints("/"),
+ new String[] {"user", "usr", "tmp", "readonly"});
+
+ // /tmp currently points to namespace 2
+ assertEquals("2", mountTable.getDestinationForPath("/tmp/testfile.txt")
+ .getDefaultLocation().getNameserviceId());
+
+ // Remove tmp
+ mountTable.removeEntry("/tmp");
+
+ // Now 2 mount points are present /user, /usr
+ compareLists(mountTable.getMountPoints("/"),
+ new String[] {"user", "usr", "readonly"});
+
+ // /tmp no longer exists, uses default namespace for mapping /
+ assertEquals("1", mountTable.getDestinationForPath("/tmp/testfile.txt")
+ .getDefaultLocation().getNameserviceId());
+ }
+
+ @Test
+ public void testRemoveVirtualNode()
+ throws UnsupportedOperationException, IOException {
+
+ // 3 mount points are present /tmp, /user, /usr
+ compareLists(mountTable.getMountPoints("/"),
+ new String[] {"user", "usr", "tmp", "readonly"});
+
+ // /usr is virtual, uses namespace 1->/
+ assertEquals("1", mountTable.getDestinationForPath("/usr/testfile.txt")
+ .getDefaultLocation().getNameserviceId());
+
+ // Attempt to remove /usr
+ mountTable.removeEntry("/usr");
+
+ // Verify the remove failed
+ compareLists(mountTable.getMountPoints("/"),
+ new String[] {"user", "usr", "tmp", "readonly"});
+ }
+
+ @Test
+ public void testRemoveLeafNode()
+ throws UnsupportedOperationException, IOException {
+
+ // /user/a/demo/test/a currently points to namespace 1
+ assertEquals("1", mountTable.getDestinationForPath("/user/a/demo/test/a")
+ .getDefaultLocation().getNameserviceId());
+
+ // Remove /user/a/demo/test/a
+ mountTable.removeEntry("/user/a/demo/test/a");
+
+ // Now /user/a/demo/test/a points to namespace 2 using the entry for /user/a
+ assertEquals("2", mountTable.getDestinationForPath("/user/a/demo/test/a")
+ .getDefaultLocation().getNameserviceId());
+
+ // Verify the virtual node at /user/a/demo still exists and was not deleted
+ compareLists(mountTable.getMountPoints("/user/a"), new String[] {"demo"});
+
+ // Verify the sibling node was unaffected and still points to ns 3
+ assertEquals("3", mountTable.getDestinationForPath("/user/a/demo/test/b")
+ .getDefaultLocation().getNameserviceId());
+ }
+
+ @Test
+ public void testRefreshEntries()
+ throws UnsupportedOperationException, IOException {
+
+ // Initial table loaded
+ testDestination();
+ assertEquals(9, mountTable.getMounts("/").size());
+
+ // Replace table with /1 and /2
+ List<MountTable> records = new ArrayList<>();
+ Map<String, String> map1 = getMountTableEntry("1", "/");
+ records.add(MountTable.newInstance("/1", map1));
+ Map<String, String> map2 = getMountTableEntry("2", "/");
+ records.add(MountTable.newInstance("/2", map2));
+ mountTable.refreshEntries(records);
+
+ // Verify addition
+ PathLocation destination1 = mountTable.getDestinationForPath("/1");
+ RemoteLocation defaultLoc1 = destination1.getDefaultLocation();
+ assertEquals("1", defaultLoc1.getNameserviceId());
+
+ PathLocation destination2 = mountTable.getDestinationForPath("/2");
+ RemoteLocation defaultLoc2 = destination2.getDefaultLocation();
+ assertEquals("2", defaultLoc2.getNameserviceId());
+
+ // Verify existing entries were removed
+ assertEquals(2, mountTable.getMounts("/").size());
+ boolean assertionThrown = false;
+ try {
+ testDestination();
+ fail();
+ } catch (AssertionError e) {
+ // The / entry was removed, so it triggers an exception
+ assertionThrown = true;
+ }
+ assertTrue(assertionThrown);
+ }
+
+ @Test
+ public void testMountTableScalability() throws IOException {
+
+ List<MountTable> emptyList = new ArrayList<>();
+ mountTable.refreshEntries(emptyList);
+
+ // Add 100,000 entries in flat list
+ for (int i = 0; i < 100000; i++) {
+ Map<String, String> map = getMountTableEntry("1", "/" + i);
+ MountTable record = MountTable.newInstance("/" + i, map);
+ mountTable.addEntry(record);
+ if (i % 10000 == 0) {
+ LOG.info("Adding flat mount record {}: {}", i, record);
+ }
+ }
+
+ assertEquals(100000, mountTable.getMountPoints("/").size());
+ assertEquals(100000, mountTable.getMounts("/").size());
+
+ // Add 1000 entries in deep list
+ mountTable.refreshEntries(emptyList);
+ String parent = "/";
+ for (int i = 0; i < 1000; i++) {
+ final int index = i;
+ Map<String, String> map = getMountTableEntry("1", "/" + index);
+ if (i > 0) {
+ parent = parent + "/";
+ }
+ parent = parent + i;
+ MountTable record = MountTable.newInstance(parent, map);
+ mountTable.addEntry(record);
+ }
+
+ assertEquals(1, mountTable.getMountPoints("/").size());
+ assertEquals(1000, mountTable.getMounts("/").size());
+
+ // Add 100,000 entries in deep and wide tree
+ mountTable.refreshEntries(emptyList);
+ Random rand = new Random();
+ parent = "/" + Integer.toString(rand.nextInt());
+ int numRootTrees = 1;
+ for (int i = 0; i < 100000; i++) {
+ final int index = i;
+ Map<String, String> map = getMountTableEntry("1", "/" + index);
+ parent = parent + "/" + i;
+ if (parent.length() > 2000) {
+ // Start new tree
+ parent = "/" + Integer.toString(rand.nextInt());
+ numRootTrees++;
+ }
+ MountTable record = MountTable.newInstance(parent, map);
+ mountTable.addEntry(record);
+ }
+
+ assertEquals(numRootTrees, mountTable.getMountPoints("/").size());
+ assertEquals(100000, mountTable.getMounts("/").size());
+ }
+
+ @Test
+ public void testUpdate() throws IOException {
+
+ // Add entry to update later
+ Map<String, String> map = getMountTableEntry("1", "/");
+ mountTable.addEntry(MountTable.newInstance("/testupdate", map));
+
+ MountTable entry = mountTable.getMountPoint("/testupdate");
+ List<RemoteLocation> dests = entry.getDestinations();
+ assertEquals(1, dests.size());
+ RemoteLocation dest = dests.get(0);
+ assertEquals("1", dest.getNameserviceId());
+
+ // Update entry
+ Collection<MountTable> entries = Collections.singletonList(
+ MountTable.newInstance("/testupdate", getMountTableEntry("2", "/")));
+ mountTable.refreshEntries(entries);
+
+ MountTable entry1 = mountTable.getMountPoint("/testupdate");
+ List<RemoteLocation> dests1 = entry1.getDestinations();
+ assertEquals(1, dests1.size());
+ RemoteLocation dest1 = dests1.get(0);
+ assertEquals("2", dest1.getNameserviceId());
+
+ // Remove the entry to test updates and check
+ mountTable.removeEntry("/testupdate");
+ MountTable entry2 = mountTable.getMountPoint("/testupdate");
+ assertNull(entry2);
+ }
+
+ @Test
+ public void testCacheCleaning() throws Exception {
+ for (int i = 0; i < 1000; i++) {
+ String filename = String.format("/user/a/file-%04d.txt", i);
+ mountTable.getDestinationForPath(filename);
+ }
+ long cacheSize = mountTable.getCacheSize();
+ assertTrue(cacheSize <= TEST_MAX_CACHE_SIZE);
+ }
+
+ @Test
+ public void testLocationCache() throws Exception {
+ List<MountTable> entries = new ArrayList<>();
+
+ // Add entry and test location cache
+ Map<String, String> map1 = getMountTableEntry("1", "/testlocationcache");
+ MountTable entry1 = MountTable.newInstance("/testlocationcache", map1);
+ entries.add(entry1);
+
+ Map<String, String> map2 = getMountTableEntry("2",
+ "/anothertestlocationcache");
+ MountTable entry2 = MountTable.newInstance("/anothertestlocationcache",
+ map2);
+ entries.add(entry2);
+ mountTable.refreshEntries(entries);
+ assertEquals("1->/testlocationcache",
+ mountTable.getDestinationForPath("/testlocationcache").toString());
+ assertEquals("2->/anothertestlocationcache",
+ mountTable.getDestinationForPath("/anothertestlocationcache")
+ .toString());
+
+ // Remove the entry1
+ entries.remove(entry1);
+ mountTable.refreshEntries(entries);
+
+ // Add the default location and test location cache
+ assertEquals("0->/testlocationcache",
+ mountTable.getDestinationForPath("/testlocationcache").toString());
+
+ // Add the entry again but mount to another ns
+ Map<String, String> map3 = getMountTableEntry("3", "/testlocationcache");
+ MountTable entry3 = MountTable.newInstance("/testlocationcache", map3);
+ entries.add(entry3);
+ mountTable.refreshEntries(entries);
+
+ // Ensure location cache update correctly
+ assertEquals("3->/testlocationcache",
+ mountTable.getDestinationForPath("/testlocationcache").toString());
+
+ // Cleanup before exit
+ mountTable.removeEntry("/testlocationcache");
+ mountTable.removeEntry("/anothertestlocationcache");
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org