You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ay...@apache.org on 2023/05/11 03:15:44 UTC
[hadoop] branch trunk updated: HDFS-16978. RBF: Admin command to support bulk add of mount points (#5554). Contributed by Viraj Jasani.
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new fe61d8f073b HDFS-16978. RBF: Admin command to support bulk add of mount points (#5554). Contributed by Viraj Jasani.
fe61d8f073b is described below
commit fe61d8f073b07be250bcbbb07fb44baed2acf6c2
Author: Viraj Jasani <vj...@apache.org>
AuthorDate: Wed May 10 17:15:34 2023 -1000
HDFS-16978. RBF: Admin command to support bulk add of mount points (#5554). Contributed by Viraj Jasani.
Reviewed-by: Inigo Goiri <in...@apache.org>
Reviewed-by: Simbarashe Dzinamarira <sd...@linkedin.com>
Signed-off-by: Ayush Saxena <ay...@apache.org>
---
.../dev-support/findbugsExcludeFile.xml | 12 +
hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 5 +
.../RouterAdminProtocolServerSideTranslatorPB.java | 20 ++
.../RouterAdminProtocolTranslatorPB.java | 19 ++
.../federation/resolver/MountTableManager.java | 13 +
.../federation/router/RouterAdminServer.java | 37 ++-
.../federation/store/impl/MountTableStoreImpl.java | 25 ++
.../protocol/AddMountTableEntriesRequest.java | 51 ++++
.../protocol/AddMountTableEntriesResponse.java | 42 +++
.../impl/pb/AddMountTableEntriesRequestPBImpl.java | 88 ++++++
.../pb/AddMountTableEntriesResponsePBImpl.java | 72 +++++
.../hdfs/tools/federation/AddMountAttributes.java | 190 +++++++++++++
.../hadoop/hdfs/tools/federation/RouterAdmin.java | 311 +++++++++++++--------
.../src/main/proto/FederationProtocol.proto | 8 +
.../src/main/proto/RouterProtocol.proto | 5 +
.../federation/router/TestRouterAdminCLI.java | 80 ++++++
16 files changed, 844 insertions(+), 134 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/dev-support/findbugsExcludeFile.xml
index 40771bfebdb..7051b7f25ec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/dev-support/findbugsExcludeFile.xml
@@ -19,4 +19,16 @@
<Match>
<Package name="org.apache.hadoop.hdfs.federation.protocol.proto" />
</Match>
+ <!-- Only to be used by Router Admin while preparing for bulk add request -->
+ <Match>
+ <Class name="org.apache.hadoop.hdfs.tools.federation.AddMountAttributes" />
+ <Method name="getNss" />
+ <Bug pattern="EI_EXPOSE_REP" />
+ </Match>
+ <!-- Only to be used by Router Admin while preparing for bulk add request -->
+ <Match>
+ <Class name="org.apache.hadoop.hdfs.tools.federation.AddMountAttributes" />
+ <Method name="setNss" />
+ <Bug pattern="EI_EXPOSE_REP2" />
+ </Match>
</FindBugsFilter>
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
index 9e6f12ba8a4..c9e69456b80 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
@@ -301,6 +301,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
+ <configuration>
+ <excludePackageNames>
+ org.apache.hadoop.hdfs.federation.protocol.proto:org.apache.hadoop.hdfs.protocol.proto
+ </excludePackageNames>
+ </configuration>
</plugin>
<plugin>
<groupId>org.apache.rat</groupId>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java
index 6768c3e6706..b4cc482500b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java
@@ -21,6 +21,8 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntriesRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntriesResponseProto;
import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryRequestProto;
import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryResponseProto;
import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.DisableNameserviceRequestProto;
@@ -48,6 +50,8 @@ import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProt
import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshSuperUserGroupsConfigurationRequestProto;
import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshSuperUserGroupsConfigurationResponseProto;
import org.apache.hadoop.hdfs.server.federation.router.RouterAdminServer;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntriesResponse;
import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
import org.apache.hadoop.hdfs.server.federation.store.protocol.DisableNameserviceRequest;
@@ -73,6 +77,8 @@ import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableE
import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.AddMountTableEntriesRequestPBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.AddMountTableEntriesResponsePBImpl;
import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.AddMountTableEntryRequestPBImpl;
import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.AddMountTableEntryResponsePBImpl;
import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.DisableNameserviceRequestPBImpl;
@@ -162,6 +168,20 @@ public class RouterAdminProtocolServerSideTranslatorPB implements
}
}
+ @Override
+ public AddMountTableEntriesResponseProto addMountTableEntries(RpcController controller,
+ AddMountTableEntriesRequestProto request) throws ServiceException {
+ try {
+ AddMountTableEntriesRequest req = new AddMountTableEntriesRequestPBImpl(request);
+ AddMountTableEntriesResponse response = server.addMountTableEntries(req);
+ AddMountTableEntriesResponsePBImpl responsePB =
+ (AddMountTableEntriesResponsePBImpl) response;
+ return responsePB.getProto();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
/**
* Get matching mount table entries.
*/
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java
index 6c26217b76a..6b0df23c406 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java
@@ -22,6 +22,8 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntriesRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntriesResponseProto;
import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryRequestProto;
import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryResponseProto;
import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.DisableNameserviceRequestProto;
@@ -52,6 +54,8 @@ import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
import org.apache.hadoop.hdfs.server.federation.resolver.RouterGenericManager;
import org.apache.hadoop.hdfs.server.federation.router.NameserviceManager;
import org.apache.hadoop.hdfs.server.federation.router.RouterStateManager;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntriesResponse;
import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
import org.apache.hadoop.hdfs.server.federation.store.protocol.DisableNameserviceRequest;
@@ -76,6 +80,8 @@ import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableE
import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.AddMountTableEntriesRequestPBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.AddMountTableEntriesResponsePBImpl;
import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.AddMountTableEntryRequestPBImpl;
import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.AddMountTableEntryResponsePBImpl;
import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.DisableNameserviceRequestPBImpl;
@@ -154,6 +160,19 @@ public class RouterAdminProtocolTranslatorPB
}
}
+ @Override
+ public AddMountTableEntriesResponse addMountTableEntries(AddMountTableEntriesRequest request)
+ throws IOException {
+ AddMountTableEntriesRequestPBImpl requestPB = (AddMountTableEntriesRequestPBImpl) request;
+ AddMountTableEntriesRequestProto proto = requestPB.getProto();
+ try {
+ AddMountTableEntriesResponseProto response = rpcProxy.addMountTableEntries(null, proto);
+ return new AddMountTableEntriesResponsePBImpl(response);
+ } catch (ServiceException e) {
+ throw new IOException(ProtobufHelper.getRemoteException(e).getMessage());
+ }
+ }
+
@Override
public UpdateMountTableEntryResponse updateMountTableEntry(
UpdateMountTableEntryRequest request) throws IOException {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableManager.java
index 5ff2e283292..f49d0407c17 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableManager.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.hdfs.server.federation.resolver;
import java.io.IOException;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntriesResponse;
import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
import org.apache.hadoop.hdfs.server.federation.store.protocol.GetDestinationRequest;
@@ -48,6 +50,17 @@ public interface MountTableManager {
AddMountTableEntryResponse addMountTableEntry(
AddMountTableEntryRequest request) throws IOException;
+ /**
+ * Add multiple entries to the mount table.
+ *
+ * @param request Request object with fully populated list of mount point entries.
+ * @return True if all the mount table entries were successfully committed to the
+ * data store.
+ * @throws IOException Throws exception if the data store is not initialized.
+ */
+ AddMountTableEntriesResponse addMountTableEntries(
+ AddMountTableEntriesRequest request) throws IOException;
+
/**
* Updates an existing entry in the mount table.
*
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
index db1922ad808..21736e520fd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
@@ -31,6 +31,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntriesResponse;
import org.apache.hadoop.util.Preconditions;
import org.apache.hadoop.conf.Configuration;
@@ -349,15 +351,26 @@ public class RouterAdminServer extends AbstractService
MountTable mountTable = request.getEntry();
verifyMaxComponentLength(mountTable);
if (this.mountTableCheckDestination) {
- List<String> nsIds = verifyFileInDestinations(mountTable);
- if (!nsIds.isEmpty()) {
- throw new IllegalArgumentException("File not found in downstream " +
- "nameservices: " + StringUtils.join(",", nsIds));
- }
+ verifyFileExistenceInDest(mountTable);
}
return getMountTableStore().addMountTableEntry(request);
}
+ @Override
+ public AddMountTableEntriesResponse addMountTableEntries(AddMountTableEntriesRequest request)
+ throws IOException {
+ List<MountTable> mountTables = request.getEntries();
+ for (MountTable mountTable : mountTables) {
+ verifyMaxComponentLength(mountTable);
+ }
+ if (this.mountTableCheckDestination) {
+ for (MountTable mountTable : mountTables) {
+ verifyFileExistenceInDest(mountTable);
+ }
+ }
+ return getMountTableStore().addMountTableEntries(request);
+ }
+
@Override
public UpdateMountTableEntryResponse updateMountTableEntry(
UpdateMountTableEntryRequest request) throws IOException {
@@ -366,11 +379,7 @@ public class RouterAdminServer extends AbstractService
// Checks max component length limit.
verifyMaxComponentLength(updateEntry);
if (this.mountTableCheckDestination) {
- List<String> nsIds = verifyFileInDestinations(updateEntry);
- if (!nsIds.isEmpty()) {
- throw new IllegalArgumentException("File not found in downstream " +
- "nameservices: " + StringUtils.join(",", nsIds));
- }
+ verifyFileExistenceInDest(updateEntry);
}
if (this.router.getSubclusterResolver() instanceof MountTableResolver) {
MountTableResolver mResolver =
@@ -408,6 +417,14 @@ public class RouterAdminServer extends AbstractService
return response;
}
+ private void verifyFileExistenceInDest(MountTable mountTable) throws IOException {
+ List<String> nsIds = verifyFileInDestinations(mountTable);
+ if (!nsIds.isEmpty()) {
+ throw new IllegalArgumentException(
+ "File not found in downstream nameservices: " + StringUtils.join(",", nsIds));
+ }
+ }
+
/**
* Checks whether quota needs to be synchronized with namespace or not. Quota
* needs to be synchronized either if there is change in mount entry quota or
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MountTableStoreImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MountTableStoreImpl.java
index a9b0174e6a8..b6428f7923b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MountTableStoreImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MountTableStoreImpl.java
@@ -33,6 +33,8 @@ import org.apache.hadoop.hdfs.server.federation.router.RouterPermissionChecker;
import org.apache.hadoop.hdfs.server.federation.router.RouterQuotaUsage;
import org.apache.hadoop.hdfs.server.federation.store.MountTableStore;
import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntriesResponse;
import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
import org.apache.hadoop.hdfs.server.federation.store.protocol.GetDestinationRequest;
@@ -129,6 +131,29 @@ public class MountTableStoreImpl extends MountTableStore {
}
}
+ @Override
+ public AddMountTableEntriesResponse addMountTableEntries(AddMountTableEntriesRequest request)
+ throws IOException {
+ List<MountTable> mountTables = request.getEntries();
+ if (mountTables == null || mountTables.size() == 0) {
+ AddMountTableEntriesResponse response = AddMountTableEntriesResponse.newInstance();
+ response.setStatus(false);
+ return response;
+ }
+ for (MountTable mountTable : mountTables) {
+ mountTable.validate();
+ final String src = mountTable.getSourcePath();
+ checkMountTablePermission(src);
+ }
+ boolean status = getDriver().putAll(mountTables, false, true);
+ AddMountTableEntriesResponse response = AddMountTableEntriesResponse.newInstance();
+ response.setStatus(status);
+ if (status) {
+ updateCacheAllRouters();
+ }
+ return response;
+ }
+
@Override
public UpdateMountTableEntryResponse updateMountTableEntry(
UpdateMountTableEntryRequest request) throws IOException {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntriesRequest.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntriesRequest.java
new file mode 100644
index 00000000000..fcd5d9ac3c8
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntriesRequest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+
+/**
+ * API request for adding all mount table entries to the state store.
+ */
+public abstract class AddMountTableEntriesRequest {
+
+ public static AddMountTableEntriesRequest newInstance() {
+ return StateStoreSerializer.newRecord(AddMountTableEntriesRequest.class);
+ }
+
+ public static AddMountTableEntriesRequest newInstance(List<MountTable> newEntry) {
+ AddMountTableEntriesRequest request = newInstance();
+ request.setEntries(newEntry);
+ return request;
+ }
+
+ @InterfaceAudience.Public
+ @InterfaceStability.Unstable
+ public abstract List<MountTable> getEntries();
+
+ @InterfaceAudience.Public
+ @InterfaceStability.Unstable
+ public abstract void setEntries(List<MountTable> mount);
+
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntriesResponse.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntriesResponse.java
new file mode 100644
index 00000000000..b010c79932c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/AddMountTableEntriesResponse.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API response for adding multiple mount table entries to the state store.
+ */
+public abstract class AddMountTableEntriesResponse {
+
+ public static AddMountTableEntriesResponse newInstance() throws IOException {
+ return StateStoreSerializer.newRecord(AddMountTableEntriesResponse.class);
+ }
+
+ @Public
+ @Unstable
+ public abstract boolean getStatus();
+
+ @Public
+ @Unstable
+ public abstract void setStatus(boolean result);
+}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntriesRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntriesRequestPBImpl.java
new file mode 100644
index 00000000000..c3ff434be13
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntriesRequestPBImpl.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.thirdparty.protobuf.Message;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntriesRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntriesRequestProtoOrBuilder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProto;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.MountTablePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+/**
+ * Protobuf implementation of the state store API object addMountTableEntriesRequest.
+ */
+public class AddMountTableEntriesRequestPBImpl
+ extends AddMountTableEntriesRequest implements PBRecord {
+
+ private final FederationProtocolPBTranslator<AddMountTableEntriesRequestProto,
+ AddMountTableEntriesRequestProto.Builder,
+ AddMountTableEntriesRequestProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<>(AddMountTableEntriesRequestProto.class);
+
+ public AddMountTableEntriesRequestPBImpl() {
+ }
+
+ public AddMountTableEntriesRequestPBImpl(AddMountTableEntriesRequestProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public AddMountTableEntriesRequestProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public List<MountTable> getEntries() {
+ List<MountTableRecordProto> entryProto = this.translator.getProtoOrBuilder().getEntryList();
+ if (entryProto == null) {
+ return null;
+ }
+ List<MountTable> mountTables = new ArrayList<>();
+ entryProto.forEach(e -> mountTables.add(new MountTablePBImpl(e)));
+ return mountTables;
+ }
+
+ @Override
+ public void setEntries(List<MountTable> mountTables) {
+ for (MountTable mountTable : mountTables) {
+ if (mountTable instanceof MountTablePBImpl) {
+ MountTablePBImpl mountPB = (MountTablePBImpl) mountTable;
+ MountTableRecordProto mountProto = mountPB.getProto();
+ translator.getBuilder().addEntry(mountProto);
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntriesResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntriesResponsePBImpl.java
new file mode 100644
index 00000000000..41bea2976b1
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/AddMountTableEntriesResponsePBImpl.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import java.io.IOException;
+
+import org.apache.hadoop.thirdparty.protobuf.Message;
+
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntriesResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntriesResponseProtoOrBuilder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+/**
+ * Protobuf implementation of the state store API object addMountTableEntriesResponse.
+ */
+public class AddMountTableEntriesResponsePBImpl
+ extends AddMountTableEntriesResponse implements PBRecord {
+
+ private final FederationProtocolPBTranslator<AddMountTableEntriesResponseProto,
+ AddMountTableEntriesResponseProto.Builder,
+ AddMountTableEntriesResponseProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<>(AddMountTableEntriesResponseProto.class);
+
+ public AddMountTableEntriesResponsePBImpl() {
+ }
+
+ public AddMountTableEntriesResponsePBImpl(
+ AddMountTableEntriesResponseProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public AddMountTableEntriesResponseProto getProto() {
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public boolean getStatus() {
+ return this.translator.getProtoOrBuilder().getStatus();
+ }
+
+ @Override
+ public void setStatus(boolean result) {
+ this.translator.getBuilder().setStatus(result);
+ }
+}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/AddMountAttributes.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/AddMountAttributes.java
new file mode 100644
index 00000000000..4059e10c9bb
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/AddMountAttributes.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.tools.federation;
+
+import java.io.IOException;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+
+/**
+ * Add mount entry attributes to be used by Router admin.
+ */
+public class AddMountAttributes {
+
+ private String mount;
+ private String[] nss;
+ private String dest;
+ private boolean readonly;
+ private boolean faultTolerant;
+ private DestinationOrder order;
+ private RouterAdmin.ACLEntity aclInfo;
+ private int paramIndex;
+
+ public String getMount() {
+ return mount;
+ }
+
+ public void setMount(String mount) {
+ this.mount = mount;
+ }
+
+ public String[] getNss() {
+ return nss;
+ }
+
+ public void setNss(String[] nss) {
+ this.nss = nss;
+ }
+
+ public String getDest() {
+ return dest;
+ }
+
+ public void setDest(String dest) {
+ this.dest = dest;
+ }
+
+ public boolean isReadonly() {
+ return readonly;
+ }
+
+ public void setReadonly(boolean readonly) {
+ this.readonly = readonly;
+ }
+
+ public boolean isFaultTolerant() {
+ return faultTolerant;
+ }
+
+ public void setFaultTolerant(boolean faultTolerant) {
+ this.faultTolerant = faultTolerant;
+ }
+
+ public DestinationOrder getOrder() {
+ return order;
+ }
+
+ public void setOrder(DestinationOrder order) {
+ this.order = order;
+ }
+
+ public RouterAdmin.ACLEntity getAclInfo() {
+ return aclInfo;
+ }
+
+ public void setAclInfo(RouterAdmin.ACLEntity aclInfo) {
+ this.aclInfo = aclInfo;
+ }
+
+ public int getParamIndex() {
+ return paramIndex;
+ }
+
+ public void setParamIndex(int paramIndex) {
+ this.paramIndex = paramIndex;
+ }
+
+ /**
+ * Retrieve mount table object with all attributes derived from this object.
+ *
+ * @return MountTable object with updated attributes.
+ * @throws IOException If mount table instantiation fails.
+ */
+ public MountTable getMountTableEntryWithAttributes() throws IOException {
+ String normalizedMount = RouterAdmin.normalizeFileSystemPath(this.getMount());
+ return getMountTableForAddRequest(normalizedMount);
+ }
+
+ /**
+ * Retrieve mount table object with all attributes derived from this object.
+ * The returned mount table could be either new or existing one with updated attributes.
+ *
+ * @param existingEntry Existing mount table entry. If null, new mount table object is created,
+ * otherwise the existing mount table object is updated.
+ * @return MountTable object with updated attributes.
+ * @throws IOException If mount table instantiation fails.
+ */
+ public MountTable getNewOrUpdatedMountTableEntryWithAttributes(MountTable existingEntry)
+ throws IOException {
+ if (existingEntry == null) {
+ return getMountTableForAddRequest(this.mount);
+ } else {
+ // Update the existing entry if it exists
+ for (String nsId : this.getNss()) {
+ if (!existingEntry.addDestination(nsId, this.getDest())) {
+ System.err.println("Cannot add destination at " + nsId + " " + this.getDest());
+ return null;
+ }
+ }
+ updateCommonAttributes(existingEntry);
+ return existingEntry;
+ }
+ }
+
+ /**
+ * Create a new mount table object from the given mount point and update its attributes.
+ *
+ * @param mountSrc mount point src.
+ * @return MountTable object with updated attributes.
+ * @throws IOException If mount table instantiation fails.
+ */
+ private MountTable getMountTableForAddRequest(String mountSrc) throws IOException {
+ Map<String, String> destMap = new LinkedHashMap<>();
+ for (String ns : this.getNss()) {
+ destMap.put(ns, this.getDest());
+ }
+ MountTable newEntry = MountTable.newInstance(mountSrc, destMap);
+ updateCommonAttributes(newEntry);
+ return newEntry;
+ }
+
+ /**
+ * Common attributes like read-only, fault-tolerant, dest order, owner, group, mode etc are
+ * updated for the given mount table object.
+ *
+ * @param existingEntry Mount table object.
+ */
+ private void updateCommonAttributes(MountTable existingEntry) {
+ if (this.isReadonly()) {
+ existingEntry.setReadOnly(true);
+ }
+ if (this.isFaultTolerant()) {
+ existingEntry.setFaultTolerant(true);
+ }
+ if (this.getOrder() != null) {
+ existingEntry.setDestOrder(this.getOrder());
+ }
+ RouterAdmin.ACLEntity mountAclInfo = this.getAclInfo();
+ // Update ACL info of mount table entry
+ if (mountAclInfo.getOwner() != null) {
+ existingEntry.setOwnerName(mountAclInfo.getOwner());
+ }
+ if (mountAclInfo.getGroup() != null) {
+ existingEntry.setGroupName(mountAclInfo.getGroup());
+ }
+ if (mountAclInfo.getMode() != null) {
+ existingEntry.setMode(mountAclInfo.getMode());
+ }
+ existingEntry.validate();
+ }
+
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
index 3ecb4c2caba..a3f7911034e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
@@ -20,13 +20,16 @@ package org.apache.hadoop.hdfs.tools.federation;
import java.io.IOException;
import java.io.PrintStream;
import java.net.InetSocketAddress;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
import java.util.TreeMap;
import java.util.regex.Pattern;
@@ -52,6 +55,8 @@ import org.apache.hadoop.hdfs.server.federation.store.CachedRecordStore;
import org.apache.hadoop.hdfs.server.federation.store.RecordStore;
import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
import org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntriesResponse;
import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
import org.apache.hadoop.hdfs.server.federation.store.protocol.DisableNameserviceRequest;
@@ -106,6 +111,7 @@ public class RouterAdmin extends Configured implements Tool {
private static final Logger LOG = LoggerFactory.getLogger(RouterAdmin.class);
private static final String DUMP_COMMAND = "-dumpState";
+ private static final String ADD_ALL_COMMAND = "-addAll";
private RouterClient client;
@@ -132,15 +138,15 @@ public class RouterAdmin extends Configured implements Tool {
System.out.println(usage);
}
- private void printUsage(String cmd) {
+ private static void printUsage(String cmd) {
String usage = getUsage(cmd);
System.out.println(usage);
}
- private String getUsage(String cmd) {
+ private static String getUsage(String cmd) {
if (cmd == null) {
String[] commands =
- {"-add", "-update", "-rm", "-ls", "-getDestination", "-setQuota",
+ {"-add", ADD_ALL_COMMAND, "-update", "-rm", "-ls", "-getDestination", "-setQuota",
"-setStorageTypeQuota", "-clrQuota", "-clrStorageTypeQuota",
DUMP_COMMAND, "-safemode", "-nameservice", "-getDisabledNameservices",
"-refresh", "-refreshRouterArgs",
@@ -160,6 +166,16 @@ public class RouterAdmin extends Configured implements Tool {
+ "[-readonly] [-faulttolerant] "
+ "[-order HASH|LOCAL|RANDOM|HASH_ALL|SPACE] "
+ "-owner <owner> -group <group> -mode <mode>]";
+ } else if (cmd.equals(ADD_ALL_COMMAND)) {
+ return "\t[" + ADD_ALL_COMMAND + " "
+ + "<source1> <nameservice1,nameservice2,...> <destination1> "
+ + "[-readonly] [-faulttolerant] " + "[-order HASH|LOCAL|RANDOM|HASH_ALL|SPACE] "
+ + "-owner <owner1> -group <group1> -mode <mode1>"
+ + " , "
+ + "<source2> <nameservice1,nameservice2,...> <destination2> "
+ + "[-readonly] [-faulttolerant] " + "[-order HASH|LOCAL|RANDOM|HASH_ALL|SPACE] "
+ + "-owner <owner2> -group <group2> -mode <mode2>"
+ + " , ...]";
} else if (cmd.equals("-update")) {
return "\t[-update <source>"
+ " [<nameservice1, nameservice2, ...> <destination>] "
@@ -423,6 +439,12 @@ public class RouterAdmin extends Configured implements Tool {
exitCode = refreshSuperUserGroupsConfiguration();
} else if ("-refreshCallQueue".equals(cmd)) {
exitCode = refreshCallQueue();
+ } else if (ADD_ALL_COMMAND.equals(cmd)) {
+ if (addAllMount(argv, i)) {
+ System.out.println("Successfully added all mount points ");
+ } else {
+ exitCode = -1;
+ }
} else {
throw new IllegalArgumentException("Unknown Command: " + cmd);
}
@@ -462,6 +484,152 @@ public class RouterAdmin extends Configured implements Tool {
return exitCode;
}
+ /**
+ * Add all mount point entries provided in the request.
+ *
+ * @param parameters Parameters for the mount points.
+ * @param i Current index on the parameters array.
+ * @return True if adding all mount points was successful, False otherwise.
+ * @throws IOException If the RPC call to add the mount points fail.
+ */
+ private boolean addAllMount(String[] parameters, int i) throws IOException {
+ List<AddMountAttributes> addMountAttributesList = new ArrayList<>();
+ Set<String> mounts = new HashSet<>();
+ while (i < parameters.length) {
+ AddMountAttributes addMountAttributes = getAddMountAttributes(parameters, i, true);
+ if (addMountAttributes == null) {
+ return false;
+ }
+ if (!mounts.add(addMountAttributes.getMount())) {
+ System.err.println("Multiple inputs for mount: " + addMountAttributes.getMount());
+ return false;
+ }
+ i = addMountAttributes.getParamIndex();
+ addMountAttributesList.add(addMountAttributes);
+ }
+ List<MountTable> addEntries = getMountTablesFromAddAllAttributes(addMountAttributesList);
+ AddMountTableEntriesRequest request =
+ AddMountTableEntriesRequest.newInstance(addEntries);
+ MountTableManager mountTable = client.getMountTableManager();
+ AddMountTableEntriesResponse addResponse =
+ mountTable.addMountTableEntries(request);
+ boolean added = addResponse.getStatus();
+ if (!added) {
+ System.err.println("Cannot add some or all mount points");
+ }
+ return added;
+ }
+
+ /**
+ * From the given params, form and retrieve AddMountAttributes object. This object is meant
+ * to be used while adding single or multiple mount points with their own specific attributes.
+ *
+ * @param parameters Parameters for the mount point.
+ * @param i Current index on the parameters array.
+ * @param isMultipleAdd True if multiple mount points are to be added, False if single mount
+ * point is to be added.
+ * @return AddMountAttributes object.
+ */
+ private static AddMountAttributes getAddMountAttributes(String[] parameters, int i,
+ boolean isMultipleAdd) {
+ // Mandatory parameters
+ String mount = parameters[i++];
+ String[] nss = parameters[i++].split(",");
+ String destination = parameters[i++];
+
+ if (isMultipleAdd) {
+ String[] destinations = destination.split(",");
+ if (nss.length != destinations.length && destinations.length > 1) {
+ String message =
+ "Invalid namespaces and destinations. The number of destinations " + destinations.length
+ + " is not matched with the number of namespaces " + nss.length;
+ System.err.println(message);
+ return null;
+ }
+ }
+
+ // Optional parameters
+ boolean readOnly = false;
+ boolean faultTolerant = false;
+ String owner = null;
+ String group = null;
+ FsPermission mode = null;
+ DestinationOrder order = DestinationOrder.HASH;
+ while (i < parameters.length) {
+ if (isMultipleAdd && ",".equals(parameters[i])) {
+ i++;
+ break;
+ }
+ switch (parameters[i]) {
+ case "-readonly": {
+ readOnly = true;
+ break;
+ }
+ case "-faulttolerant": {
+ faultTolerant = true;
+ break;
+ }
+ case "-order": {
+ i++;
+ try {
+ order = DestinationOrder.valueOf(parameters[i]);
+ } catch (Exception e) {
+ System.err.println("Cannot parse order: " + parameters[i]);
+ }
+ break;
+ }
+ case "-owner": {
+ i++;
+ owner = parameters[i];
+ break;
+ }
+ case "-group": {
+ i++;
+ group = parameters[i];
+ break;
+ }
+ case "-mode": {
+ i++;
+ short modeValue = Short.parseShort(parameters[i], 8);
+ mode = new FsPermission(modeValue);
+ break;
+ }
+ default: {
+ printUsage(isMultipleAdd ? ADD_ALL_COMMAND : "-add");
+ return null;
+ }
+ }
+ i++;
+ }
+ AddMountAttributes addMountAttributes = new AddMountAttributes();
+ addMountAttributes.setMount(mount);
+ addMountAttributes.setNss(nss);
+ addMountAttributes.setDest(destination);
+ addMountAttributes.setReadonly(readOnly);
+ addMountAttributes.setFaultTolerant(faultTolerant);
+ addMountAttributes.setOrder(order);
+ addMountAttributes.setAclInfo(new ACLEntity(owner, group, mode));
+ addMountAttributes.setParamIndex(i);
+ return addMountAttributes;
+ }
+
+ /**
+ * Prepare and return the list of mount table objects from the given list of
+ * AddMountAttributes objects.
+ *
+ * @param addMountAttributesList The list of AddMountAttributes objects.
+ * @return The list of MountTable objects.
+ * @throws IOException If the creation of the mount table objects fail.
+ */
+ private List<MountTable> getMountTablesFromAddAllAttributes(
+ List<AddMountAttributes> addMountAttributesList) throws IOException {
+ List<MountTable> mountTables = new ArrayList<>();
+ for (AddMountAttributes addMountAttributes : addMountAttributesList) {
+ mountTables.add(addMountAttributes.getMountTableEntryWithAttributes());
+ }
+ return mountTables;
+ }
+
/**
* Refresh superuser proxy groups mappings on Router.
*
@@ -511,149 +679,44 @@ public class RouterAdmin extends Configured implements Tool {
* @throws IOException If it cannot add the mount point.
*/
public boolean addMount(String[] parameters, int i) throws IOException {
- // Mandatory parameters
- String mount = parameters[i++];
- String[] nss = parameters[i++].split(",");
- String dest = parameters[i++];
-
- // Optional parameters
- boolean readOnly = false;
- boolean faultTolerant = false;
- String owner = null;
- String group = null;
- FsPermission mode = null;
- DestinationOrder order = DestinationOrder.HASH;
- while (i < parameters.length) {
- if (parameters[i].equals("-readonly")) {
- readOnly = true;
- } else if (parameters[i].equals("-faulttolerant")) {
- faultTolerant = true;
- } else if (parameters[i].equals("-order")) {
- i++;
- try {
- order = DestinationOrder.valueOf(parameters[i]);
- } catch(Exception e) {
- System.err.println("Cannot parse order: " + parameters[i]);
- }
- } else if (parameters[i].equals("-owner")) {
- i++;
- owner = parameters[i];
- } else if (parameters[i].equals("-group")) {
- i++;
- group = parameters[i];
- } else if (parameters[i].equals("-mode")) {
- i++;
- short modeValue = Short.parseShort(parameters[i], 8);
- mode = new FsPermission(modeValue);
- } else {
- printUsage("-add");
- return false;
- }
-
- i++;
+ AddMountAttributes addMountAttributes = getAddMountAttributes(parameters, i, false);
+ if (addMountAttributes == null) {
+ return false;
}
-
- return addMount(mount, nss, dest, readOnly, faultTolerant, order,
- new ACLEntity(owner, group, mode));
+ return addMount(addMountAttributes);
}
/**
* Add a mount table entry or update if it exists.
*
- * @param mount Mount point.
- * @param nss Namespaces where this is mounted to.
- * @param dest Destination path.
- * @param readonly If the mount point is read only.
- * @param order Order of the destination locations.
- * @param aclInfo the ACL info for mount point.
+ * @param addMountAttributes attributes associated with add mount point request.
* @return If the mount point was added.
* @throws IOException Error adding the mount point.
*/
- public boolean addMount(String mount, String[] nss, String dest,
- boolean readonly, boolean faultTolerant, DestinationOrder order,
- ACLEntity aclInfo)
+ public boolean addMount(AddMountAttributes addMountAttributes)
throws IOException {
- mount = normalizeFileSystemPath(mount);
+ String mount = normalizeFileSystemPath(addMountAttributes.getMount());
// Get the existing entry
MountTableManager mountTable = client.getMountTableManager();
MountTable existingEntry = getMountEntry(mount, mountTable);
+ MountTable existingOrNewEntry =
+ addMountAttributes.getNewOrUpdatedMountTableEntryWithAttributes(existingEntry);
+ if (existingOrNewEntry == null) {
+ return false;
+ }
if (existingEntry == null) {
- // Create and add the entry if it doesn't exist
- Map<String, String> destMap = new LinkedHashMap<>();
- for (String ns : nss) {
- destMap.put(ns, dest);
- }
- MountTable newEntry = MountTable.newInstance(mount, destMap);
- if (readonly) {
- newEntry.setReadOnly(true);
- }
- if (faultTolerant) {
- newEntry.setFaultTolerant(true);
- }
- if (order != null) {
- newEntry.setDestOrder(order);
- }
-
- // Set ACL info for mount table entry
- if (aclInfo.getOwner() != null) {
- newEntry.setOwnerName(aclInfo.getOwner());
- }
-
- if (aclInfo.getGroup() != null) {
- newEntry.setGroupName(aclInfo.getGroup());
- }
-
- if (aclInfo.getMode() != null) {
- newEntry.setMode(aclInfo.getMode());
- }
-
- newEntry.validate();
-
- AddMountTableEntryRequest request =
- AddMountTableEntryRequest.newInstance(newEntry);
- AddMountTableEntryResponse addResponse =
- mountTable.addMountTableEntry(request);
+ AddMountTableEntryRequest request = AddMountTableEntryRequest
+ .newInstance(existingOrNewEntry);
+ AddMountTableEntryResponse addResponse = mountTable.addMountTableEntry(request);
boolean added = addResponse.getStatus();
if (!added) {
System.err.println("Cannot add mount point " + mount);
}
return added;
} else {
- // Update the existing entry if it exists
- for (String nsId : nss) {
- if (!existingEntry.addDestination(nsId, dest)) {
- System.err.println("Cannot add destination at " + nsId + " " + dest);
- return false;
- }
- }
- if (readonly) {
- existingEntry.setReadOnly(true);
- }
- if (faultTolerant) {
- existingEntry.setFaultTolerant(true);
- }
- if (order != null) {
- existingEntry.setDestOrder(order);
- }
-
- // Update ACL info of mount table entry
- if (aclInfo.getOwner() != null) {
- existingEntry.setOwnerName(aclInfo.getOwner());
- }
-
- if (aclInfo.getGroup() != null) {
- existingEntry.setGroupName(aclInfo.getGroup());
- }
-
- if (aclInfo.getMode() != null) {
- existingEntry.setMode(aclInfo.getMode());
- }
-
- existingEntry.validate();
-
UpdateMountTableEntryRequest updateRequest =
- UpdateMountTableEntryRequest.newInstance(existingEntry);
+ UpdateMountTableEntryRequest.newInstance(existingOrNewEntry);
UpdateMountTableEntryResponse updateResponse =
mountTable.updateMountTableEntry(updateRequest);
boolean updated = updateResponse.getStatus();
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto
index c8636826c3c..e83f3227320 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto
@@ -162,6 +162,14 @@ message AddMountTableEntryResponseProto {
optional bool status = 1;
}
+message AddMountTableEntriesRequestProto {
+ repeated MountTableRecordProto entry = 1;
+}
+
+message AddMountTableEntriesResponseProto {
+ optional bool status = 1;
+}
+
message UpdateMountTableEntryRequestProto {
optional MountTableRecordProto entry = 1;
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/RouterProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/RouterProtocol.proto
index 18bad54274b..ea25d296b0b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/RouterProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/RouterProtocol.proto
@@ -40,6 +40,11 @@ service RouterAdminProtocolService {
*/
rpc removeMountTableEntry(RemoveMountTableEntryRequestProto) returns(RemoveMountTableEntryResponseProto);
+ /**
+ * Add all mount table entries that are present in the request.
+ */
+ rpc addMountTableEntries(AddMountTableEntriesRequestProto) returns(AddMountTableEntriesResponseProto);
+
/**
* Get matching mount entries
*/
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
index 75c79dd2c2e..ab2496f3ae1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.federation.router;
import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createNamenodeReport;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import java.io.ByteArrayOutputStream;
@@ -853,6 +854,12 @@ public class TestRouterAdminCLI {
+ "[-readonly] [-faulttolerant] "
+ "[-order HASH|LOCAL|RANDOM|HASH_ALL|SPACE] "
+ "-owner <owner> -group <group> -mode <mode>]\n"
+ + "\t[-addAll <source1> <nameservice1,nameservice2,...> <destination1> "
+ + "[-readonly] [-faulttolerant] [-order HASH|LOCAL|RANDOM|HASH_ALL|SPACE] "
+ + "-owner <owner1> -group <group1> -mode <mode1>"
+ + " , <source2> <nameservice1,nameservice2,...> <destination2> "
+ + "[-readonly] [-faulttolerant] [-order HASH|LOCAL|RANDOM|HASH_ALL|SPACE] "
+ + "-owner <owner2> -group <group2> -mode <mode2> , ...]\n"
+ "\t[-update <source> [<nameservice1, nameservice2, ...> "
+ "<destination>] [-readonly true|false]"
+ " [-faulttolerant true|false] "
@@ -1840,6 +1847,79 @@ public class TestRouterAdminCLI {
buffer.toString().trim().replaceAll("[0-9]{4,}+", "XXX"));
}
+ @Test
+ public void testAddMultipleMountPointsSuccess() throws Exception {
+ String[] argv =
+ new String[] {"-addAll", "/testAddMultipleMountPoints-01", "ns01", "/dest01", ",",
+ "/testAddMultipleMountPoints-02", "ns02,ns03", "/dest02", "-order", "HASH_ALL",
+ "-faulttolerant", ",", "/testAddMultipleMountPoints-03", "ns03", "/dest03"};
+ assertEquals(0, ToolRunner.run(admin, argv));
+
+ stateStore.loadCache(MountTableStoreImpl.class, true);
+
+ validateMountEntry("/testAddMultipleMountPoints-01", 1, new String[] {"/dest01"},
+ new String[] {"ns01"});
+ validateMountEntry("/testAddMultipleMountPoints-02", 2, new String[] {"/dest02", "/dest02"},
+ new String[] {"ns02", "ns03"});
+ validateMountEntry("/testAddMultipleMountPoints-03", 1, new String[] {"/dest03"},
+ new String[] {"ns03"});
+ }
+
+ private static void validateMountEntry(String mountName, int numDest, String[] dest, String[] nss)
+ throws IOException {
+ GetMountTableEntriesRequest request = GetMountTableEntriesRequest.newInstance(mountName);
+ GetMountTableEntriesResponse response =
+ client.getMountTableManager().getMountTableEntries(request);
+ assertEquals(1, response.getEntries().size());
+ List<RemoteLocation> destinations = response.getEntries().get(0).getDestinations();
+ assertEquals(numDest, destinations.size());
+ for (int i = 0; i < numDest; i++) {
+ assertEquals(mountName, destinations.get(i).getSrc());
+ assertEquals(dest[i], destinations.get(i).getDest());
+ assertEquals(nss[i], destinations.get(i).getNameserviceId());
+ }
+ }
+
+ @Test
+ public void testAddMultipleMountPointsFailure() throws Exception {
+ String[] argv =
+ new String[] {"-addAll", "/testAddMultiMountPoints-01", "ns01", ",", "/dest01", ",",
+ "/testAddMultiMountPoints-02", "ns02,ns03", "/dest02", "-order", "HASH_ALL",
+ "-faulttolerant", ",", "/testAddMultiMountPoints-03", "ns03", "/dest03", ",",
+ "/testAddMultiMountPoints-01", "ns02", "/dest02"};
+ // syntax issue
+ assertNotEquals(0, ToolRunner.run(admin, argv));
+
+ argv =
+ new String[] {"-addAll", "/testAddMultiMountPoints-01", "ns01", "/dest01", ",",
+ "/testAddMultiMountPoints-02", "ns02,ns03", "/dest02", "-order", "HASH_ALL",
+ "-faulttolerant", ",", "/testAddMultiMountPoints-03", "ns03", "/dest03", ",",
+ "/testAddMultiMountPoints-01", "ns02", "/dest02"};
+ // multiple inputs with same mount
+ assertNotEquals(0, ToolRunner.run(admin, argv));
+
+ argv =
+ new String[] {"-addAll", "/testAddMultiMountPoints-01", "ns01", "/dest01,/dest02", ",",
+ "/testAddMultiMountPoints-02", "ns02,ns03", "/dest02", "-order", "HASH_ALL",
+ "-faulttolerant"};
+ // multiple dest entries
+ assertNotEquals(0, ToolRunner.run(admin, argv));
+
+ argv =
+ new String[] {"-addAll", "/testAddMultiMountPoints-01", "ns01", "/dest01", ",",
+ "/testAddMultiMountPoints-02", "ns02,ns03", "/dest02", "-order", "HASH_ALL",
+ "-faulttolerant"};
+ // success
+ assertEquals(0, ToolRunner.run(admin, argv));
+
+ argv =
+ new String[] {"-addAll", "/testAddMultiMountPoints-01", "ns01", "/dest01", ",",
+ "/testAddMultiMountPoints-02", "ns02,ns03", "/dest02", "-order", "HASH_ALL",
+ "-faulttolerant"};
+ // mount points were already added
+ assertNotEquals(0, ToolRunner.run(admin, argv));
+ }
+
private void addMountTable(String src, String nsId, String dst)
throws Exception {
String[] argv = new String[] {"-add", src, nsId, dst};
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org