You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ta...@apache.org on 2019/12/12 01:57:04 UTC
[hadoop] branch trunk updated: HDFS-14983. RBF: Add dfsrouteradmin
-refreshSuperUserGroupsConfiguration command option. Contributed by Xieming
Li
This is an automated email from the ASF dual-hosted git repository.
tasanuma pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 93bb368 HDFS-14983. RBF: Add dfsrouteradmin -refreshSuperUserGroupsConfiguration command option. Contributed by Xieming Li
93bb368 is described below
commit 93bb368094e48e752c0732d979fbcd24e432bfc1
Author: Takanobu Asanuma <ta...@apache.org>
AuthorDate: Thu Dec 12 10:56:19 2019 +0900
HDFS-14983. RBF: Add dfsrouteradmin -refreshSuperUserGroupsConfiguration command option. Contributed by Xieming Li
---
.../hdfs/protocolPB/RouterAdminProtocol.java | 4 +-
.../RouterAdminProtocolServerSideTranslatorPB.java | 26 +++
.../RouterAdminProtocolTranslatorPB.java | 21 ++-
.../federation/resolver/RouterGenericManager.java} | 22 ++-
.../federation/router/RouterAdminServer.java | 7 +
.../server/federation/router/RouterClient.java | 5 +
...efreshSuperUserGroupsConfigurationRequest.java} | 25 ++-
...efreshSuperUserGroupsConfigurationResponse.java | 51 ++++++
...hSuperUserGroupsConfigurationRequestPBImpl.java | 66 +++++++
...SuperUserGroupsConfigurationResponsePBImpl.java | 76 ++++++++
.../hadoop/hdfs/tools/federation/RouterAdmin.java | 31 +++-
.../src/main/proto/FederationProtocol.proto | 7 +
.../src/main/proto/RouterProtocol.proto | 5 +
...tRouterRefreshSuperUserGroupsConfiguration.java | 198 +++++++++++++++++++++
.../hadoop-hdfs/src/site/markdown/HDFSCommands.md | 2 +
15 files changed, 518 insertions(+), 28 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocol.java
index d885989..07c8a69 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocol.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.protocolPB;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
+import org.apache.hadoop.hdfs.server.federation.resolver.RouterGenericManager;
import org.apache.hadoop.hdfs.server.federation.router.NameserviceManager;
import org.apache.hadoop.hdfs.server.federation.router.RouterStateManager;
import org.apache.hadoop.ipc.GenericRefreshProtocol;
@@ -30,5 +31,6 @@ import org.apache.hadoop.ipc.GenericRefreshProtocol;
@InterfaceAudience.Private
@InterfaceStability.Stable
public interface RouterAdminProtocol extends MountTableManager,
- RouterStateManager, NameserviceManager, GenericRefreshProtocol {
+ RouterStateManager, NameserviceManager, GenericRefreshProtocol,
+ RouterGenericManager {
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java
index 6f6724e..861df81 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java
@@ -45,6 +45,8 @@ import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProt
import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryResponseProto;
import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryRequestProto;
import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshSuperUserGroupsConfigurationRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshSuperUserGroupsConfigurationResponseProto;
import org.apache.hadoop.hdfs.server.federation.router.RouterAdminServer;
import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
@@ -54,6 +56,7 @@ import org.apache.hadoop.hdfs.server.federation.store.protocol.EnableNameservice
import org.apache.hadoop.hdfs.server.federation.store.protocol.EnableNameserviceResponse;
import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeRequest;
import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RefreshSuperUserGroupsConfigurationResponse;
import org.apache.hadoop.hdfs.server.federation.store.protocol.GetDisabledNameservicesRequest;
import org.apache.hadoop.hdfs.server.federation.store.protocol.GetDisabledNameservicesResponse;
import org.apache.hadoop.hdfs.server.federation.store.protocol.GetDestinationRequest;
@@ -90,6 +93,7 @@ import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.LeaveSafe
import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.LeaveSafeModeResponsePBImpl;
import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RefreshMountTableEntriesRequestPBImpl;
import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RefreshMountTableEntriesResponsePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RefreshSuperUserGroupsConfigurationResponsePBImpl;
import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryRequestPBImpl;
import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryResponsePBImpl;
import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.UpdateMountTableEntryRequestPBImpl;
@@ -197,6 +201,28 @@ public class RouterAdminProtocolServerSideTranslatorPB implements
}
}
+ /**
+ * Refresh superuser proxy groups mappings.
+ */
+ @Override
+ public RefreshSuperUserGroupsConfigurationResponseProto
+ refreshSuperUserGroupsConfiguration(
+ RpcController controller,
+ RefreshSuperUserGroupsConfigurationRequestProto request)
+ throws ServiceException {
+ try {
+ boolean result = server.refreshSuperUserGroupsConfiguration();
+ RefreshSuperUserGroupsConfigurationResponse response =
+ RefreshSuperUserGroupsConfigurationResponsePBImpl.newInstance(result);
+ RefreshSuperUserGroupsConfigurationResponsePBImpl responsePB =
+ (RefreshSuperUserGroupsConfigurationResponsePBImpl) response;
+ return responsePB.getProto();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+
@Override
public EnterSafeModeResponseProto enterSafeMode(RpcController controller,
EnterSafeModeRequestProto request) throws ServiceException {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java
index 9cdc3c1..c4cf73b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java
@@ -42,11 +42,14 @@ import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProt
import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.LeaveSafeModeResponseProto;
import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshMountTableEntriesRequestProto;
import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshMountTableEntriesResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshSuperUserGroupsConfigurationRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshSuperUserGroupsConfigurationResponseProto;
import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryRequestProto;
import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryResponseProto;
import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryRequestProto;
import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryResponseProto;
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
+import org.apache.hadoop.hdfs.server.federation.resolver.RouterGenericManager;
import org.apache.hadoop.hdfs.server.federation.router.NameserviceManager;
import org.apache.hadoop.hdfs.server.federation.router.RouterStateManager;
import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
@@ -89,6 +92,7 @@ import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.GetSafeMo
import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.LeaveSafeModeResponsePBImpl;
import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RefreshMountTableEntriesRequestPBImpl;
import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RefreshMountTableEntriesResponsePBImpl;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RefreshSuperUserGroupsConfigurationResponsePBImpl;
import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryRequestPBImpl;
import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryResponsePBImpl;
import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.UpdateMountTableEntryRequestPBImpl;
@@ -110,7 +114,8 @@ import com.google.protobuf.ServiceException;
@InterfaceStability.Stable
public class RouterAdminProtocolTranslatorPB
implements ProtocolMetaInterface, MountTableManager,
- Closeable, ProtocolTranslator, RouterStateManager, NameserviceManager {
+ Closeable, ProtocolTranslator, RouterStateManager, NameserviceManager,
+ RouterGenericManager {
final private RouterAdminProtocolPB rpcProxy;
public RouterAdminProtocolTranslatorPB(RouterAdminProtocolPB proxy) {
@@ -309,4 +314,18 @@ public class RouterAdminProtocolTranslatorPB
throw new IOException(ProtobufHelper.getRemoteException(e).getMessage());
}
}
+
+ @Override
+ public boolean refreshSuperUserGroupsConfiguration() throws IOException {
+ RefreshSuperUserGroupsConfigurationRequestProto proto =
+ RefreshSuperUserGroupsConfigurationRequestProto.newBuilder().build();
+ try {
+ RefreshSuperUserGroupsConfigurationResponseProto response =
+ rpcProxy.refreshSuperUserGroupsConfiguration(null, proto);
+ return new RefreshSuperUserGroupsConfigurationResponsePBImpl(response)
+ .getStatus();
+ } catch (ServiceException e) {
+ throw new IOException(ProtobufHelper.getRemoteException(e).getMessage());
+ }
+ }
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RouterGenericManager.java
similarity index 54%
copy from hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocol.java
copy to hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RouterGenericManager.java
index d885989..efd898a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RouterGenericManager.java
@@ -15,20 +15,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.hdfs.protocolPB;
+package org.apache.hadoop.hdfs.server.federation.resolver;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
-import org.apache.hadoop.hdfs.server.federation.router.NameserviceManager;
-import org.apache.hadoop.hdfs.server.federation.router.RouterStateManager;
-import org.apache.hadoop.ipc.GenericRefreshProtocol;
+import java.io.IOException;
/**
- * Protocol used by routeradmin to communicate with statestore.
+ * Generic methods for managing Router.
*/
-@InterfaceAudience.Private
-@InterfaceStability.Stable
-public interface RouterAdminProtocol extends MountTableManager,
- RouterStateManager, NameserviceManager, GenericRefreshProtocol {
+public interface RouterGenericManager {
+ /**
+ * Refresh superuser proxy groups mappings (used in RBF).
+ * @return true if the operation was successful.
+ * @throws IOException if operation was not successful.
+ */
+ boolean refreshSuperUserGroupsConfiguration() throws IOException;
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
index 5975cec..cd58926 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
@@ -84,6 +84,7 @@ import org.apache.hadoop.ipc.protocolPB.GenericRefreshProtocolPB;
import org.apache.hadoop.ipc.protocolPB.GenericRefreshProtocolServerSideTranslatorPB;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.service.AbstractService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -619,4 +620,10 @@ public class RouterAdminServer extends AbstractService
// Let the registry handle as needed
return RefreshRegistry.defaultRegistry().dispatch(identifier, args);
}
+
+ @Override // RouterGenericManager
+ public boolean refreshSuperUserGroupsConfiguration() throws IOException {
+ ProxyUsers.refreshSuperUserGroupsConfiguration();
+ return true;
+ }
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClient.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClient.java
index 95fe1a8..0641c0b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClient.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolPB;
import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolTranslatorPB;
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
+import org.apache.hadoop.hdfs.server.federation.resolver.RouterGenericManager;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.net.NetUtils;
@@ -77,6 +78,10 @@ public class RouterClient implements Closeable {
return proxy;
}
+ public RouterGenericManager getRouterGenericManager() {
+ return proxy;
+ }
+
@Override
public synchronized void close() throws IOException {
RPC.stopProxy(proxy);
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RefreshSuperUserGroupsConfigurationRequest.java
similarity index 54%
copy from hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocol.java
copy to hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RefreshSuperUserGroupsConfigurationRequest.java
index d885989..8cc8e89 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RefreshSuperUserGroupsConfigurationRequest.java
@@ -15,20 +15,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.hdfs.protocolPB;
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
-import org.apache.hadoop.hdfs.server.federation.router.NameserviceManager;
-import org.apache.hadoop.hdfs.server.federation.router.RouterStateManager;
-import org.apache.hadoop.ipc.GenericRefreshProtocol;
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
/**
- * Protocol used by routeradmin to communicate with statestore.
+ * API request for refreshing super user groups on router.
*/
-@InterfaceAudience.Private
-@InterfaceStability.Stable
-public interface RouterAdminProtocol extends MountTableManager,
- RouterStateManager, NameserviceManager, GenericRefreshProtocol {
-}
+public abstract class RefreshSuperUserGroupsConfigurationRequest {
+ public static RefreshSuperUserGroupsConfigurationRequest newInstance()
+ throws IOException {
+ return StateStoreSerializer
+ .newRecord(RefreshSuperUserGroupsConfigurationRequest.class);
+ }
+}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RefreshSuperUserGroupsConfigurationResponse.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RefreshSuperUserGroupsConfigurationResponse.java
new file mode 100644
index 0000000..11d265d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RefreshSuperUserGroupsConfigurationResponse.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API response for refreshing super user groups on router.
+ */
+public abstract class RefreshSuperUserGroupsConfigurationResponse {
+
+ public static RefreshSuperUserGroupsConfigurationResponse newInstance()
+ throws IOException {
+ return StateStoreSerializer.
+ newRecord(RefreshSuperUserGroupsConfigurationResponse.class);
+ }
+
+ public static RefreshSuperUserGroupsConfigurationResponse
+ newInstance(boolean status) throws IOException {
+ RefreshSuperUserGroupsConfigurationResponse response = newInstance();
+ response.setStatus(status);
+ return response;
+ }
+
+ @Public
+ @Unstable
+ public abstract boolean getStatus();
+
+ @Public
+ @Unstable
+ public abstract void setStatus(boolean result);
+}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RefreshSuperUserGroupsConfigurationRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RefreshSuperUserGroupsConfigurationRequestPBImpl.java
new file mode 100644
index 0000000..00573fe
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RefreshSuperUserGroupsConfigurationRequestPBImpl.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import com.google.protobuf.Message;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshSuperUserGroupsConfigurationRequestProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshSuperUserGroupsConfigurationRequestProtoOrBuilder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshSuperUserGroupsConfigurationRequestProto.Builder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RefreshSuperUserGroupsConfigurationRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import java.io.IOException;
+
+/**
+ * Protobuf implementation of the state store API object
+ * RefreshSuperUserGroupsConfigurationRequest.
+ */
+public class RefreshSuperUserGroupsConfigurationRequestPBImpl
+ extends RefreshSuperUserGroupsConfigurationRequest
+ implements PBRecord {
+
+ private FederationProtocolPBTranslator<
+ RefreshSuperUserGroupsConfigurationRequestProto, Builder,
+ RefreshSuperUserGroupsConfigurationRequestProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<>(
+ RefreshSuperUserGroupsConfigurationRequestProto.class);
+
+ public RefreshSuperUserGroupsConfigurationRequestPBImpl() {
+ }
+
+ public RefreshSuperUserGroupsConfigurationRequestPBImpl(
+ RefreshSuperUserGroupsConfigurationRequestProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public RefreshSuperUserGroupsConfigurationRequestProto getProto() {
+ this.translator.getBuilder();
+ return this.translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RefreshSuperUserGroupsConfigurationResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RefreshSuperUserGroupsConfigurationResponsePBImpl.java
new file mode 100644
index 0000000..47ae3be
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RefreshSuperUserGroupsConfigurationResponsePBImpl.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb;
+
+import com.google.protobuf.Message;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshSuperUserGroupsConfigurationResponseProto;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshSuperUserGroupsConfigurationResponseProtoOrBuilder;
+import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshSuperUserGroupsConfigurationResponseProto.Builder;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RefreshSuperUserGroupsConfigurationResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
+
+import java.io.IOException;
+
+/**
+ * Protobuf implementation of the state store API object
+ * RefreshSuperUserGroupsConfigurationResponse.
+ */
+public class RefreshSuperUserGroupsConfigurationResponsePBImpl
+ extends RefreshSuperUserGroupsConfigurationResponse
+ implements PBRecord {
+
+ private FederationProtocolPBTranslator<
+ RefreshSuperUserGroupsConfigurationResponseProto,
+ Builder,
+ RefreshSuperUserGroupsConfigurationResponseProtoOrBuilder> translator =
+ new FederationProtocolPBTranslator<>(
+ RefreshSuperUserGroupsConfigurationResponseProto.class);
+
+ public RefreshSuperUserGroupsConfigurationResponsePBImpl() {
+ }
+
+ public RefreshSuperUserGroupsConfigurationResponsePBImpl(
+ RefreshSuperUserGroupsConfigurationResponseProto proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public RefreshSuperUserGroupsConfigurationResponseProto getProto() {
+ return translator.build();
+ }
+
+ @Override
+ public void setProto(Message proto) {
+ this.translator.setProto(proto);
+ }
+
+ @Override
+ public void readInstance(String base64String) throws IOException {
+ this.translator.readInstance(base64String);
+ }
+
+ @Override
+ public boolean getStatus() {
+ return this.translator.getProtoOrBuilder().getStatus();
+ }
+
+ @Override
+ public void setStatus(boolean result) {
+ this.translator.getBuilder().setStatus(result);
+ }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
index 4f8dcde..1f050ad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.federation.resolver.RouterGenericManager;
import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
import org.apache.hadoop.hdfs.server.federation.router.NameserviceManager;
import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
@@ -127,7 +128,8 @@ public class RouterAdmin extends Configured implements Tool {
{"-add", "-update", "-rm", "-ls", "-getDestination",
"-setQuota", "-clrQuota",
"-safemode", "-nameservice", "-getDisabledNameservices",
- "-refresh", "-refreshRouterArgs"};
+ "-refresh", "-refreshRouterArgs",
+ "-refreshSuperUserGroupsConfiguration"};
StringBuilder usage = new StringBuilder();
usage.append("Usage: hdfs dfsrouteradmin :\n");
for (int i = 0; i < commands.length; i++) {
@@ -170,6 +172,8 @@ public class RouterAdmin extends Configured implements Tool {
return "\t[-refresh]";
} else if (cmd.equals("-refreshRouterArgs")) {
return "\t[-refreshRouterArgs <host:ipc_port> <key> [arg1..argn]]";
+ } else if (cmd.equals("-refreshSuperUserGroupsConfiguration")) {
+ return "\t[-refreshSuperUserGroupsConfiguration]";
}
return getUsage(null);
}
@@ -203,6 +207,10 @@ public class RouterAdmin extends Configured implements Tool {
if (arg.length > 1) {
throw new IllegalArgumentException("No arguments allowed");
}
+ } else if (arg[0].equals("-refreshSuperUserGroupsConfiguration")) {
+ if (arg.length > 1) {
+ throw new IllegalArgumentException("No arguments allowed");
+ }
}
}
@@ -348,6 +356,8 @@ public class RouterAdmin extends Configured implements Tool {
refresh(address);
} else if ("-refreshRouterArgs".equals(cmd)) {
exitCode = genericRefresh(argv, i);
+ } else if ("-refreshSuperUserGroupsConfiguration".equals(cmd)) {
+ exitCode = refreshSuperUserGroupsConfiguration();
} else {
throw new IllegalArgumentException("Unknown Command: " + cmd);
}
@@ -387,6 +397,25 @@ public class RouterAdmin extends Configured implements Tool {
return exitCode;
}
+ /**
+ * Refresh superuser proxy groups mappings on Router.
+ *
+ * @throws IOException if the operation was not successful.
+ */
+ private int refreshSuperUserGroupsConfiguration()
+ throws IOException{
+ RouterGenericManager proxy = client.getRouterGenericManager();
+ String address = getConf().getTrimmed(
+ RBFConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_KEY,
+ RBFConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_DEFAULT);
+ if(proxy.refreshSuperUserGroupsConfiguration()){
+ System.out.println(
+ "Successfully updated superuser proxy groups on router " + address);
+ return 0;
+ }
+ return -1;
+ }
+
private void refresh(String address) throws IOException {
if (refreshRouterCache()) {
System.out.println(
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto
index 35566ae..5b6dfd9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto
@@ -242,6 +242,13 @@ message RefreshMountTableEntriesResponseProto {
optional bool result = 1;
}
+message RefreshSuperUserGroupsConfigurationRequestProto {
+}
+
+message RefreshSuperUserGroupsConfigurationResponseProto {
+ optional bool status = 1;
+}
+
/////////////////////////////////////////////////
// Route State
/////////////////////////////////////////////////
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/RouterProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/RouterProtocol.proto
index e8fc268..18bad54 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/RouterProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/RouterProtocol.proto
@@ -84,4 +84,9 @@ service RouterAdminProtocolService {
* Get the destination of a file/directory in the federation.
*/
rpc getDestination(GetDestinationRequestProto) returns (GetDestinationResponseProto);
+
+ /**
+ * Refresh superuser proxy groups mappings on Router.
+ */
+ rpc refreshSuperUserGroupsConfiguration(RefreshSuperUserGroupsConfigurationRequestProto) returns (RefreshSuperUserGroupsConfigurationResponseProto);
}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRefreshSuperUserGroupsConfiguration.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRefreshSuperUserGroupsConfiguration.java
new file mode 100644
index 0000000..fb88882
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRefreshSuperUserGroupsConfiguration.java
@@ -0,0 +1,198 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider;
+import org.apache.hadoop.hdfs.tools.federation.RouterAdmin;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AuthorizationException;
+import org.apache.hadoop.security.authorize.ProxyUsers;
+
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Test refreshSuperUserGroupsConfiguration on Routers.
+ * Notice that this test is intended to test
+ * {@link RouterAdminServer#refreshSuperUserGroupsConfiguration}
+ * which invoked by {@link RouterAdmin}
+ */
+public class TestRouterRefreshSuperUserGroupsConfiguration {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ TestRouterRefreshSuperUserGroupsConfiguration.class);
+
+ private MiniRouterDFSCluster cluster;
+ private static final String ROUTER_NS = "rbfns";
+ private static final String HDFS_SCHEMA = "hdfs://";
+ private static final String LOOPBACK_ADDRESS = "127.0.0.1";
+
+ private String tempResource = null;
+ @Before
+ public void setUpCluster() throws Exception {
+ Configuration conf = new RouterConfigBuilder()
+ .rpc()
+ .admin()
+ .build();
+ cluster = new MiniRouterDFSCluster(false, 1);
+ cluster.addRouterOverrides(conf);
+ cluster.startRouters();
+ }
+
+ @After
+ public void tearDown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+
+ if (tempResource != null) {
+ File f = new File(tempResource);
+ f.delete();
+ tempResource = null;
+ }
+ }
+
+ private Configuration initializeClientConfig() throws Exception {
+
+ Configuration conf = new Configuration(false);
+ Router router = cluster.getRouters().get(0).getRouter();
+
+ conf.set(DFSConfigKeys.DFS_NAMESERVICES, "ns0,ns1,"+ ROUTER_NS);
+ conf.set(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "."+ ROUTER_NS, "r1");
+ conf.set(RBFConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_KEY,
+ LOOPBACK_ADDRESS + ":" + router.getAdminServerAddress().getPort());
+ conf.set(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY
+ + "." + ROUTER_NS + ".r1",
+ LOOPBACK_ADDRESS + ":" + router.getRpcServerAddress().getPort());
+ conf.set(HdfsClientConfigKeys.Failover.PROXY_PROVIDER_KEY_PREFIX
+ + "." + ROUTER_NS,
+ ConfiguredFailoverProxyProvider.class.getCanonicalName());
+ conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY,
+ HDFS_SCHEMA + ROUTER_NS);
+ conf.setBoolean("dfs.client.failover.random.order", false);
+ conf.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, HDFS_SCHEMA + ROUTER_NS);
+
+ return conf;
+ }
+
+ @Test
+ public void testRefreshSuperUserGroupsConfiguration()
+ throws Exception {
+ Configuration conf = initializeClientConfig();
+ testRefreshSuperUserGroupsConfigurationInternal(conf);
+ }
+
+ private void testRefreshSuperUserGroupsConfigurationInternal(
+ Configuration conf) throws Exception {
+
+ UserGroupInformation ugi = mock(UserGroupInformation.class);
+ UserGroupInformation impersonator = mock(UserGroupInformation.class);
+
+ // Setting for impersonator
+ when(impersonator.getShortUserName()).thenReturn("impersonator");
+ when(impersonator.getUserName()).thenReturn("impersonator");
+
+ // Setting for victim
+ when(ugi.getRealUser()).thenReturn(impersonator);
+ when(ugi.getUserName()).thenReturn("victim");
+ when(ugi.getGroups()).thenReturn(Arrays.asList("groupVictim"));
+
+ // Exception should be thrown before applying config
+ LambdaTestUtils.intercept(
+ AuthorizationException.class,
+ "User: impersonator is not allowed to impersonate victim",
+ () -> ProxyUsers.authorize(ugi, LOOPBACK_ADDRESS));
+
+ // refresh will look at configuration on the server side
+ // add additional resource with the new value
+ // so the server side will pick it up
+ String tfile = "testRouterRefreshSuperUserGroupsConfiguration_rsrc.xml";
+ ArrayList<String> keys = new ArrayList<>(
+ Arrays.asList(
+ "hadoop.proxyuser.impersonator.groups",
+ "hadoop.proxyuser.impersonator.hosts"));
+ ArrayList<String> values = new ArrayList<>(
+ Arrays.asList(
+ "groupVictim",
+ LOOPBACK_ADDRESS));
+ tempResource = addFileBasedConfigResource(tfile, keys, values);
+ Configuration.addDefaultResource(tfile);
+
+ // Mimic CLI Access
+ RouterAdmin routerAdmin = new RouterAdmin(conf);
+ int clientRes =
+ routerAdmin.run(new String[]{"-refreshSuperUserGroupsConfiguration"});
+
+ assertEquals("CLI command was not successful", 0, clientRes);
+ ProxyUsers.authorize(ugi, LOOPBACK_ADDRESS);
+ }
+
+ private static String addFileBasedConfigResource(String configFileName,
+ ArrayList<String> keyArray,
+ ArrayList<String> valueArray)
+ throws IOException {
+
+ if(keyArray.size() != valueArray.size()) {
+ throw new IOException("keyArray and valueArray should be equal in size");
+ }
+
+ URL url = new Configuration().getResource("hdfs-site.xml");
+ Path dir = new Path(URLDecoder.decode(url.getPath(), "UTF-8")).getParent();
+ String tmp = dir.toString() + "/" + configFileName;
+
+ StringBuilder configItems = new StringBuilder();
+ configItems.append("<configuration>");
+ for (int i = 0; i < keyArray.size(); i++){
+ configItems
+ .append("<property>")
+ .append("<name>").append(keyArray.get(i)).append("</name>")
+ .append("<value>").append(valueArray.get(i)).append("</value>")
+ .append("</property>");
+ }
+ configItems.append("</configuration>");
+
+ PrintWriter writer = new PrintWriter(new FileOutputStream(tmp));
+ writer.println(configItems.toString());
+ writer.close();
+ return tmp;
+ }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index 92e7c4d..7014331 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -444,6 +444,7 @@ Usage:
[-getDisabledNameservices]
[-refresh]
[-refreshRouterArgs <host:ipc_port> <key> [arg1..argn]]
+ [-refreshSuperUserGroupsConfiguration]
| COMMAND\_OPTION | Description |
|:---- |:---- |
@@ -459,6 +460,7 @@ Usage:
| `-getDisabledNameservices` | Get the name services that are disabled in the federation. |
| `-refresh` | Update mount table cache of the connected router. |
| `refreshRouterArgs` \<host:ipc\_port\> \<key\> [arg1..argn] | To trigger a runtime-refresh of the resource specified by \<key\> on \<host:ipc\_port\>. For example, to enable white list checking, we just need to send a refresh command other than restart the router server. |
+| `-refreshSuperUserGroupsConfiguration` | Refresh superuser proxy groups mappings on Router. |
The commands for managing Router-based federation. See [Mount table management](../hadoop-hdfs-rbf/HDFSRouterFederation.html#Mount_table_management) for more info.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org