You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2023/02/24 21:43:26 UTC
[hadoop] branch branch-3.3 updated (35e04ff52a4 -> 5fe19a0f016)
This is an automated email from the ASF dual-hosted git repository.
omalley pushed a change to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
from 35e04ff52a4 HADOOP-18470. Remove HDFS RBF text in the 3.3.5 index.md file
new 9a89deca1d1 HDFS-16756. RBF proxies the client's user by the login user to enable CacheEntry (#4853). Contributed by ZanderXu.
new 5fe19a0f016 HDFS-16901: RBF: Propagates real user's username via the caller context, when a proxy user is being used. (#5346)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/hadoop/ipc/CallerContext.java | 1 +
.../server/federation/router/RouterRpcClient.java | 19 +++++++---
.../server/federation/router/TestRouterRpc.java | 41 ++++++++++++++++++++++
3 files changed, 57 insertions(+), 4 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/02: HDFS-16901: RBF: Propagates real user's username via the caller context, when a proxy user is being used. (#5346)
Posted by om...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
omalley pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 5fe19a0f01644dee22a847ac9497e097cadc7866
Author: Simbarashe Dzinamarira <sd...@linkedin.com>
AuthorDate: Wed Feb 22 13:58:44 2023 -0800
HDFS-16901: RBF: Propagates real user's username via the caller context, when a proxy user is being used. (#5346)
---
.../java/org/apache/hadoop/ipc/CallerContext.java | 1 +
.../server/federation/router/RouterRpcClient.java | 11 ++++--
.../server/federation/router/TestRouterRpc.java | 41 ++++++++++++++++++++++
3 files changed, 50 insertions(+), 3 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
index dbd9184a2b9..ba627adc2c4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
@@ -47,6 +47,7 @@ public final class CallerContext {
// field names
public static final String CLIENT_IP_STR = "clientIp";
public static final String CLIENT_PORT_STR = "clientPort";
+ public static final String REAL_USER_STR = "realUser";
/** The caller context.
*
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
index 6c55edde112..c4d408dd007 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
@@ -423,7 +423,7 @@ public class RouterRpcClient {
+ router.getRouterId());
}
- addClientIpToCallerContext();
+ addClientInfoToCallerContext(ugi);
Object ret = null;
if (rpcMonitor != null) {
@@ -541,19 +541,24 @@ public class RouterRpcClient {
/**
* For tracking which is the actual client address.
- * It adds trace info "clientIp:ip" and "clientPort:port"
+ * It adds trace info "clientIp:ip", "clientPort:port" and "realUser:userName"
* in the caller context, removing the old values if they were
* already present.
*/
- private void addClientIpToCallerContext() {
+ private void addClientInfoToCallerContext(UserGroupInformation ugi) {
CallerContext ctx = CallerContext.getCurrent();
String origContext = ctx == null ? null : ctx.getContext();
byte[] origSignature = ctx == null ? null : ctx.getSignature();
+ String realUser = null;
+ if (ugi.getRealUser() != null) {
+ realUser = ugi.getRealUser().getUserName();
+ }
CallerContext.Builder builder =
new CallerContext.Builder("", contextFieldSeparator)
.append(CallerContext.CLIENT_IP_STR, Server.getRemoteAddress())
.append(CallerContext.CLIENT_PORT_STR,
Integer.toString(Server.getRemotePort()))
+ .append(CallerContext.REAL_USER_STR, realUser)
.setSignature(origSignature);
// Append the original caller context
if (origContext != null) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
index ae0908894de..48420ed416c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
@@ -39,6 +39,7 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import java.lang.reflect.Method;
import java.net.URISyntaxException;
+import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
@@ -210,6 +211,14 @@ public class TestRouterRpc {
cluster.setIndependentDNs();
Configuration conf = new Configuration();
+ // Setup proxy users.
+ conf.set("hadoop.proxyuser.testRealUser.groups", "*");
+ conf.set("hadoop.proxyuser.testRealUser.hosts", "*");
+ String loginUser = UserGroupInformation.getLoginUser().getUserName();
+ conf.set(String.format("hadoop.proxyuser.%s.groups", loginUser), "*");
+ conf.set(String.format("hadoop.proxyuser.%s.hosts", loginUser), "*");
+ // Enable IP proxy users.
+ conf.set(DFSConfigKeys.DFS_NAMENODE_IP_PROXY_USERS, "placeholder");
conf.setInt(DFSConfigKeys.DFS_LIST_LIMIT, 5);
cluster.addNamenodeOverrides(conf);
// Start NNs and DNs and wait until ready
@@ -1871,6 +1880,38 @@ public class TestRouterRpc {
assertTrue(verifyFileExists(routerFS, dirPath));
}
+ @Test
+ public void testRealUserPropagationInCallerContext()
+ throws IOException, InterruptedException {
+ GenericTestUtils.LogCapturer auditlog =
+ GenericTestUtils.LogCapturer.captureLogs(FSNamesystem.auditLog);
+
+ // Current callerContext is null
+ assertNull(CallerContext.getCurrent());
+
+ UserGroupInformation loginUser = UserGroupInformation.getLoginUser();
+ UserGroupInformation realUser = UserGroupInformation
+ .createUserForTesting("testRealUser", new String[]{"group"});
+ UserGroupInformation proxyUser = UserGroupInformation
+ .createProxyUser("testProxyUser", realUser);
+ FileSystem proxyFs = proxyUser.doAs(
+ (PrivilegedExceptionAction<FileSystem>) () -> router.getFileSystem());
+ proxyFs.listStatus(new Path("/"));
+
+
+ final String logOutput = auditlog.getOutput();
+ // Login user, which is used as the router's user, is different from the realUser.
+ assertNotEquals(loginUser.getUserName(), realUser.getUserName());
+ // Login user is used in the audit log's ugi field.
+ assertTrue("The login user is the proxyUser in the UGI field",
+ logOutput.contains(String.format("ugi=%s (auth:PROXY) via %s (auth:SIMPLE)",
+ proxyUser.getUserName(),
+ loginUser.getUserName())));
+ // Real user is added to the caller context.
+ assertTrue("The audit log should contain the real user.",
+ logOutput.contains(String.format("realUser:%s", realUser.getUserName())));
+ }
+
@Test
public void testSetBalancerBandwidth() throws Exception {
long defaultBandwidth =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/02: HDFS-16756. RBF proxies the client's user by the login user to enable CacheEntry (#4853). Contributed by ZanderXu.
Posted by om...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
omalley pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 9a89deca1d12cc572d81ce37f8cb9aa329f31c00
Author: ZanderXu <15...@163.com>
AuthorDate: Fri Sep 9 15:17:36 2022 +0800
HDFS-16756. RBF proxies the client's user by the login user to enable CacheEntry (#4853). Contributed by ZanderXu.
Reviewed-by: Inigo Goiri <in...@apache.org>
Signed-off-by: Ayush Saxena <ay...@apache.org>
---
.../hadoop/hdfs/server/federation/router/RouterRpcClient.java | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
index ef84f301a90..6c55edde112 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
@@ -22,6 +22,8 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_C
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_IP_PROXY_USERS;
+import static org.apache.hadoop.hdfs.server.federation.fairness.RouterRpcFairnessConstants.CONCURRENT_NS;
import java.io.EOFException;
import java.io.FileNotFoundException;
@@ -130,6 +132,8 @@ public class RouterRpcClient {
private static final String CLIENT_IP_STR = "clientIp";
private static final String CLIENT_PORT_STR = "clientPort";
+ private final boolean enableProxyUser;
+
/**
* Create a router RPC client to manage remote procedure calls to NNs.
*
@@ -185,6 +189,8 @@ public class RouterRpcClient {
this.retryPolicy = RetryPolicies.failoverOnNetworkException(
RetryPolicies.TRY_ONCE_THEN_FAIL, maxFailoverAttempts, maxRetryAttempts,
failoverSleepBaseMillis, failoverSleepMaxMillis);
+ String[] ipProxyUsers = conf.getStrings(DFS_NAMENODE_IP_PROXY_USERS);
+ this.enableProxyUser = ipProxyUsers != null && ipProxyUsers.length > 0;
}
/**
@@ -316,7 +322,7 @@ public class RouterRpcClient {
// TODO Add tokens from the federated UGI
UserGroupInformation connUGI = ugi;
- if (UserGroupInformation.isSecurityEnabled()) {
+ if (UserGroupInformation.isSecurityEnabled() || this.enableProxyUser) {
UserGroupInformation routerUser = UserGroupInformation.getLoginUser();
connUGI = UserGroupInformation.createProxyUser(
ugi.getUserName(), routerUser);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org