You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2022/03/14 17:42:44 UTC
[hadoop] branch branch-3.3 updated (743db6e -> c52b97d)
This is an automated email from the ASF dual-hosted git repository.
omalley pushed a change to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
from 743db6e HADOOP-18155. Refactor tests in TestFileUtil (#4063)
new 5a38ed2 HADOOP-17276. Extend CallerContext to make it include many items (#2327)
new 8e129e5 HDFS-13293. RBF: The RouterRPCServer should transfer client IP via CallerContext to NamenodeRpcServer (#2363)
new 2479d4a HDFS-15630. RBF: Fix wrong client IP info in CallerContext when requests mount points with multi-destinations. Contributed by Chengwei Wang
new f9d40ed HDFS-16266. Add remote port information to HDFS audit log (#3538)
new 0029f22 HADOOP-18003. Add a method appendIfAbsent for CallerContext (#3644)
new 496657c HDFS-16310. RBF: Add client port to CallerContext for Router (#3635)
new 52fb9d7 HADOOP-18014. CallerContext should not include some characters. (#3698)
new c52b97d HDFS-16495: RBF should prepend the client ip rather than append it.
The 8 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hadoop/fs/CommonConfigurationKeysPublic.java | 3 +
.../java/org/apache/hadoop/ipc/CallerContext.java | 131 ++++++++++++++++++++-
.../main/java/org/apache/hadoop/ipc/Server.java | 31 ++++-
.../src/main/resources/core-default.xml | 10 ++
.../org/apache/hadoop/ipc/TestCallerContext.java | 84 +++++++++++++
.../server/federation/router/RouterRpcClient.java | 74 +++++++++++-
.../server/federation/router/TestRouterRpc.java | 77 +++++++++++-
.../router/TestRouterRpcMultiDestination.java | 42 +++++++
.../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +
.../hadoop/hdfs/server/common/JspHelper.java | 3 +
.../hadoop/hdfs/server/namenode/AuditLogger.java | 5 +-
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 49 ++++++--
.../hadoop/hdfs/server/namenode/FsckServlet.java | 4 +-
.../hdfs/server/namenode/HdfsAuditLogger.java | 4 +-
.../web/resources/NamenodeWebHdfsMethods.java | 10 ++
.../src/main/resources/hdfs-default.xml | 8 ++
.../hdfs/server/namenode/TestAuditLogger.java | 94 ++++++++++++++-
.../hadoop/hdfs/server/namenode/TestAuditLogs.java | 6 +-
.../hadoop/hdfs/server/namenode/TestFsck.java | 8 +-
.../metrics/TimelineServiceV1Publisher.java | 2 +-
.../metrics/TimelineServiceV2Publisher.java | 2 +-
.../impl/pb/ApplicationStateDataPBImpl.java | 7 +-
.../server/resourcemanager/TestRMAuditLogger.java | 4 +-
23 files changed, 618 insertions(+), 44 deletions(-)
create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallerContext.java
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 04/08: HDFS-16266. Add remote port information to HDFS audit log (#3538)
Posted by om...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
omalley pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit f9d40ed7b73232c5b08e946e1e78eae2d053910e
Author: litao <to...@gmail.com>
AuthorDate: Thu Nov 4 09:16:03 2021 +0800
HDFS-16266. Add remote port information to HDFS audit log (#3538)
Reviewed-by: Akira Ajisaka <aa...@apache.org>
Reviewed-by: Wei-Chiu Chuang <we...@apache.org>
Signed-off-by: Takanobu Asanuma <ta...@apache.org>
Cherry-picked from 359b03c8 by Owen O'Malley
---
.../main/java/org/apache/hadoop/ipc/Server.java | 31 ++++++++++--
.../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ++
.../hadoop/hdfs/server/common/JspHelper.java | 3 ++
.../hadoop/hdfs/server/namenode/AuditLogger.java | 5 +-
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 38 ++++++++++++++-
.../hadoop/hdfs/server/namenode/FsckServlet.java | 4 +-
.../hdfs/server/namenode/HdfsAuditLogger.java | 4 +-
.../web/resources/NamenodeWebHdfsMethods.java | 10 ++++
.../src/main/resources/hdfs-default.xml | 8 ++++
.../hdfs/server/namenode/TestAuditLogger.java | 55 ++++++++++++++++++++++
.../hadoop/hdfs/server/namenode/TestAuditLogs.java | 6 +--
.../hadoop/hdfs/server/namenode/TestFsck.java | 8 ++--
12 files changed, 156 insertions(+), 20 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index 6cff297..308eb28 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -369,11 +369,20 @@ public abstract class Server {
}
/** Returns the remote side ip address when invoked inside an RPC
- * Returns null incase of an error.
+ * Returns null in case of an error.
*/
public static InetAddress getRemoteIp() {
Call call = CurCall.get();
- return (call != null ) ? call.getHostInetAddress() : null;
+ return (call != null) ? call.getHostInetAddress() : null;
+ }
+
+ /**
+ * Returns the remote side port when invoked inside an RPC
+ * Returns 0 in case of an error.
+ */
+ public static int getRemotePort() {
+ Call call = CurCall.get();
+ return (call != null) ? call.getRemotePort() : 0;
}
/**
@@ -409,7 +418,7 @@ public abstract class Server {
Call call = CurCall.get();
return call != null ? call.clientId : RpcConstants.DUMMY_CLIENT_ID;
}
-
+
/** Returns remote address as a string when invoked inside an RPC.
* Returns null in case of an error.
*/
@@ -447,7 +456,7 @@ public abstract class Server {
return call != null? call.getPriorityLevel() : 0;
}
- private String bindAddress;
+ private String bindAddress;
private int port; // port we listen on
private int handlerCount; // number of handler threads
private int readThreads; // number of read threads
@@ -455,7 +464,7 @@ public abstract class Server {
private Class<? extends Writable> rpcRequestClass; // class used for deserializing the rpc request
final protected RpcMetrics rpcMetrics;
final protected RpcDetailedMetrics rpcDetailedMetrics;
-
+
private Configuration conf;
private String portRangeConfig = null;
private SecretManager<TokenIdentifier> secretManager;
@@ -871,6 +880,9 @@ public abstract class Server {
public InetAddress getHostInetAddress() {
return null;
}
+ public int getRemotePort() {
+ return 0;
+ }
public String getHostAddress() {
InetAddress addr = getHostInetAddress();
return (addr != null) ? addr.getHostAddress() : null;
@@ -1029,6 +1041,11 @@ public abstract class Server {
}
@Override
+ public int getRemotePort() {
+ return connection.getRemotePort();
+ }
+
+ @Override
public Void run() throws Exception {
if (!connection.channel.isOpen()) {
Server.LOG.info(Thread.currentThread().getName() + ": skipped " + this);
@@ -1909,6 +1926,10 @@ public abstract class Server {
return ingressPort;
}
+ public int getRemotePort() {
+ return remotePort;
+ }
+
public InetAddress getHostInetAddress() {
return addr;
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index b0e7530..f47ad6c 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -707,6 +707,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
"dfs.datanode.metrics.logger.period.seconds";
public static final int DFS_DATANODE_METRICS_LOGGER_PERIOD_SECONDS_DEFAULT =
600;
+ public static final String DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_KEY =
+ "dfs.namenode.audit.log.with.remote.port";
+ public static final boolean DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_DEFAULT =
+ false;
/**
* The maximum number of getBlocks RPCs data movement utilities can make to
* a NameNode per second. Values <= 0 disable throttling. This affects
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
index 2f24965..4265c28 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
@@ -198,6 +198,9 @@ public class JspHelper {
return remoteAddr;
}
+ public static int getRemotePort(HttpServletRequest request) {
+ return request.getRemotePort();
+ }
/**
* Expected user name should be a short name.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AuditLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AuditLogger.java
index 614eb63..49dcb8c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AuditLogger.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AuditLogger.java
@@ -17,14 +17,13 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
-import java.net.InetAddress;
-import java.security.Principal;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
+import java.net.InetAddress;
+
/**
* Interface defining an audit logger.
*/
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 243f622..e890322 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -24,12 +24,16 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_C
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_MAX_SIZE_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_MAX_SIZE_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SIGNATURE_MAX_SIZE_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SIGNATURE_MAX_SIZE_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_PERMISSIONS_SUPERUSER_ONLY_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_PERMISSIONS_SUPERUSER_ONLY_KEY;
@@ -383,6 +387,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
@Metric final MutableRatesWithAggregation detailedLockHoldTimeMetrics =
registry.newRatesWithAggregation("detailedLockHoldTimeMetrics");
+ private static final String CLIENT_PORT_STR = "clientPort";
+ private final String contextFieldSeparator;
+
boolean isAuditEnabled() {
return (!isDefaultAuditLogger || auditLog.isInfoEnabled())
&& !auditLoggers.isEmpty();
@@ -397,7 +404,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
String dst, FileStatus stat) throws IOException {
if (isAuditEnabled() && isExternalInvocation()) {
logAuditEvent(succeeded, Server.getRemoteUser(), Server.getRemoteIp(),
- cmd, src, dst, stat);
+ cmd, src, dst, stat);
}
}
@@ -428,6 +435,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
for (AuditLogger logger : auditLoggers) {
if (logger instanceof HdfsAuditLogger) {
HdfsAuditLogger hdfsLogger = (HdfsAuditLogger) logger;
+ if (auditLogWithRemotePort) {
+ appendClientPortToCallerContextIfAbsent();
+ }
hdfsLogger.logAuditEvent(succeeded, ugiStr, addr, cmd, src, dst,
status, CallerContext.getCurrent(), ugi, dtSecretManager);
} else {
@@ -436,6 +446,25 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
}
+ private void appendClientPortToCallerContextIfAbsent() {
+ final CallerContext ctx = CallerContext.getCurrent();
+ if (isClientPortInfoAbsent(CLIENT_PORT_STR + ":" + Server.getRemotePort(),
+ ctx)) {
+ String origContext = ctx == null ? null : ctx.getContext();
+ byte[] origSignature = ctx == null ? null : ctx.getSignature();
+ CallerContext.setCurrent(
+ new CallerContext.Builder(origContext, contextFieldSeparator)
+ .append(CLIENT_PORT_STR, String.valueOf(Server.getRemotePort()))
+ .setSignature(origSignature)
+ .build());
+ }
+ }
+
+ private boolean isClientPortInfoAbsent(String clientPortInfo, CallerContext ctx){
+ return ctx == null || ctx.getContext() == null
+ || !ctx.getContext().contains(clientPortInfo);
+ }
+
/**
* Logger for audit events, noting successful FSNamesystem operations. Emits
* to FSNamesystem.audit at INFO. Each event causes a set of tab-separated
@@ -486,6 +515,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
// underlying logger is disabled, and avoid some unnecessary work.
private final boolean isDefaultAuditLogger;
private final List<AuditLogger> auditLoggers;
+ private final boolean auditLogWithRemotePort;
/** The namespace tree. */
FSDirectory dir;
@@ -817,6 +847,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
LOG.info("Enabling async auditlog");
enableAsyncAuditLog(conf);
}
+ auditLogWithRemotePort =
+ conf.getBoolean(DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_KEY,
+ DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_DEFAULT);
+ this.contextFieldSeparator =
+ conf.get(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY,
+ HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT);
fsLock = new FSNamesystemLock(conf, detailedLockHoldTimeMetrics);
cond = fsLock.newWriteLockCondition();
cpLock = new ReentrantLock();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsckServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsckServlet.java
index e5c02e8..059b653 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsckServlet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsckServlet.java
@@ -48,9 +48,9 @@ public class FsckServlet extends DfsServlet {
@SuppressWarnings("unchecked")
final Map<String,String[]> pmap = request.getParameterMap();
final PrintWriter out = response.getWriter();
- final InetAddress remoteAddress =
+ final InetAddress remoteAddress =
InetAddress.getByName(request.getRemoteAddr());
- final ServletContext context = getServletContext();
+ final ServletContext context = getServletContext();
final Configuration conf = NameNodeHttpServer.getConfFromContext(context);
final UserGroupInformation ugi = getUGI(request, conf);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/HdfsAuditLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/HdfsAuditLogger.java
index 0a355d0..5c0a34c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/HdfsAuditLogger.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/HdfsAuditLogger.java
@@ -17,8 +17,6 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
-import java.net.InetAddress;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FileStatus;
@@ -26,6 +24,8 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretMan
import org.apache.hadoop.ipc.CallerContext;
import org.apache.hadoop.security.UserGroupInformation;
+import java.net.InetAddress;
+
/**
* Extension of {@link AuditLogger}.
*/
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
index c75fbe0..9ae5af2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
@@ -131,6 +131,7 @@ public class NamenodeWebHdfsMethods {
private String scheme;
private Principal userPrincipal;
private String remoteAddr;
+ private int remotePort;
private @Context ServletContext context;
private @Context HttpServletResponse response;
@@ -145,6 +146,7 @@ public class NamenodeWebHdfsMethods {
// get the remote address, if coming in via a trusted proxy server then
// the address with be that of the proxied client
remoteAddr = JspHelper.getRemoteAddr(request);
+ remotePort = JspHelper.getRemotePort(request);
supportEZ =
Boolean.valueOf(request.getHeader(WebHdfsFileSystem.EZ_HEADER));
}
@@ -223,6 +225,10 @@ public class NamenodeWebHdfsMethods {
return getRemoteAddr();
}
@Override
+ public int getRemotePort() {
+ return getRemotePortFromJSPHelper();
+ }
+ @Override
public InetAddress getHostInetAddress() {
try {
return InetAddress.getByName(getHostAddress());
@@ -253,6 +259,10 @@ public class NamenodeWebHdfsMethods {
return remoteAddr;
}
+ protected int getRemotePortFromJSPHelper() {
+ return remotePort;
+ }
+
protected void queueExternalCall(ExternalCall call)
throws IOException, InterruptedException {
final NameNode namenode = (NameNode)context.getAttribute("name.node");
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 303e302..c24e288 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -5014,6 +5014,14 @@
</property>
<property>
+ <name>dfs.namenode.audit.log.with.remote.port</name>
+ <value>false</value>
+ <description>
+ If true, adds a port of RPC call to callerContext for all audit log events.
+ </description>
+</property>
+
+<property>
<name>dfs.namenode.available-space-block-placement-policy.balanced-space-preference-fraction</name>
<value>0.6</value>
<description>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java
index d02232f..681ea78 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java
@@ -55,6 +55,7 @@ import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
+import java.util.regex.Pattern;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_MAX_SIZE_KEY;
@@ -69,6 +70,7 @@ import static org.apache.hadoop.fs.permission.FsAction.EXECUTE;
import static org.apache.hadoop.fs.permission.FsAction.READ_EXECUTE;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AUDIT_LOGGERS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.NNTOP_ENABLED_KEY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -89,6 +91,20 @@ public class TestAuditLogger {
}
private static final short TEST_PERMISSION = (short) 0654;
+ private static final Pattern AUDIT_PATTERN = Pattern.compile(
+ ".*allowed=.*?\\s" +
+ "ugi=.*?\\s" +
+ "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" +
+ "cmd=.*?\\ssrc=.*?\\sdst=null\\s" +
+ "perm=.*?");
+ private static final Pattern AUDIT_WITH_PORT_PATTERN = Pattern.compile(
+ ".*allowed=.*?\\s" +
+ "ugi=.*?\\s" +
+ "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" +
+ "cmd=.*?\\ssrc=.*?\\sdst=null\\s" +
+ "perm=.*?" +
+ "proto=.*?" +
+ "callerContext=.*?clientPort\\:(\\d{0,9}).*?");
@Before
public void setup() {
@@ -544,6 +560,45 @@ public class TestAuditLogger {
}
}
+ /**
+ * Test adding remote port to audit log.
+ */
+ @Test
+ public void testAuditLogWithRemotePort() throws Exception {
+ // Audit log without remote port by default.
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster1 = new MiniDFSCluster.Builder(conf).build();
+ try {
+ LogCapturer auditLog = LogCapturer.captureLogs(FSNamesystem.auditLog);
+ cluster1.waitClusterUp();
+ FileSystem fs = cluster1.getFileSystem();
+ long time = System.currentTimeMillis();
+ fs.setTimes(new Path("/"), time, time);
+ assertTrue(AUDIT_PATTERN.matcher(auditLog.getOutput().trim()).matches());
+ assertFalse(auditLog.getOutput().contains("clientPort"));
+ auditLog.clearOutput();
+ } finally {
+ cluster1.shutdown();
+ }
+
+ // Audit log with remote port.
+ conf.setBoolean(DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_KEY, true);
+ conf.setBoolean(HADOOP_CALLER_CONTEXT_ENABLED_KEY, true);
+ MiniDFSCluster cluster2 = new MiniDFSCluster.Builder(conf).build();
+ try {
+ LogCapturer auditLog = LogCapturer.captureLogs(FSNamesystem.auditLog);
+ cluster2.waitClusterUp();
+ FileSystem fs = cluster2.getFileSystem();
+ long time = System.currentTimeMillis();
+ fs.setTimes(new Path("/"), time, time);
+ assertTrue(AUDIT_WITH_PORT_PATTERN.matcher(
+ auditLog.getOutput().trim()).matches());
+ auditLog.clearOutput();
+ } finally {
+ cluster2.shutdown();
+ }
+ }
+
public static class DummyAuditLogger implements AuditLogger {
static boolean initialized;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java
index 1033919..6fa9af7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java
@@ -90,9 +90,9 @@ public class TestAuditLogs {
// allowed=(true|false) ugi=name ip=/address cmd={cmd} src={path} dst=null perm=null
static final Pattern auditPattern = Pattern.compile(
"allowed=.*?\\s" +
- "ugi=.*?\\s" +
- "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" +
- "cmd=.*?\\ssrc=.*?\\sdst=null\\s" +
+ "ugi=.*?\\s" +
+ "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" +
+ "cmd=.*?\\ssrc=.*?\\sdst=null\\s" +
"perm=.*?");
static final Pattern successPattern = Pattern.compile(
".*allowed=true.*");
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
index f2630d2..d24db79 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
@@ -145,14 +145,14 @@ public class TestFsck {
// allowed=true ugi=name ip=/address cmd=FSCK src=/ dst=null perm=null
static final Pattern FSCK_PATTERN = Pattern.compile(
"allowed=.*?\\s" +
- "ugi=.*?\\s" +
- "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" +
+ "ugi=.*?\\s" +
+ "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" +
"cmd=fsck\\ssrc=\\/\\sdst=null\\s" +
"perm=null\\s" + "proto=.*");
static final Pattern GET_FILE_INFO_PATTERN = Pattern.compile(
"allowed=.*?\\s" +
- "ugi=.*?\\s" +
- "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" +
+ "ugi=.*?\\s" +
+ "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" +
"cmd=getfileinfo\\ssrc=\\/\\sdst=null\\s" +
"perm=null\\s" + "proto=.*");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 03/08: HDFS-15630. RBF: Fix wrong client IP info in CallerContext when requests mount points with multi-destinations. Contributed by Chengwei Wang
Posted by om...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
omalley pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 2479d4ab6ce0567d9476b778c5401a03b5970638
Author: Hui Fei <fe...@apache.org>
AuthorDate: Fri Oct 23 13:41:43 2020 +0800
HDFS-15630. RBF: Fix wrong client IP info in CallerContext when requests mount points with multi-destinations. Contributed by Chengwei Wang
Cherry-picked from 264c948e by Owen O'Malley
---
.../server/federation/router/RouterRpcClient.java | 60 +++++++++++++++++-----
.../router/TestRouterRpcMultiDestination.java | 43 ++++++++++++++++
2 files changed, 91 insertions(+), 12 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
index cf73638..97d3fba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
@@ -72,6 +72,7 @@ import org.apache.hadoop.ipc.CallerContext;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.ipc.RetriableException;
import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.ipc.Server.Call;
import org.apache.hadoop.ipc.StandbyException;
import org.apache.hadoop.net.ConnectTimeoutException;
import org.apache.hadoop.security.UserGroupInformation;
@@ -414,7 +415,8 @@ public class RouterRpcClient {
" with params " + Arrays.deepToString(params) + " from "
+ router.getRouterId());
}
- appendClientIpToCallerContext();
+
+ appendClientIpToCallerContextIfAbsent();
Object ret = null;
if (rpcMonitor != null) {
@@ -531,17 +533,26 @@ public class RouterRpcClient {
}
/**
- * For Tracking which is the actual client address.
- * It adds key/value (clientIp/"ip") pair to the caller context.
+ * For tracking which is the actual client address.
+ * It adds trace info "clientIp:ip" to caller context if it's absent.
*/
- private void appendClientIpToCallerContext() {
+ private void appendClientIpToCallerContextIfAbsent() {
+ String clientIpInfo = CLIENT_IP_STR + ":" + Server.getRemoteAddress();
final CallerContext ctx = CallerContext.getCurrent();
- String origContext = ctx == null ? null : ctx.getContext();
- byte[] origSignature = ctx == null ? null : ctx.getSignature();
- CallerContext.setCurrent(
- new CallerContext.Builder(origContext, contextFieldSeparator)
- .append(CLIENT_IP_STR, Server.getRemoteAddress())
- .setSignature(origSignature).build());
+ if (isClientIpInfoAbsent(clientIpInfo, ctx)) {
+ String origContext = ctx == null ? null : ctx.getContext();
+ byte[] origSignature = ctx == null ? null : ctx.getSignature();
+ CallerContext.setCurrent(
+ new CallerContext.Builder(origContext, contextFieldSeparator)
+ .append(clientIpInfo)
+ .setSignature(origSignature)
+ .build());
+ }
+ }
+
+ private boolean isClientIpInfoAbsent(String clientIpInfo, CallerContext ctx){
+ return ctx == null || ctx.getContext() == null
+ || !ctx.getContext().contains(clientIpInfo);
}
/**
@@ -1257,6 +1268,9 @@ public class RouterRpcClient {
List<T> orderedLocations = new ArrayList<>();
List<Callable<Object>> callables = new ArrayList<>();
+ // transfer originCall & callerContext to worker threads of executor.
+ final Call originCall = Server.getCurCall().get();
+ final CallerContext originContext = CallerContext.getCurrent();
for (final T location : locations) {
String nsId = location.getNameserviceId();
final List<? extends FederationNamenodeContext> namenodes =
@@ -1274,12 +1288,20 @@ public class RouterRpcClient {
nnLocation = (T)new RemoteLocation(nsId, nnId, location.getDest());
}
orderedLocations.add(nnLocation);
- callables.add(() -> invokeMethod(ugi, nnList, proto, m, paramList));
+ callables.add(
+ () -> {
+ transferThreadLocalContext(originCall, originContext);
+ return invokeMethod(ugi, nnList, proto, m, paramList);
+ });
}
} else {
// Call the objectGetter in order of nameservices in the NS list
orderedLocations.add(location);
- callables.add(() -> invokeMethod(ugi, namenodes, proto, m, paramList));
+ callables.add(
+ () -> {
+ transferThreadLocalContext(originCall, originContext);
+ return invokeMethod(ugi, namenodes, proto, m, paramList);
+ });
}
}
@@ -1347,6 +1369,20 @@ public class RouterRpcClient {
}
/**
+ * Transfer origin thread local context which is necessary to current
+ * worker thread when invoking method concurrently by executor service.
+ *
+ * @param originCall origin Call required for getting remote client ip.
+ * @param originContext origin CallerContext which should be transferred
+ * to server side.
+ */
+ private void transferThreadLocalContext(
+ final Call originCall, final CallerContext originContext) {
+ Server.getCurCall().set(originCall);
+ CallerContext.setCurrent(originContext);
+ }
+
+ /**
* Get a prioritized list of NNs that share the same nameservice ID (in the
* same namespace). NNs that are reported as ACTIVE will be first in the list.
*
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
index bab1fd2..e50464c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
@@ -32,6 +32,7 @@ import static org.apache.hadoop.test.Whitebox.setInternalState;
import java.io.IOException;
import java.lang.reflect.Method;
+import java.net.InetAddress;
import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.EnumSet;
@@ -67,6 +68,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ipc.CallerContext;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.ipc.StandbyException;
import org.apache.hadoop.test.GenericTestUtils;
@@ -434,4 +436,45 @@ public class TestRouterRpcMultiDestination extends TestRouterRpc {
setInternalState(ns0, "haContext", nn0haCtx);
setInternalState(router0ClientProtocol, "allowPartialList", true);
}
+
+ @Test
+ public void testCallerContextWithMultiDestinations() throws IOException {
+ GenericTestUtils.LogCapturer auditLog =
+ GenericTestUtils.LogCapturer.captureLogs(FSNamesystem.auditLog);
+
+ // set client context
+ CallerContext.setCurrent(
+ new CallerContext.Builder("clientContext").build());
+ // assert the initial caller context as expected
+ assertEquals("clientContext", CallerContext.getCurrent().getContext());
+
+ DistributedFileSystem routerFs =
+ (DistributedFileSystem) getRouterFileSystem();
+ // create a directory via the router
+ Path dirPath = new Path("/test_caller_context_with_multi_destinations");
+ routerFs.mkdirs(dirPath);
+ // invoke concurrently in RouterRpcClient
+ routerFs.listStatus(dirPath);
+ // invoke sequentially in RouterRpcClient
+ routerFs.getFileStatus(dirPath);
+
+ String auditFlag = "src=" + dirPath.toString();
+ String clientIpInfo = "clientIp:"
+ + InetAddress.getLocalHost().getHostAddress();
+ for (String line : auditLog.getOutput().split("\n")) {
+ if (line.contains(auditFlag)) {
+ // assert origin caller context exist in audit log
+ assertTrue(line.contains("callerContext=clientContext"));
+ String callerContext = line.substring(
+ line.indexOf("callerContext=clientContext"));
+ // assert client ip info exist in caller context
+ assertTrue(callerContext.contains(clientIpInfo));
+ // assert client ip info appears only once in caller context
+ assertEquals(callerContext.indexOf(clientIpInfo),
+ callerContext.lastIndexOf(clientIpInfo));
+ }
+ }
+ // clear client context
+ CallerContext.setCurrent(null);
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 05/08: HADOOP-18003. Add a method appendIfAbsent for CallerContext (#3644)
Posted by om...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
omalley pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 0029f22d7dfc77d64aad4202e23a1fc90517fd2a
Author: litao <to...@gmail.com>
AuthorDate: Mon Nov 15 21:44:49 2021 +0800
HADOOP-18003. Add a method appendIfAbsent for CallerContext (#3644)
Cherry-picked from 573b358f by Owen O'Malley
---
.../java/org/apache/hadoop/ipc/CallerContext.java | 20 ++++++++++++++
.../org/apache/hadoop/ipc/TestCallerContext.java | 31 ++++++++++++++++++++++
2 files changed, 51 insertions(+)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
index 378b83d..9f9c974 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
@@ -228,6 +228,26 @@ public final class CallerContext {
return this;
}
+ /**
+ * Append new field which contains key and value to the context
+ * if the key("key:") is absent.
+ * @param key the key of field.
+ * @param value the value of field.
+ * @return the builder.
+ */
+ public Builder appendIfAbsent(String key, String value) {
+ if (sb.toString().contains(key + KEY_VALUE_SEPARATOR)) {
+ return this;
+ }
+ if (isValid(key) && isValid(value)) {
+ if (sb.length() > 0) {
+ sb.append(fieldSeparator);
+ }
+ sb.append(key).append(KEY_VALUE_SEPARATOR).append(value);
+ }
+ return this;
+ }
+
public CallerContext build() {
return new CallerContext(this);
}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallerContext.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallerContext.java
index fc1057b..bb4a119 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallerContext.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallerContext.java
@@ -42,6 +42,37 @@ public class TestCallerContext {
builder.build().getContext());
}
+ @Test
+ public void testBuilderAppendIfAbsent() {
+ Configuration conf = new Configuration();
+ conf.set(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY, "$");
+ CallerContext.Builder builder = new CallerContext.Builder(null, conf);
+ builder.append("key1", "value1");
+ Assert.assertEquals("key1:value1",
+ builder.build().getContext());
+
+ // Append an existed key with different value.
+ builder.appendIfAbsent("key1", "value2");
+ String[] items = builder.build().getContext().split("\\$");
+ Assert.assertEquals(1, items.length);
+ Assert.assertEquals("key1:value1",
+ builder.build().getContext());
+
+ // Append an absent key.
+ builder.appendIfAbsent("key2", "value2");
+ String[] items2 = builder.build().getContext().split("\\$");
+ Assert.assertEquals(2, items2.length);
+ Assert.assertEquals("key1:value1$key2:value2",
+ builder.build().getContext());
+
+ // Append a key that is a substring of an existing key.
+ builder.appendIfAbsent("key", "value");
+ String[] items3 = builder.build().getContext().split("\\$");
+ Assert.assertEquals(3, items3.length);
+ Assert.assertEquals("key1:value1$key2:value2$key:value",
+ builder.build().getContext());
+ }
+
@Test(expected = IllegalArgumentException.class)
public void testNewBuilder() {
Configuration conf = new Configuration();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 08/08: HDFS-16495: RBF should prepend the client ip rather than append it.
Posted by om...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
omalley pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit c52b97d08491a869ecefd39bfd08c8f57eaeab84
Author: Owen O'Malley <oo...@linkedin.com>
AuthorDate: Fri Mar 4 16:17:46 2022 -0800
HDFS-16495: RBF should prepend the client ip rather than append it.
Fixes #4054
Signed-off-by: Owen O'Malley <oo...@linkedin.com>
---
.../java/org/apache/hadoop/ipc/CallerContext.java | 2 +-
.../server/federation/router/RouterRpcClient.java | 31 +++++++++++++++-------
.../server/federation/router/TestRouterRpc.java | 13 ++++-----
.../router/TestRouterRpcMultiDestination.java | 5 ++--
4 files changed, 32 insertions(+), 19 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
index ed8dcf2..c8b4135 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
@@ -116,7 +116,7 @@ public final class CallerContext {
/** The caller context builder. */
public static final class Builder {
- private static final String KEY_VALUE_SEPARATOR = ":";
+ public static final String KEY_VALUE_SEPARATOR = ":";
/**
* The illegal separators include '\t', '\n', '='.
* User should not set illegal separator.
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
index 351926b..16cd7a5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
@@ -417,7 +417,7 @@ public class RouterRpcClient {
+ router.getRouterId());
}
- appendClientIpPortToCallerContextIfAbsent();
+ addClientIpToCallerContext();
Object ret = null;
if (rpcMonitor != null) {
@@ -536,19 +536,32 @@ public class RouterRpcClient {
/**
* For tracking which is the actual client address.
* It adds trace info "clientIp:ip" and "clientPort:port"
- * to caller context if they are absent.
+ * in the caller context, removing the old values if they were
+ * already present.
*/
- private void appendClientIpPortToCallerContextIfAbsent() {
+ private void addClientIpToCallerContext() {
CallerContext ctx = CallerContext.getCurrent();
String origContext = ctx == null ? null : ctx.getContext();
byte[] origSignature = ctx == null ? null : ctx.getSignature();
- CallerContext.setCurrent(
- new CallerContext.Builder(origContext, contextFieldSeparator)
- .appendIfAbsent(CLIENT_IP_STR, Server.getRemoteAddress())
- .appendIfAbsent(CLIENT_PORT_STR,
+ CallerContext.Builder builder =
+ new CallerContext.Builder("", contextFieldSeparator)
+ .append(CLIENT_IP_STR, Server.getRemoteAddress())
+ .append(CLIENT_PORT_STR,
Integer.toString(Server.getRemotePort()))
- .setSignature(origSignature)
- .build());
+ .setSignature(origSignature);
+ // Append the original caller context
+ if (origContext != null) {
+ for (String part : origContext.split(contextFieldSeparator)) {
+ String[] keyValue =
+ part.split(CallerContext.Builder.KEY_VALUE_SEPARATOR, 2);
+ if (keyValue.length == 2) {
+ builder.appendIfAbsent(keyValue[0], keyValue[1]);
+ } else if (keyValue.length == 1) {
+ builder.append(keyValue[0]);
+ }
+ }
+ }
+ CallerContext.setCurrent(builder.build());
}
/**
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
index f2b86dc..94b3a5e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
@@ -1863,9 +1863,10 @@ public class TestRouterRpc {
FsPermission permission = new FsPermission("755");
routerProtocol.mkdirs(dirPath, permission, false);
- // The audit log should contains "callerContext=clientContext,clientIp:"
- assertTrue(auditlog.getOutput()
- .contains("callerContext=clientContext,clientIp:"));
+ // The audit log should contains "callerContext=clientIp:...,clientContext"
+ final String logOutput = auditlog.getOutput();
+ assertTrue(logOutput.contains("callerContext=clientIp:"));
+ assertTrue(logOutput.contains(",clientContext"));
assertTrue(verifyFileExists(routerFS, dirPath));
}
@@ -1910,9 +1911,9 @@ public class TestRouterRpc {
// Create a directory via the router.
routerProtocol.getFileInfo(dirPath);
- // The audit log should contains the original clientIp and clientPort
+ // The audit log should not contain the original clientIp and clientPort
// set by client.
- assertTrue(auditLog.getOutput().contains("clientIp:1.1.1.1"));
- assertTrue(auditLog.getOutput().contains("clientPort:1234"));
+ assertFalse(auditLog.getOutput().contains("clientIp:1.1.1.1"));
+ assertFalse(auditLog.getOutput().contains("clientPort:1234"));
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
index e50464c..370a125 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
@@ -464,9 +464,8 @@ public class TestRouterRpcMultiDestination extends TestRouterRpc {
for (String line : auditLog.getOutput().split("\n")) {
if (line.contains(auditFlag)) {
// assert origin caller context exist in audit log
- assertTrue(line.contains("callerContext=clientContext"));
- String callerContext = line.substring(
- line.indexOf("callerContext=clientContext"));
+ String callerContext = line.substring(line.indexOf("callerContext="));
+ assertTrue(callerContext.contains("clientContext"));
// assert client ip info exist in caller context
assertTrue(callerContext.contains(clientIpInfo));
// assert client ip info appears only once in caller context
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/08: HDFS-13293. RBF: The RouterRPCServer should transfer client IP via CallerContext to NamenodeRpcServer (#2363)
Posted by om...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
omalley pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 8e129e5b8d99d9efac4d2f8cc2e319155d062cfe
Author: Hui Fei <fe...@gmail.com>
AuthorDate: Fri Oct 9 16:12:22 2020 +0800
HDFS-13293. RBF: The RouterRPCServer should transfer client IP via CallerContext to NamenodeRpcServer (#2363)
Cherry-picked from 518a212c by Owen O'Malley
---
.../java/org/apache/hadoop/ipc/CallerContext.java | 10 +++++++-
.../server/federation/router/RouterRpcClient.java | 25 +++++++++++++++++++
.../server/federation/router/TestRouterRpc.java | 28 +++++++++++++++++++++-
3 files changed, 61 insertions(+), 2 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
index e15340c..378b83d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
@@ -129,7 +129,7 @@ public final class CallerContext {
private byte[] signature;
public Builder(String context) {
- this(context, new Configuration());
+ this(context, HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT);
}
public Builder(String context, Configuration conf) {
@@ -141,6 +141,14 @@ public final class CallerContext {
checkFieldSeparator(fieldSeparator);
}
+ public Builder(String context, String separator) {
+ if (isValid(context)) {
+ sb.append(context);
+ }
+ fieldSeparator = separator;
+ checkFieldSeparator(fieldSeparator);
+ }
+
/**
* Check whether the separator is legal.
* The illegal separators include '\t', '\n', '='.
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
index aeb6461..cf73638 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.hdfs.server.federation.router;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY;
@@ -66,8 +68,10 @@ import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.io.retry.RetryPolicy.RetryAction.RetryDecision;
+import org.apache.hadoop.ipc.CallerContext;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.ipc.RetriableException;
+import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.ipc.StandbyException;
import org.apache.hadoop.net.ConnectTimeoutException;
import org.apache.hadoop.security.UserGroupInformation;
@@ -115,11 +119,14 @@ public class RouterRpcClient {
private final RetryPolicy retryPolicy;
/** Optional perf monitor. */
private final RouterRpcMonitor rpcMonitor;
+ /** Field separator of CallerContext. */
+ private final String contextFieldSeparator;
/** Pattern to parse a stack trace line. */
private static final Pattern STACK_TRACE_PATTERN =
Pattern.compile("\\tat (.*)\\.(.*)\\((.*):(\\d*)\\)");
+ private static final String CLIENT_IP_STR = "clientIp";
/**
* Create a router RPC client to manage remote procedure calls to NNs.
@@ -136,6 +143,9 @@ public class RouterRpcClient {
this.namenodeResolver = resolver;
Configuration clientConf = getClientConfiguration(conf);
+ this.contextFieldSeparator =
+ clientConf.get(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY,
+ HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT);
this.connectionManager = new ConnectionManager(clientConf);
this.connectionManager.start();
@@ -404,6 +414,7 @@ public class RouterRpcClient {
" with params " + Arrays.deepToString(params) + " from "
+ router.getRouterId());
}
+ appendClientIpToCallerContext();
Object ret = null;
if (rpcMonitor != null) {
@@ -520,6 +531,20 @@ public class RouterRpcClient {
}
/**
+ * For Tracking which is the actual client address.
+ * It adds key/value (clientIp/"ip") pair to the caller context.
+ */
+ private void appendClientIpToCallerContext() {
+ final CallerContext ctx = CallerContext.getCurrent();
+ String origContext = ctx == null ? null : ctx.getContext();
+ byte[] origSignature = ctx == null ? null : ctx.getSignature();
+ CallerContext.setCurrent(
+ new CallerContext.Builder(origContext, contextFieldSeparator)
+ .append(CLIENT_IP_STR, Server.getRemoteAddress())
+ .setSignature(origSignature).build());
+ }
+
+ /**
* Invokes a method on the designated object. Catches exceptions specific to
* the invocation.
*
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
index 6c755ab..ceb291d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
@@ -118,6 +118,7 @@ import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
+import org.apache.hadoop.ipc.CallerContext;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.Service.STATE;
import org.apache.hadoop.test.GenericTestUtils;
@@ -196,6 +197,8 @@ public class TestRouterRpc {
@BeforeClass
public static void globalSetUp() throws Exception {
Configuration namenodeConf = new Configuration();
+ namenodeConf.setBoolean(DFSConfigKeys.HADOOP_CALLER_CONTEXT_ENABLED_KEY,
+ true);
// It's very easy to become overloaded for some specific dn in this small
// cluster, which will cause the EC file block allocation failure. To avoid
// this issue, we disable considerLoad option.
@@ -1841,4 +1844,27 @@ public class TestRouterRpc {
}
return null;
}
-}
\ No newline at end of file
+
+ @Test
+ public void testMkdirsWithCallerContext() throws IOException {
+ GenericTestUtils.LogCapturer auditlog =
+ GenericTestUtils.LogCapturer.captureLogs(FSNamesystem.auditLog);
+
+ // Current callerContext is null
+ assertNull(CallerContext.getCurrent());
+
+ // Set client context
+ CallerContext.setCurrent(
+ new CallerContext.Builder("clientContext").build());
+
+ // Create a directory via the router
+ String dirPath = "/test_dir_with_callercontext";
+ FsPermission permission = new FsPermission("755");
+ routerProtocol.mkdirs(dirPath, permission, false);
+
+ // The audit log should contains "callerContext=clientContext,clientIp:"
+ assertTrue(auditlog.getOutput()
+ .contains("callerContext=clientContext,clientIp:"));
+ assertTrue(verifyFileExists(routerFS, dirPath));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 07/08: HADOOP-18014. CallerContext should not include some characters. (#3698)
Posted by om...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
omalley pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 52fb9d7ce227ae2c8296bb8a8f724960fb915bfc
Author: Takanobu Asanuma <ta...@apache.org>
AuthorDate: Thu Nov 25 14:05:04 2021 +0900
HADOOP-18014. CallerContext should not include some characters. (#3698)
Reviewed-by: Viraj Jasani <vj...@apache.org>
Reviewed-by: Mingliang Liu <li...@apache.org>
Reviewed-by: Hui Fei <fe...@apache.org>
Cherry-picked from 9c887e5b by Owen O'Malley
---
.../java/org/apache/hadoop/ipc/CallerContext.java | 2 --
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 12 +++----
.../hdfs/server/namenode/TestAuditLogger.java | 39 +++++++++++++++++++---
3 files changed, 40 insertions(+), 13 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
index 9f9c974..ed8dcf2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
@@ -164,8 +164,6 @@ public final class CallerContext {
/**
* Whether the field is valid.
- * The field should not contain '\t', '\n', '='.
- * Because the context could be written to audit log.
* @param field one of the fields in context.
* @return true if the field is not null or empty.
*/
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 964879b..318442d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -8512,18 +8512,18 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
callerContext != null &&
callerContext.isContextValid()) {
sb.append("\t").append("callerContext=");
- if (callerContext.getContext().length() > callerContextMaxLen) {
- sb.append(callerContext.getContext().substring(0,
- callerContextMaxLen));
+ String context = escapeJava(callerContext.getContext());
+ if (context.length() > callerContextMaxLen) {
+ sb.append(context, 0, callerContextMaxLen);
} else {
- sb.append(callerContext.getContext());
+ sb.append(context);
}
if (callerContext.getSignature() != null &&
callerContext.getSignature().length > 0 &&
callerContext.getSignature().length <= callerSignatureMaxLen) {
sb.append(":")
- .append(new String(callerContext.getSignature(),
- CallerContext.SIGNATURE_ENCODING));
+ .append(escapeJava(new String(callerContext.getSignature(),
+ CallerContext.SIGNATURE_ENCODING)));
}
}
logAuditMessage(sb.toString());
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java
index 681ea78..45bcf3e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java
@@ -256,10 +256,9 @@ public class TestAuditLogger {
conf.setBoolean(HADOOP_CALLER_CONTEXT_ENABLED_KEY, true);
conf.setInt(HADOOP_CALLER_CONTEXT_MAX_SIZE_KEY, 128);
conf.setInt(HADOOP_CALLER_CONTEXT_SIGNATURE_MAX_SIZE_KEY, 40);
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
- LogCapturer auditlog = LogCapturer.captureLogs(FSNamesystem.auditLog);
- try {
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build()) {
+ LogCapturer auditlog = LogCapturer.captureLogs(FSNamesystem.auditLog);
cluster.waitClusterUp();
final FileSystem fs = cluster.getFileSystem();
final long time = System.currentTimeMillis();
@@ -414,8 +413,8 @@ public class TestAuditLogger {
assertFalse(auditlog.getOutput().contains("callerContext="));
auditlog.clearOutput();
- } finally {
- cluster.shutdown();
+ // clear client context
+ CallerContext.setCurrent(null);
}
}
@@ -599,6 +598,36 @@ public class TestAuditLogger {
}
}
+ @Test
+ public void testCallerContextCharacterEscape() throws IOException {
+ Configuration conf = new HdfsConfiguration();
+ conf.setBoolean(HADOOP_CALLER_CONTEXT_ENABLED_KEY, true);
+ conf.setInt(HADOOP_CALLER_CONTEXT_MAX_SIZE_KEY, 128);
+ conf.setInt(HADOOP_CALLER_CONTEXT_SIGNATURE_MAX_SIZE_KEY, 40);
+
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build()) {
+ LogCapturer auditlog = LogCapturer.captureLogs(FSNamesystem.auditLog);
+ cluster.waitClusterUp();
+ final FileSystem fs = cluster.getFileSystem();
+ final long time = System.currentTimeMillis();
+ final Path p = new Path("/");
+
+ assertNull(CallerContext.getCurrent());
+
+ CallerContext context = new CallerContext.Builder("c1\nc2").append("c3\tc4")
+ .setSignature("s1\ns2".getBytes(CallerContext.SIGNATURE_ENCODING)).build();
+ CallerContext.setCurrent(context);
+ LOG.info("Set current caller context as {}", CallerContext.getCurrent());
+ fs.setTimes(p, time, time);
+ assertTrue(auditlog.getOutput().endsWith(
+ String.format("callerContext=c1\\nc2,c3\\tc4:s1\\ns2%n")));
+ auditlog.clearOutput();
+
+ // clear client context
+ CallerContext.setCurrent(null);
+ }
+ }
+
public static class DummyAuditLogger implements AuditLogger {
static boolean initialized;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/08: HADOOP-17276. Extend CallerContext to make it include many items (#2327)
Posted by om...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
omalley pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 5a38ed2f2274ccfad100cc89c729d470063e270d
Author: Fei Hui <fe...@gmail.com>
AuthorDate: Sun Oct 4 01:02:28 2020 +0800
HADOOP-17276. Extend CallerContext to make it include many items (#2327)
Cherry-picked from d0d10f7e by Owen O'Malley
---
.../hadoop/fs/CommonConfigurationKeysPublic.java | 3 +
.../java/org/apache/hadoop/ipc/CallerContext.java | 105 ++++++++++++++++++++-
.../src/main/resources/core-default.xml | 10 ++
.../org/apache/hadoop/ipc/TestCallerContext.java | 53 +++++++++++
.../metrics/TimelineServiceV1Publisher.java | 2 +-
.../metrics/TimelineServiceV2Publisher.java | 2 +-
.../impl/pb/ApplicationStateDataPBImpl.java | 7 +-
.../server/resourcemanager/TestRMAuditLogger.java | 4 +-
8 files changed, 176 insertions(+), 10 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index 1fa84c7..d769b28 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -371,6 +371,9 @@ public class CommonConfigurationKeysPublic {
"hadoop.caller.context.signature.max.size";
public static final int HADOOP_CALLER_CONTEXT_SIGNATURE_MAX_SIZE_DEFAULT =
40;
+ public static final String HADOOP_CALLER_CONTEXT_SEPARATOR_KEY =
+ "hadoop.caller.context.separator";
+ public static final String HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT = ",";
/**
* @see
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
index dd14ba1..e15340c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
@@ -21,10 +21,17 @@ import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_KEY;
/**
* A class defining the caller context for auditing coarse granularity
@@ -54,8 +61,8 @@ public final class CallerContext {
private final byte[] signature;
private CallerContext(Builder builder) {
- this.context = builder.context;
- this.signature = builder.signature;
+ this.context = builder.getContext();
+ this.signature = builder.getSignature();
}
public String getContext() {
@@ -109,11 +116,53 @@ public final class CallerContext {
/** The caller context builder. */
public static final class Builder {
- private final String context;
+ private static final String KEY_VALUE_SEPARATOR = ":";
+ /**
+ * The illegal separators include '\t', '\n', '='.
+ * User should not set illegal separator.
+ */
+ private static final Set<String> ILLEGAL_SEPARATORS =
+ Collections.unmodifiableSet(
+ new HashSet<>(Arrays.asList("\t", "\n", "=")));
+ private final String fieldSeparator;
+ private final StringBuilder sb = new StringBuilder();
private byte[] signature;
public Builder(String context) {
- this.context = context;
+ this(context, new Configuration());
+ }
+
+ public Builder(String context, Configuration conf) {
+ if (isValid(context)) {
+ sb.append(context);
+ }
+ fieldSeparator = conf.get(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY,
+ HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT);
+ checkFieldSeparator(fieldSeparator);
+ }
+
+ /**
+ * Check whether the separator is legal.
+ * The illegal separators include '\t', '\n', '='.
+ * Throw IllegalArgumentException if the separator is Illegal.
+ * @param separator the separator of fields.
+ */
+ private void checkFieldSeparator(String separator) {
+ if (ILLEGAL_SEPARATORS.contains(separator)) {
+ throw new IllegalArgumentException("Illegal field separator: "
+ + separator);
+ }
+ }
+
+ /**
+ * Whether the field is valid.
+ * The field should not contain '\t', '\n', '='.
+ * Because the context could be written to audit log.
+ * @param field one of the fields in context.
+ * @return true if the field is not null or empty.
+ */
+ private boolean isValid(String field) {
+ return field != null && field.length() > 0;
}
public Builder setSignature(byte[] signature) {
@@ -123,6 +172,54 @@ public final class CallerContext {
return this;
}
+ /**
+ * Get the context.
+ * For example, the context is "key1:value1,key2:value2".
+ * @return the valid context or null.
+ */
+ public String getContext() {
+ return sb.length() > 0 ? sb.toString() : null;
+ }
+
+ /**
+ * Get the signature.
+ * @return the signature.
+ */
+ public byte[] getSignature() {
+ return signature;
+ }
+
+ /**
+ * Append new field to the context.
+ * @param field one of fields to append.
+ * @return the builder.
+ */
+ public Builder append(String field) {
+ if (isValid(field)) {
+ if (sb.length() > 0) {
+ sb.append(fieldSeparator);
+ }
+ sb.append(field);
+ }
+ return this;
+ }
+
+ /**
+ * Append new field which contains key and value to the context.
+ * @param key the key of field.
+ * @param value the value of field.
+ * @return the builder.
+ */
+ public Builder append(String key, String value) {
+ if (isValid(key) && isValid(value)) {
+ if (sb.length() > 0) {
+ sb.append(fieldSeparator);
+ }
+ sb.append(key).append(KEY_VALUE_SEPARATOR).append(value);
+ }
+ return this;
+ }
+
public CallerContext build() {
return new CallerContext(this);
}
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index f5978a3..8a59f3c 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -3716,6 +3716,16 @@ The switch to turn S3A auditing on or off.
in audit logs.
</description>
</property>
+ <property>
+ <name>hadoop.caller.context.separator</name>
+ <value>,</value>
+ <description>
+ The separator is for context which maybe contain many fields. For example,
+ if the separator is ',', and there are two key/value fields in context,
+ in which case the context string is "key1:value1,key2:value2". The
+ separator should not contain '\t', '\n', '='.
+ </description>
+ </property>
<!-- SequenceFile's Sorter properties -->
<property>
<name>seq.io.sort.mb</name>
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallerContext.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallerContext.java
new file mode 100644
index 0000000..fc1057b
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallerContext.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ipc;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_KEY;
+
+public class TestCallerContext {
+ @Test
+ public void testBuilderAppend() {
+ Configuration conf = new Configuration();
+ conf.set(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY, "$");
+ CallerContext.Builder builder = new CallerContext.Builder(null, conf);
+ CallerContext context = builder.append("context1")
+ .append("context2").append("key3", "value3").build();
+ Assert.assertEquals(true,
+ context.getContext().contains("$"));
+ String[] items = context.getContext().split("\\$");
+ Assert.assertEquals(3, items.length);
+ Assert.assertEquals("key3:value3", items[2]);
+
+ builder.append("$$");
+ Assert.assertEquals("context1$context2$key3:value3$$$",
+ builder.build().getContext());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testNewBuilder() {
+ Configuration conf = new Configuration();
+ // Set illegal separator.
+ conf.set(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY, "\t");
+ CallerContext.Builder builder = new CallerContext.Builder(null, conf);
+ builder.build();
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
index 1d89f1e..f1b80a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
@@ -176,7 +176,7 @@ public class TimelineServiceV1Publisher extends AbstractSystemMetricsPublisher {
entityInfo.put(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION,
app.getAppNodeLabelExpression());
if (app.getCallerContext() != null) {
- if (app.getCallerContext().getContext() != null) {
+ if (app.getCallerContext().isContextValid()) {
entityInfo.put(ApplicationMetricsConstants.YARN_APP_CALLER_CONTEXT,
app.getCallerContext().getContext());
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
index d359173..8047acb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
@@ -127,7 +127,7 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher {
ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION,
app.getAppNodeLabelExpression());
if (app.getCallerContext() != null) {
- if (app.getCallerContext().getContext() != null) {
+ if (app.getCallerContext().isContextValid()) {
entityInfo.put(ApplicationMetricsConstants.YARN_APP_CALLER_CONTEXT,
app.getCallerContext().getContext());
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java
index 5dcb70b..c0f7bb0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java
@@ -274,14 +274,17 @@ public class ApplicationStateDataPBImpl extends ApplicationStateData {
RpcHeaderProtos.RPCCallerContextProto.Builder b = RpcHeaderProtos.RPCCallerContextProto
.newBuilder();
- if (callerContext.getContext() != null) {
+ if (callerContext.isContextValid()) {
b.setContext(callerContext.getContext());
}
if (callerContext.getSignature() != null) {
b.setSignature(ByteString.copyFrom(callerContext.getSignature()));
}
- builder.setCallerContext(b);
+ if(callerContext.isContextValid()
+ || callerContext.getSignature() != null) {
+ builder.setCallerContext(b);
+ }
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
index e8a532d..c78a778 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
@@ -177,7 +177,7 @@ public class TestRMAuditLogger {
expLog.append("\tRESOURCE=<memory:1536, vcores:1>");
}
if (callerContext != null) {
- if (callerContext.getContext() != null) {
+ if (callerContext.isContextValid()) {
expLog.append("\tCALLERCONTEXT=context");
}
if (callerContext.getSignature() != null) {
@@ -328,7 +328,7 @@ public class TestRMAuditLogger {
expLog.append("\tRESOURCE=<memory:1536, vcores:1>");
}
if (callerContext != null) {
- if (callerContext.getContext() != null) {
+ if (callerContext.isContextValid()) {
expLog.append("\tCALLERCONTEXT=context");
}
if (callerContext.getSignature() != null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 06/08: HDFS-16310. RBF: Add client port to CallerContext for Router (#3635)
Posted by om...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
omalley pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 496657c63fc504dfeef47a3da56214800575367b
Author: litao <to...@gmail.com>
AuthorDate: Thu Nov 18 12:46:35 2021 +0800
HDFS-16310. RBF: Add client port to CallerContext for Router (#3635)
Cherry-picked from 5b05068f by Owen O'Malley
---
.../server/federation/router/RouterRpcClient.java | 34 +++++++--------
.../server/federation/router/TestRouterRpc.java | 48 ++++++++++++++++++++++
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 7 ++--
3 files changed, 66 insertions(+), 23 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
index 97d3fba..351926b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
@@ -128,6 +128,7 @@ public class RouterRpcClient {
Pattern.compile("\\tat (.*)\\.(.*)\\((.*):(\\d*)\\)");
private static final String CLIENT_IP_STR = "clientIp";
+ private static final String CLIENT_PORT_STR = "clientPort";
/**
* Create a router RPC client to manage remote procedure calls to NNs.
@@ -416,7 +417,7 @@ public class RouterRpcClient {
+ router.getRouterId());
}
- appendClientIpToCallerContextIfAbsent();
+ appendClientIpPortToCallerContextIfAbsent();
Object ret = null;
if (rpcMonitor != null) {
@@ -534,25 +535,20 @@ public class RouterRpcClient {
/**
* For tracking which is the actual client address.
- * It adds trace info "clientIp:ip" to caller context if it's absent.
+ * It adds trace info "clientIp:ip" and "clientPort:port"
+ * to caller context if they are absent.
*/
- private void appendClientIpToCallerContextIfAbsent() {
- String clientIpInfo = CLIENT_IP_STR + ":" + Server.getRemoteAddress();
- final CallerContext ctx = CallerContext.getCurrent();
- if (isClientIpInfoAbsent(clientIpInfo, ctx)) {
- String origContext = ctx == null ? null : ctx.getContext();
- byte[] origSignature = ctx == null ? null : ctx.getSignature();
- CallerContext.setCurrent(
- new CallerContext.Builder(origContext, contextFieldSeparator)
- .append(clientIpInfo)
- .setSignature(origSignature)
- .build());
- }
- }
-
- private boolean isClientIpInfoAbsent(String clientIpInfo, CallerContext ctx){
- return ctx == null || ctx.getContext() == null
- || !ctx.getContext().contains(clientIpInfo);
+ private void appendClientIpPortToCallerContextIfAbsent() {
+ CallerContext ctx = CallerContext.getCurrent();
+ String origContext = ctx == null ? null : ctx.getContext();
+ byte[] origSignature = ctx == null ? null : ctx.getSignature();
+ CallerContext.setCurrent(
+ new CallerContext.Builder(origContext, contextFieldSeparator)
+ .appendIfAbsent(CLIENT_IP_STR, Server.getRemoteAddress())
+ .appendIfAbsent(CLIENT_PORT_STR,
+ Integer.toString(Server.getRemotePort()))
+ .setSignature(origSignature)
+ .build());
}
/**
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
index ceb291d..f2b86dc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
@@ -97,6 +97,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster;
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext;
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext;
@@ -1867,4 +1868,51 @@ public class TestRouterRpc {
.contains("callerContext=clientContext,clientIp:"));
assertTrue(verifyFileExists(routerFS, dirPath));
}
+
+ @Test
+ public void testSetBalancerBandwidth() throws Exception {
+ long defaultBandwidth =
+ DFSConfigKeys.DFS_DATANODE_BALANCE_BANDWIDTHPERSEC_DEFAULT;
+ long newBandwidth = defaultBandwidth * 2;
+ routerProtocol.setBalancerBandwidth(newBandwidth);
+ ArrayList<DataNode> datanodes = cluster.getCluster().getDataNodes();
+ GenericTestUtils.waitFor(() -> {
+ return datanodes.get(0).getBalancerBandwidth() == newBandwidth;
+ }, 100, 60 * 1000);
+ }
+
+ @Test
+ public void testAddClientIpPortToCallerContext() throws IOException {
+ GenericTestUtils.LogCapturer auditLog =
+ GenericTestUtils.LogCapturer.captureLogs(FSNamesystem.auditLog);
+
+ // 1. ClientIp and ClientPort are not set on the client.
+ // Set client context.
+ CallerContext.setCurrent(
+ new CallerContext.Builder("clientContext").build());
+
+ // Create a directory via the router.
+ String dirPath = "/test";
+ routerProtocol.mkdirs(dirPath, new FsPermission("755"), false);
+
+ // The audit log should contains "clientIp:" and "clientPort:".
+ assertTrue(auditLog.getOutput().contains("clientIp:"));
+ assertTrue(auditLog.getOutput().contains("clientPort:"));
+ assertTrue(verifyFileExists(routerFS, dirPath));
+ auditLog.clearOutput();
+
+ // 2. ClientIp and ClientPort are set on the client.
+ // Reset client context.
+ CallerContext.setCurrent(
+ new CallerContext.Builder(
+ "clientContext,clientIp:1.1.1.1,clientPort:1234").build());
+
+ // Create a directory via the router.
+ routerProtocol.getFileInfo(dirPath);
+
+ // The audit log should contains the original clientIp and clientPort
+ // set by client.
+ assertTrue(auditLog.getOutput().contains("clientIp:1.1.1.1"));
+ assertTrue(auditLog.getOutput().contains("clientPort:1234"));
+ }
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index e890322..964879b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -448,8 +448,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
private void appendClientPortToCallerContextIfAbsent() {
final CallerContext ctx = CallerContext.getCurrent();
- if (isClientPortInfoAbsent(CLIENT_PORT_STR + ":" + Server.getRemotePort(),
- ctx)) {
+ if (isClientPortInfoAbsent(ctx)) {
String origContext = ctx == null ? null : ctx.getContext();
byte[] origSignature = ctx == null ? null : ctx.getSignature();
CallerContext.setCurrent(
@@ -460,9 +459,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
}
- private boolean isClientPortInfoAbsent(String clientPortInfo, CallerContext ctx){
+ private boolean isClientPortInfoAbsent(CallerContext ctx){
return ctx == null || ctx.getContext() == null
- || !ctx.getContext().contains(clientPortInfo);
+ || !ctx.getContext().contains(CLIENT_PORT_STR);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org