You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2022/03/14 17:42:45 UTC
[hadoop] 01/08: HADOOP-17276. Extend CallerContext to make it include many items (#2327)
This is an automated email from the ASF dual-hosted git repository.
omalley pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 5a38ed2f2274ccfad100cc89c729d470063e270d
Author: Fei Hui <fe...@gmail.com>
AuthorDate: Sun Oct 4 01:02:28 2020 +0800
HADOOP-17276. Extend CallerContext to make it include many items (#2327)
Cherry-picked from d0d10f7e by Owen O'Malley
---
.../hadoop/fs/CommonConfigurationKeysPublic.java | 3 +
.../java/org/apache/hadoop/ipc/CallerContext.java | 105 ++++++++++++++++++++-
.../src/main/resources/core-default.xml | 10 ++
.../org/apache/hadoop/ipc/TestCallerContext.java | 53 +++++++++++
.../metrics/TimelineServiceV1Publisher.java | 2 +-
.../metrics/TimelineServiceV2Publisher.java | 2 +-
.../impl/pb/ApplicationStateDataPBImpl.java | 7 +-
.../server/resourcemanager/TestRMAuditLogger.java | 4 +-
8 files changed, 176 insertions(+), 10 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index 1fa84c7..d769b28 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -371,6 +371,9 @@ public class CommonConfigurationKeysPublic {
"hadoop.caller.context.signature.max.size";
public static final int HADOOP_CALLER_CONTEXT_SIGNATURE_MAX_SIZE_DEFAULT =
40;
+ public static final String HADOOP_CALLER_CONTEXT_SEPARATOR_KEY =
+ "hadoop.caller.context.separator";
+ public static final String HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT = ",";
/**
* @see
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
index dd14ba1..e15340c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
@@ -21,10 +21,17 @@ import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_KEY;
/**
* A class defining the caller context for auditing coarse granularity
@@ -54,8 +61,8 @@ public final class CallerContext {
private final byte[] signature;
private CallerContext(Builder builder) {
- this.context = builder.context;
- this.signature = builder.signature;
+ this.context = builder.getContext();
+ this.signature = builder.getSignature();
}
public String getContext() {
@@ -109,11 +116,53 @@ public final class CallerContext {
/** The caller context builder. */
public static final class Builder {
- private final String context;
+ private static final String KEY_VALUE_SEPARATOR = ":";
+ /**
+ * The illegal separators include '\t', '\n', '='.
+ * User should not set illegal separator.
+ */
+ private static final Set<String> ILLEGAL_SEPARATORS =
+ Collections.unmodifiableSet(
+ new HashSet<>(Arrays.asList("\t", "\n", "=")));
+ private final String fieldSeparator;
+ private final StringBuilder sb = new StringBuilder();
private byte[] signature;
public Builder(String context) {
- this.context = context;
+ this(context, new Configuration());
+ }
+
+ public Builder(String context, Configuration conf) {
+ if (isValid(context)) {
+ sb.append(context);
+ }
+ fieldSeparator = conf.get(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY,
+ HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT);
+ checkFieldSeparator(fieldSeparator);
+ }
+
+ /**
+ * Check whether the separator is legal.
+ * The illegal separators include '\t', '\n', '='.
+ * Throw IllegalArgumentException if the separator is Illegal.
+ * @param separator the separator of fields.
+ */
+ private void checkFieldSeparator(String separator) {
+ if (ILLEGAL_SEPARATORS.contains(separator)) {
+ throw new IllegalArgumentException("Illegal field separator: "
+ + separator);
+ }
+ }
+
+ /**
+ * Whether the field is valid.
+ * The field should not contain '\t', '\n', '='.
+ * Because the context could be written to audit log.
+ * @param field one of the fields in context.
+ * @return true if the field is not null or empty.
+ */
+ private boolean isValid(String field) {
+ return field != null && field.length() > 0;
}
public Builder setSignature(byte[] signature) {
@@ -123,6 +172,54 @@ public final class CallerContext {
return this;
}
+ /**
+ * Get the context.
+ * For example, the context is "key1:value1,key2:value2".
+ * @return the valid context or null.
+ */
+ public String getContext() {
+ return sb.length() > 0 ? sb.toString() : null;
+ }
+
+ /**
+ * Get the signature.
+ * @return the signature.
+ */
+ public byte[] getSignature() {
+ return signature;
+ }
+
+ /**
+ * Append new field to the context.
+ * @param field one of fields to append.
+ * @return the builder.
+ */
+ public Builder append(String field) {
+ if (isValid(field)) {
+ if (sb.length() > 0) {
+ sb.append(fieldSeparator);
+ }
+ sb.append(field);
+ }
+ return this;
+ }
+
+ /**
+ * Append new field which contains key and value to the context.
+ * @param key the key of field.
+ * @param value the value of field.
+ * @return the builder.
+ */
+ public Builder append(String key, String value) {
+ if (isValid(key) && isValid(value)) {
+ if (sb.length() > 0) {
+ sb.append(fieldSeparator);
+ }
+ sb.append(key).append(KEY_VALUE_SEPARATOR).append(value);
+ }
+ return this;
+ }
+
public CallerContext build() {
return new CallerContext(this);
}
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index f5978a3..8a59f3c 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -3716,6 +3716,16 @@ The switch to turn S3A auditing on or off.
in audit logs.
</description>
</property>
+ <property>
+ <name>hadoop.caller.context.separator</name>
+ <value>,</value>
+ <description>
+ The separator is for context which maybe contain many fields. For example,
+ if the separator is ',', and there are two key/value fields in context,
+ in which case the context string is "key1:value1,key2:value2". The
+ separator should not contain '\t', '\n', '='.
+ </description>
+ </property>
<!-- SequenceFile's Sorter properties -->
<property>
<name>seq.io.sort.mb</name>
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallerContext.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallerContext.java
new file mode 100644
index 0000000..fc1057b
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallerContext.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ipc;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_KEY;
+
+public class TestCallerContext {
+ @Test
+ public void testBuilderAppend() {
+ Configuration conf = new Configuration();
+ conf.set(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY, "$");
+ CallerContext.Builder builder = new CallerContext.Builder(null, conf);
+ CallerContext context = builder.append("context1")
+ .append("context2").append("key3", "value3").build();
+ Assert.assertEquals(true,
+ context.getContext().contains("$"));
+ String[] items = context.getContext().split("\\$");
+ Assert.assertEquals(3, items.length);
+ Assert.assertEquals("key3:value3", items[2]);
+
+ builder.append("$$");
+ Assert.assertEquals("context1$context2$key3:value3$$$",
+ builder.build().getContext());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testNewBuilder() {
+ Configuration conf = new Configuration();
+ // Set illegal separator.
+ conf.set(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY, "\t");
+ CallerContext.Builder builder = new CallerContext.Builder(null, conf);
+ builder.build();
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
index 1d89f1e..f1b80a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
@@ -176,7 +176,7 @@ public class TimelineServiceV1Publisher extends AbstractSystemMetricsPublisher {
entityInfo.put(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION,
app.getAppNodeLabelExpression());
if (app.getCallerContext() != null) {
- if (app.getCallerContext().getContext() != null) {
+ if (app.getCallerContext().isContextValid()) {
entityInfo.put(ApplicationMetricsConstants.YARN_APP_CALLER_CONTEXT,
app.getCallerContext().getContext());
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
index d359173..8047acb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
@@ -127,7 +127,7 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher {
ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION,
app.getAppNodeLabelExpression());
if (app.getCallerContext() != null) {
- if (app.getCallerContext().getContext() != null) {
+ if (app.getCallerContext().isContextValid()) {
entityInfo.put(ApplicationMetricsConstants.YARN_APP_CALLER_CONTEXT,
app.getCallerContext().getContext());
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java
index 5dcb70b..c0f7bb0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java
@@ -274,14 +274,17 @@ public class ApplicationStateDataPBImpl extends ApplicationStateData {
RpcHeaderProtos.RPCCallerContextProto.Builder b = RpcHeaderProtos.RPCCallerContextProto
.newBuilder();
- if (callerContext.getContext() != null) {
+ if (callerContext.isContextValid()) {
b.setContext(callerContext.getContext());
}
if (callerContext.getSignature() != null) {
b.setSignature(ByteString.copyFrom(callerContext.getSignature()));
}
- builder.setCallerContext(b);
+ if(callerContext.isContextValid()
+ || callerContext.getSignature() != null) {
+ builder.setCallerContext(b);
+ }
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
index e8a532d..c78a778 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
@@ -177,7 +177,7 @@ public class TestRMAuditLogger {
expLog.append("\tRESOURCE=<memory:1536, vcores:1>");
}
if (callerContext != null) {
- if (callerContext.getContext() != null) {
+ if (callerContext.isContextValid()) {
expLog.append("\tCALLERCONTEXT=context");
}
if (callerContext.getSignature() != null) {
@@ -328,7 +328,7 @@ public class TestRMAuditLogger {
expLog.append("\tRESOURCE=<memory:1536, vcores:1>");
}
if (callerContext != null) {
- if (callerContext.getContext() != null) {
+ if (callerContext.isContextValid()) {
expLog.append("\tCALLERCONTEXT=context");
}
if (callerContext.getSignature() != null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org