You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2021/06/24 18:20:35 UTC
[hadoop] branch branch-3.2 updated: YARN-10828. Backport YARN-9789
to branch-3.2. Contributed by Tarun Parimi
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new c8b090d YARN-10828. Backport YARN-9789 to branch-3.2. Contributed by Tarun Parimi
c8b090d is described below
commit c8b090d9a492211e3be55e6adb79da57c6db304e
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Thu Jun 24 20:20:23 2021 +0200
YARN-10828. Backport YARN-9789 to branch-3.2. Contributed by Tarun Parimi
---
.../capacity/conf/LeveldbConfigurationStore.java | 17 +++++++++----
.../capacity/conf/ZKConfigurationStore.java | 20 ++++++++-------
.../conf/TestLeveldbConfigurationStore.java | 29 ++++++++++++++++++++++
3 files changed, 52 insertions(+), 14 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
index b347aba..1ff53a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
@@ -193,12 +193,14 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
@Override
public void logMutation(LogMutation logMutation) throws IOException {
- LinkedList<LogMutation> logs = deserLogMutations(db.get(bytes(LOG_KEY)));
- logs.add(logMutation);
- if (logs.size() > maxLogs) {
- logs.removeFirst();
+ if(maxLogs > 0) {
+ LinkedList<LogMutation> logs = deserLogMutations(db.get(bytes(LOG_KEY)));
+ logs.add(logMutation);
+ if (logs.size() > maxLogs) {
+ logs.removeFirst();
+ }
+ db.put(bytes(LOG_KEY), serLogMutations(logs));
}
- db.put(bytes(LOG_KEY), serLogMutations(logs));
pendingMutation = logMutation;
}
@@ -293,6 +295,11 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
return deserLogMutations(db.get(bytes(LOG_KEY)));
}
+ @VisibleForTesting
+ protected DB getDB() {
+ return db;
+ }
+
@Override
public void storeVersion() throws Exception {
try {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
index 229f680..3742c36 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
@@ -156,16 +156,18 @@ public class ZKConfigurationStore extends YarnConfigurationStore {
@Override
public void logMutation(LogMutation logMutation) throws Exception {
- byte[] storedLogs = getZkData(logsPath);
- LinkedList<LogMutation> logs = new LinkedList<>();
- if (storedLogs != null) {
- logs = unsafeCast(deserializeObject(storedLogs));
- }
- logs.add(logMutation);
- if (logs.size() > maxLogs) {
- logs.remove(logs.removeFirst());
+ if(maxLogs > 0) {
+ byte[] storedLogs = getZkData(logsPath);
+ LinkedList<LogMutation> logs = new LinkedList<>();
+ if (storedLogs != null) {
+ logs = unsafeCast(deserializeObject(storedLogs));
+ }
+ logs.add(logMutation);
+ if (logs.size() > maxLogs) {
+ logs.remove(logs.removeFirst());
+ }
+ safeSetZkData(logsPath, logs);
}
- safeSetZkData(logsPath, logs);
pendingMutation = logMutation;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java
index 4c92120..e88e7de 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java
@@ -18,7 +18,12 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.hadoop.yarn.server.records.Version;
+import org.iq80.leveldb.DB;
+import org.iq80.leveldb.DBIterator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.FileUtil;
@@ -36,6 +41,7 @@ import org.junit.Test;
import java.io.File;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
/**
@@ -119,6 +125,29 @@ public class TestLeveldbConfigurationStore extends
rm2.close();
}
+ @Test
+ public void testDisableAuditLogs() throws Exception {
+ conf.setLong(YarnConfiguration.RM_SCHEDCONF_MAX_LOGS, 0);
+ confStore.initialize(conf, schedConf, rmContext);
+
+ prepareLogMutation("key1", "val1");
+
+ boolean logKeyPresent = false;
+ DB db = ((LeveldbConfigurationStore) confStore).getDB();
+ DBIterator itr = db.iterator();
+ itr.seekToFirst();
+ while (itr.hasNext()) {
+ Map.Entry<byte[], byte[]> entry = itr.next();
+ String key = new String(entry.getKey(), StandardCharsets.UTF_8);
+ if (key.equals("log")) {
+ logKeyPresent = true;
+ break;
+ }
+ }
+ assertFalse("Audit Log is not disabled", logKeyPresent);
+ confStore.close();
+ }
+
@Override
public YarnConfigurationStore createConfStore() {
return new LeveldbConfigurationStore();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org