You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2019/12/05 19:44:09 UTC
[hadoop] branch trunk updated: YARN-9789. Disable Option for Write
Ahead Logs of LogMutation. Contributed by Prabhu Joseph
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 5cc6f94 YARN-9789. Disable Option for Write Ahead Logs of LogMutation. Contributed by Prabhu Joseph
5cc6f94 is described below
commit 5cc6f945da648a48441b00aca89f3cc1a9777143
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Thu Dec 5 20:43:26 2019 +0100
YARN-9789. Disable Option for Write Ahead Logs of LogMutation. Contributed by Prabhu Joseph
---
.../capacity/conf/LeveldbConfigurationStore.java | 17 ++++++++----
.../capacity/conf/ZKConfigurationStore.java | 22 ++++++++-------
.../conf/TestLeveldbConfigurationStore.java | 31 ++++++++++++++++++++++
.../capacity/conf/TestZKConfigurationStore.java | 23 ++++++++++++++++
4 files changed, 78 insertions(+), 15 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
index 2966c94..39cd8ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
@@ -224,12 +224,14 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
@Override
public void logMutation(LogMutation logMutation) throws IOException {
- LinkedList<LogMutation> logs = deserLogMutations(db.get(bytes(LOG_KEY)));
- logs.add(logMutation);
- if (logs.size() > maxLogs) {
- logs.removeFirst();
+ if (maxLogs > 0) {
+ LinkedList<LogMutation> logs = deserLogMutations(db.get(bytes(LOG_KEY)));
+ logs.add(logMutation);
+ if (logs.size() > maxLogs) {
+ logs.removeFirst();
+ }
+ db.put(bytes(LOG_KEY), serLogMutations(logs));
}
- db.put(bytes(LOG_KEY), serLogMutations(logs));
pendingMutation = logMutation;
}
@@ -337,6 +339,11 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
return deserLogMutations(db.get(bytes(LOG_KEY)));
}
+ @VisibleForTesting
+ protected DB getDB() {
+ return db;
+ }
+
@Override
public void storeVersion() throws Exception {
String key = VERSION_KEY;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
index 1aee415..75ae727 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
@@ -162,17 +162,19 @@ public class ZKConfigurationStore extends YarnConfigurationStore {
@Override
public void logMutation(LogMutation logMutation) throws Exception {
- byte[] storedLogs = zkManager.getData(logsPath);
- LinkedList<LogMutation> logs = new LinkedList<>();
- if (storedLogs != null) {
- logs = (LinkedList<LogMutation>) deserializeObject(storedLogs);
- }
- logs.add(logMutation);
- if (logs.size() > maxLogs) {
- logs.remove(logs.removeFirst());
+ if (maxLogs > 0) {
+ byte[] storedLogs = zkManager.getData(logsPath);
+ LinkedList<LogMutation> logs = new LinkedList<>();
+ if (storedLogs != null) {
+ logs = (LinkedList<LogMutation>) deserializeObject(storedLogs);
+ }
+ logs.add(logMutation);
+ if (logs.size() > maxLogs) {
+ logs.remove(logs.removeFirst());
+ }
+ zkManager.safeSetData(logsPath, serializeObject(logs), -1, zkAcl,
+ fencingNodePath);
}
- zkManager.safeSetData(logsPath, serializeObject(logs), -1, zkAcl,
- fencingNodePath);
pendingMutation = logMutation;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java
index 3381637..5f78aa2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java
@@ -31,13 +31,17 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import org.junit.Before;
import org.junit.Test;
+import org.iq80.leveldb.DB;
+import org.iq80.leveldb.DBIterator;
import java.io.File;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
/**
@@ -113,6 +117,33 @@ public class TestLeveldbConfigurationStore extends ConfigurationStoreBaseTest {
}
@Test
+ public void testDisableAuditLogs() throws Exception {
+ conf.setLong(YarnConfiguration.RM_SCHEDCONF_MAX_LOGS, 0);
+ confStore.initialize(conf, schedConf, rmContext);
+
+ Map<String, String> update = new HashMap<>();
+ update.put("key1", "val1");
+ YarnConfigurationStore.LogMutation mutation =
+ new YarnConfigurationStore.LogMutation(update, TEST_USER);
+ confStore.logMutation(mutation);
+
+ boolean logKeyPresent = false;
+ DB db = ((LeveldbConfigurationStore) confStore).getDB();
+ DBIterator itr = db.iterator();
+ itr.seekToFirst();
+ while (itr.hasNext()) {
+ Map.Entry<byte[], byte[]> entry = itr.next();
+ String key = new String(entry.getKey(), StandardCharsets.UTF_8);
+ if (key.equals("log")) {
+ logKeyPresent = true;
+ break;
+ }
+ }
+ assertFalse("Audit Log is not disabled", logKeyPresent);
+ confStore.close();
+ }
+
+ @Test
public void testMaxLogs() throws Exception {
conf.setLong(YarnConfiguration.RM_SCHEDCONF_MAX_LOGS, 2);
confStore.initialize(conf, schedConf, rmContext);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
index eae80d5..57ccd75 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+import org.apache.hadoop.util.curator.ZKCuratorManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.curator.framework.CuratorFramework;
@@ -221,6 +222,28 @@ public class TestZKConfigurationStore extends ConfigurationStoreBaseTest {
assertEquals("val3", logs.get(1).getUpdates().get("key3"));
}
+
+ @Test
+ public void testDisableAuditLogs() throws Exception {
+ conf.setLong(YarnConfiguration.RM_SCHEDCONF_MAX_LOGS, 0);
+ confStore.initialize(conf, schedConf, rmContext);
+ String znodeParentPath = conf.get(YarnConfiguration.
+ RM_SCHEDCONF_STORE_ZK_PARENT_PATH,
+ YarnConfiguration.DEFAULT_RM_SCHEDCONF_STORE_ZK_PARENT_PATH);
+ String logsPath = ZKCuratorManager.getNodePath(znodeParentPath, "LOGS");
+ byte[] data = null;
+ ((ZKConfigurationStore) confStore).zkManager.setData(logsPath, data, -1);
+
+ Map<String, String> update = new HashMap<>();
+ update.put("key1", "val1");
+ YarnConfigurationStore.LogMutation mutation =
+ new YarnConfigurationStore.LogMutation(update, TEST_USER);
+ confStore.logMutation(mutation);
+
+ data = ((ZKConfigurationStore) confStore).zkManager.getData(logsPath);
+ assertNull("Failed to Disable Audit Logs", data);
+ }
+
public Configuration createRMHAConf(String rmIds, String rmId,
int adminPort) {
Configuration conf = new YarnConfiguration();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org