You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2020/04/09 15:40:35 UTC
[hadoop] branch branch-3.2 updated: YARN-10003.
YarnConfigurationStore#checkVersion throws exception that belongs to
RMStateStore. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new d2853d1 YARN-10003. YarnConfigurationStore#checkVersion throws exception that belongs to RMStateStore. Contributed by Benjamin Teke
d2853d1 is described below
commit d2853d1bb0eb210d850bf5e689e351d50d8dd71f
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Thu Apr 9 17:39:25 2020 +0200
YARN-10003. YarnConfigurationStore#checkVersion throws exception that belongs to RMStateStore. Contributed by Benjamin Teke
---
.../capacity/conf/LeveldbConfigurationStore.java | 10 +++++---
...YarnConfStoreVersionIncompatibleException.java} | 23 +++++++++++++----
.../capacity/conf/YarnConfigurationStore.java | 23 ++++++++---------
.../conf/TestInMemoryConfigurationStore.java | 13 ++++++++++
.../conf/TestLeveldbConfigurationStore.java | 18 ++++++++++++++
.../capacity/conf/TestZKConfigurationStore.java | 29 ++++++++++++++++++++++
6 files changed, 96 insertions(+), 20 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
index 2966c94..5da002e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
@@ -339,11 +339,15 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
@Override
public void storeVersion() throws Exception {
- String key = VERSION_KEY;
- byte[] data = ((VersionPBImpl) CURRENT_VERSION_INFO).getProto()
+ storeVersion(CURRENT_VERSION_INFO);
+ }
+
+ @VisibleForTesting
+ protected void storeVersion(Version version) throws Exception {
+ byte[] data = ((VersionPBImpl) version).getProto()
.toByteArray();
try {
- db.put(bytes(key), data);
+ db.put(bytes(VERSION_KEY), data);
} catch (DBException e) {
throw new IOException(e);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestInMemoryConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfStoreVersionIncompatibleException.java
similarity index 58%
copy from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestInMemoryConfigurationStore.java
copy to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfStoreVersionIncompatibleException.java
index c40d16a..2213ee2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestInMemoryConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfStoreVersionIncompatibleException.java
@@ -18,13 +18,26 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
/**
- * Tests {@link InMemoryConfigurationStore}.
+ * This exception is thrown by {@link YarnConfigurationStore} if it's loading
+ * an incompatible persisted schema version.
*/
-public class TestInMemoryConfigurationStore extends ConfigurationStoreBaseTest {
+public class YarnConfStoreVersionIncompatibleException extends
+ YarnException {
+ private static final long serialVersionUID = -2829858253579013629L;
+
+ public YarnConfStoreVersionIncompatibleException(Throwable cause) {
+ super(cause);
+ }
+
+ public YarnConfStoreVersionIncompatibleException(String message) {
+ super(message);
+ }
- @Override
- protected YarnConfigurationStore createConfStore() {
- return new InMemoryConfigurationStore();
+ public YarnConfStoreVersionIncompatibleException(
+ String message, Throwable cause) {
+ super(message, cause);
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
index 6af11a3..597f67a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
@@ -23,7 +23,6 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateVersionIncompatibleException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import java.io.IOException;
@@ -103,6 +102,7 @@ public abstract class YarnConfigurationStore {
/**
* Logs the configuration change to backing store.
+ *
* @param logMutation configuration change to be persisted in write ahead log
* @throws IOException if logging fails
*/
@@ -167,23 +167,22 @@ public abstract class YarnConfigurationStore {
protected abstract Version getCurrentVersion();
public void checkVersion() throws Exception {
- // TODO this was taken from RMStateStore. Should probably refactor
Version loadedVersion = getConfStoreVersion();
- LOG.info("Loaded configuration store version info " + loadedVersion);
- if (loadedVersion != null && loadedVersion.equals(getCurrentVersion())) {
+ Version currentVersion = getCurrentVersion();
+ LOG.info("Loaded configuration store version info {}", loadedVersion);
+
+ // when hard-coded schema version (currentVersion) is null the version check
+ // is unnecessary
+ if (currentVersion == null || currentVersion.equals(loadedVersion)) {
return;
}
// if there is no version info, treat it as CURRENT_VERSION_INFO;
- if (loadedVersion == null) {
- loadedVersion = getCurrentVersion();
- }
- if (loadedVersion.isCompatibleTo(getCurrentVersion())) {
- LOG.info("Storing configuration store version info "
- + getCurrentVersion());
+ if (loadedVersion == null || loadedVersion.isCompatibleTo(currentVersion)) {
+ LOG.info("Storing configuration store version info {}", currentVersion);
storeVersion();
} else {
- throw new RMStateVersionIncompatibleException(
- "Expecting configuration store version " + getCurrentVersion()
+ throw new YarnConfStoreVersionIncompatibleException(
+ "Expecting configuration store version " + currentVersion
+ ", but loading version " + loadedVersion);
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestInMemoryConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestInMemoryConfigurationStore.java
index c40d16a..5ebad50 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestInMemoryConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestInMemoryConfigurationStore.java
@@ -18,6 +18,10 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+import org.junit.Test;
+
+import static org.junit.Assert.fail;
+
/**
* Tests {@link InMemoryConfigurationStore}.
*/
@@ -27,4 +31,13 @@ public class TestInMemoryConfigurationStore extends ConfigurationStoreBaseTest {
protected YarnConfigurationStore createConfStore() {
return new InMemoryConfigurationStore();
}
+
+ @Test
+ public void checkVersion() {
+ try {
+ confStore.checkVersion();
+ } catch (Exception e) {
+ fail("checkVersion threw exception");
+ }
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java
index 3381637..14e4001 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+import org.apache.hadoop.yarn.server.records.Version;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.FileUtil;
@@ -73,6 +74,23 @@ public class TestLeveldbConfigurationStore extends ConfigurationStoreBaseTest {
confStore.close();
}
+ @Test(expected = YarnConfStoreVersionIncompatibleException.class)
+ public void testIncompatibleVersion() throws Exception {
+ try {
+ confStore.initialize(conf, schedConf, rmContext);
+
+ Version otherVersion = Version.newInstance(1, 1);
+ ((LeveldbConfigurationStore) confStore).storeVersion(otherVersion);
+
+ assertEquals("The configuration store should have stored the new" +
+ "version.", otherVersion,
+ confStore.getConfStoreVersion());
+ confStore.checkVersion();
+ } finally {
+ confStore.close();
+ }
+ }
+
@Test
public void testPersistConfiguration() throws Exception {
schedConf.set("key", "val");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
index eae80d5..7d8b3c1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
@@ -18,6 +18,11 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+import org.apache.hadoop.util.curator.ZKCuratorManager;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.data.ACL;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.curator.framework.CuratorFramework;
@@ -47,6 +52,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedList;
+import java.util.List;
import java.util.Map;
import static org.junit.Assert.assertEquals;
@@ -111,6 +117,29 @@ public class TestZKConfigurationStore extends ConfigurationStoreBaseTest {
confStore.getConfStoreVersion());
}
+ @Test(expected = YarnConfStoreVersionIncompatibleException.class)
+ public void testIncompatibleVersion() throws Exception {
+ confStore.initialize(conf, schedConf, rmContext);
+
+ Version otherVersion = Version.newInstance(1, 1);
+ String znodeParentPath = conf.get(YarnConfiguration.
+ RM_SCHEDCONF_STORE_ZK_PARENT_PATH,
+ YarnConfiguration.DEFAULT_RM_SCHEDCONF_STORE_ZK_PARENT_PATH);
+ String zkVersionPath = ZKCuratorManager.getNodePath(znodeParentPath,
+ "VERSION");
+ String fencingNodePath = ZKCuratorManager.getNodePath(znodeParentPath,
+ "FENCING");
+ byte[] versionData =
+ ((VersionPBImpl) otherVersion).getProto().toByteArray();
+ List<ACL> zkAcl = ZKCuratorManager.getZKAcls(conf);
+ ((ZKConfigurationStore) confStore).zkManager.safeCreate(zkVersionPath,
+ versionData, zkAcl, CreateMode.PERSISTENT, zkAcl, fencingNodePath);
+
+ assertEquals("The configuration store should have stored the new" +
+ "version.", otherVersion, confStore.getConfStoreVersion());
+ confStore.checkVersion();
+ }
+
@Test
public void testPersistConfiguration() throws Exception {
schedConf.set("key", "val");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org