You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ep...@apache.org on 2017/03/31 17:31:55 UTC
hadoop git commit: YARN-6354. LeveldbRMStateStore can parse invalid
keys when recovering reservations. Contributed by Jason Lowe
Repository: hadoop
Updated Branches:
refs/heads/trunk 4d1fac5df -> 318bfb01b
YARN-6354. LeveldbRMStateStore can parse invalid keys when recovering reservations. Contributed by Jason Lowe
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/318bfb01
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/318bfb01
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/318bfb01
Branch: refs/heads/trunk
Commit: 318bfb01bc6793da09e32e9cc292eb63224b6ca2
Parents: 4d1fac5
Author: Eric Payne <ep...@apache.org>
Authored: Fri Mar 31 12:30:35 2017 -0500
Committer: Eric Payne <ep...@apache.org>
Committed: Fri Mar 31 12:30:35 2017 -0500
----------------------------------------------------------------------
.../recovery/LeveldbRMStateStore.java | 8 ++++++++
.../recovery/TestLeveldbRMStateStore.java | 19 +++++++++++++++----
2 files changed, 23 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/318bfb01/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
index 02f90dd..2ca53db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
@@ -214,6 +214,11 @@ public class LeveldbRMStateStore extends RMStateStore {
return db == null;
}
+ @VisibleForTesting
+ DB getDatabase() {
+ return db;
+ }
+
@Override
protected Version loadVersion() throws Exception {
Version version = null;
@@ -284,6 +289,9 @@ public class LeveldbRMStateStore extends RMStateStore {
while (iter.hasNext()) {
Entry<byte[],byte[]> entry = iter.next();
String key = asString(entry.getKey());
+ if (!key.startsWith(RM_RESERVATION_KEY_PREFIX)) {
+ break;
+ }
String planReservationString =
key.substring(RM_RESERVATION_KEY_PREFIX.length());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/318bfb01/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java
index 4297e73..51adbe1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.fusesource.leveldbjni.JniDBFactory;
import org.iq80.leveldb.DB;
import org.junit.After;
import org.junit.Before;
@@ -125,17 +126,27 @@ public class TestLeveldbRMStateStore extends RMStateStoreTestBase {
public void testCompactionCycle() throws Exception {
final DB mockdb = mock(DB.class);
conf.setLong(YarnConfiguration.RM_LEVELDB_COMPACTION_INTERVAL_SECS, 1);
- LeveldbRMStateStore store = new LeveldbRMStateStore() {
+ stateStore = new LeveldbRMStateStore() {
@Override
protected DB openDatabase() throws Exception {
return mockdb;
}
};
- store.init(conf);
- store.start();
+ stateStore.init(conf);
+ stateStore.start();
verify(mockdb, timeout(10000)).compactRange(
(byte[]) isNull(), (byte[]) isNull());
- store.close();
+ }
+
+ @Test
+ public void testBadKeyIteration() throws Exception {
+ stateStore = new LeveldbRMStateStore();
+ stateStore.init(conf);
+ stateStore.start();
+ DB db = stateStore.getDatabase();
+ // add an entry that appears at the end of the database when iterating
+ db.put(JniDBFactory.bytes("zzz"), JniDBFactory.bytes("z"));
+ stateStore.loadState();
}
class LeveldbStateStoreTester implements RMStateStoreHelper {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org