You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/04/09 14:35:47 UTC
[3/7] incubator-slider git commit: SLIDER-600: history entries of
unknown roles are now discarded
SLIDER-600: history entries of unknown roles are now discarded
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f10bef8a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f10bef8a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f10bef8a
Branch: refs/heads/develop
Commit: f10bef8ad7bf88e47ab4ff987ca21d42fa42029e
Parents: 0ac74d5
Author: Steve Loughran <st...@apache.org>
Authored: Thu Apr 9 11:38:38 2015 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Apr 9 11:38:38 2015 +0100
----------------------------------------------------------------------
.../slider/server/avro/RoleHistoryRecord.avsc | 21 +++--
.../server/appmaster/state/RoleHistory.java | 68 ++++++++++++---
.../slider/server/avro/LoadedRoleHistory.java | 63 ++++++++++++++
.../slider/server/avro/RoleHistoryWriter.java | 87 ++++++--------------
.../TestMockAppStateFlexDynamicRoles.groovy | 13 ++-
.../model/history/TestRoleHistoryRW.groovy | 74 ++++++++++++++---
.../slider/server/avro/history-v01-3-role.json | 6 ++
.../slider/server/avro/history-v01-4-role.json | 6 --
.../slider/server/avro/history-v01-6-role.json | 8 ++
9 files changed, 244 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f10bef8a/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc
----------------------------------------------------------------------
diff --git a/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc b/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc
index 0e42e13..778a728 100644
--- a/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc
+++ b/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc
@@ -30,10 +30,6 @@
"type": "int"
},
{
- "name": "rolename",
- "type": "string"
- },
- {
"name": "active",
"type": "boolean"
},
@@ -74,6 +70,20 @@
},
{
"type": "record",
+ "name": "RoleHistoryMapping",
+ "namespace": "org.apache.slider.server.avro",
+ "fields": [
+ {
+ "name": "rolemap",
+ "type": {
+ "type": "map",
+ "values": "long"
+ }
+ }
+ ]
+ },
+ {
+ "type": "record",
"name": "RoleHistoryFooter",
"namespace": "org.apache.slider.server.avro",
"fields": [
@@ -94,7 +104,8 @@
"type": [
"org.apache.slider.server.avro.NodeEntryRecord",
"org.apache.slider.server.avro.RoleHistoryHeader",
- "org.apache.slider.server.avro.RoleHistoryFooter"
+ "org.apache.slider.server.avro.RoleHistoryFooter",
+ "org.apache.slider.server.avro.RoleHistoryMapping"
]
}
]
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f10bef8a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index c0de492..e95e2a6 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -29,6 +29,8 @@ import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
+import org.apache.slider.server.avro.LoadedRoleHistory;
+import org.apache.slider.server.avro.NodeEntryRecord;
import org.apache.slider.server.avro.RoleHistoryHeader;
import org.apache.slider.server.avro.RoleHistoryWriter;
import org.slf4j.Logger;
@@ -189,23 +191,61 @@ public class RoleHistory {
/**
* Reset the variables -this does not adjust the fixed attributes
* of the history.
+ * <p>
* This intended for use by the RoleWriter logic.
+ * @throws BadConfigException if there is a problem rebuilding the state
*/
- public synchronized void prepareForReading(RoleHistoryHeader header) throws
- IOException,
- BadConfigException {
+ private void prepareForReading(RoleHistoryHeader header)
+ throws BadConfigException {
reset();
int roleCountInSource = header.getRoles();
if (roleCountInSource != roleSize) {
- throw new IOException("Number of roles in source " + roleCountInSource
- + " does not match the expected number of " +
- roleSize);
+ log.warn("Number of roles in source {}"
+ +" does not match the expected number of {}",
+ roleCountInSource,
+ roleSize);
}
//record when the data was loaded
setThawedDataTime(header.getSaved());
}
-
+
+ /**
+ * rebuild the placement history from the loaded role history
+ * @param loadedRoleHistory loaded history
+ * @return the number of entries discarded
+ * @throws BadConfigException if there is a problem rebuilding the state
+ */
+ @VisibleForTesting
+ public synchronized int rebuild(LoadedRoleHistory loadedRoleHistory) throws BadConfigException {
+ RoleHistoryHeader header = loadedRoleHistory.getHeader();
+ prepareForReading(header);
+ int discarded = 0;
+ Long saved = header.getSaved();
+ for (NodeEntryRecord nodeEntryRecord : loadedRoleHistory.records) {
+ Integer roleId = nodeEntryRecord.getRole();
+ NodeEntry nodeEntry = new NodeEntry(roleId);
+ nodeEntry.setLastUsed(nodeEntryRecord.getLastUsed());
+ if (nodeEntryRecord.getActive()) {
+ //if active at the time of save, make the last used time the save time
+ nodeEntry.setLastUsed(saved);
+ }
+ String hostname = SliderUtils.sequenceToString(nodeEntryRecord.getHost());
+ ProviderRole providerRole = lookupRole(roleId);
+ if (providerRole == null) {
+ // discarding entry
+ log.info("Discarding history entry with unknown role: {} on host {}",
+ roleId, hostname);
+ discarded ++;
+ } else {
+ NodeInstance instance = getOrCreateNodeInstance(hostname);
+ instance.set(roleId, nodeEntry);
+ }
+ }
+ return discarded;
+ }
+
+
public synchronized long getStartTime() {
return startTime;
}
@@ -396,24 +436,26 @@ public class RoleHistory {
* Handle the start process <i>after the history has been rebuilt</i>,
* and after any gc/purge
*/
- @VisibleForTesting
public synchronized boolean onThaw() throws BadConfigException {
assert filesystem != null;
assert historyPath != null;
boolean thawSuccessful = false;
//load in files from data dir
- Path loaded = null;
+
+ LoadedRoleHistory loadedRoleHistory = null;
try {
- loaded = historyWriter.loadFromHistoryDir(filesystem, historyPath, this);
+ loadedRoleHistory = historyWriter.loadFromHistoryDir(filesystem, historyPath);
} catch (IOException e) {
log.warn("Exception trying to load history from {}", historyPath, e);
}
- if (loaded != null) {
+ if (loadedRoleHistory != null) {
+ rebuild(loadedRoleHistory);
thawSuccessful = true;
- log.debug("loaded history from {}", loaded);
+ Path loadPath = loadedRoleHistory.getPath();;
+ log.debug("loaded history from {}", loadPath);
// delete any old entries
try {
- int count = historyWriter.purgeOlderHistoryEntries(filesystem, loaded);
+ int count = historyWriter.purgeOlderHistoryEntries(filesystem, loadPath);
log.debug("Deleted {} old history entries", count);
} catch (IOException e) {
log.info("Ignoring exception raised while trying to delete old entries",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f10bef8a/slider-core/src/main/java/org/apache/slider/server/avro/LoadedRoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/avro/LoadedRoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/avro/LoadedRoleHistory.java
new file mode 100644
index 0000000..866e716
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/avro/LoadedRoleHistory.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.avro;
+
+import org.apache.hadoop.fs.Path;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * The role history
+ */
+public class LoadedRoleHistory {
+
+ private RoleHistoryHeader header;
+
+ private Path path;
+
+ public final Map<String, Integer> roleMap = new HashMap<>();
+
+ public final List<NodeEntryRecord> records = new ArrayList<>();
+
+ public void add(NodeEntryRecord record) {
+ records.add(record);
+ }
+
+ public int size() {
+ return records.size();
+ }
+ public RoleHistoryHeader getHeader() {
+ return header;
+ }
+
+ public void setHeader(RoleHistoryHeader header) {
+ this.header = header;
+ }
+
+ public Path getPath() {
+ return path;
+ }
+
+ public void setPath(Path path) {
+ this.path = path;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f10bef8a/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java b/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
index b938f30..b638ae9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
+++ b/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
@@ -47,8 +47,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.EOFException;
-import java.io.File;
-import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
@@ -61,7 +59,7 @@ import java.util.ListIterator;
import java.util.Locale;
/**
- * Write out the role history to an output stream
+ * Write out the role history to an output stream.
*/
public class RoleHistoryWriter {
protected static final Logger log =
@@ -107,12 +105,10 @@ public class RoleHistoryWriter {
Collection<NodeInstance> instances = history.cloneNodemap().values();
for (NodeInstance instance : instances) {
for (int role = 0; role < roles; role++) {
- ProviderRole providerRole = history.lookupRole(role);
- String rolename = providerRole != null ? providerRole.name : "";
NodeEntry nodeEntry = instance.get(role);
if (nodeEntry != null) {
- NodeEntryRecord ner = build(nodeEntry, role, "", instance.hostname);
+ NodeEntryRecord ner = build(nodeEntry, role, instance.hostname);
record = new RoleHistoryRecord(ner);
writer.write(record, encoder);
count++;
@@ -174,9 +170,9 @@ public class RoleHistoryWriter {
* @param rolename
*@param hostname name @return
*/
- private NodeEntryRecord build(NodeEntry entry, int role, String rolename, String hostname) {
+ private NodeEntryRecord build(NodeEntry entry, int role, String hostname) {
NodeEntryRecord record = new NodeEntryRecord(
- hostname, role, rolename, entry.getLive() > 0, entry.getLastUsed()
+ hostname, role, entry.getLive() > 0, entry.getLastUsed()
);
return record;
}
@@ -185,16 +181,14 @@ public class RoleHistoryWriter {
* Read a history, returning one that is ready to have its onThaw()
* method called
* @param in input source
- * @param history a history set up with the expected roles;
- * this will be built up with a node map configured with the node instances
- * and entries loaded from the source
* @return no. of entries read
* @throws IOException problems
*/
- public int read(InputStream in, RoleHistory history) throws
+ public LoadedRoleHistory read(InputStream in) throws
IOException,
BadConfigException {
try {
+ LoadedRoleHistory loadedRoleHistory = new LoadedRoleHistory();
DatumReader<RoleHistoryRecord> reader =
new SpecificDatumReader<>(RoleHistoryRecord.class);
Decoder decoder =
@@ -215,7 +209,7 @@ public class RoleHistoryWriter {
header.getVersion(),
ROLE_HISTORY_VERSION));
}
- history.prepareForReading(header);
+ loadedRoleHistory.setHeader(header);
RoleHistoryFooter footer;
int records = 0;
//go through reading data
@@ -234,18 +228,7 @@ public class RoleHistoryWriter {
}
records++;
NodeEntryRecord nodeEntryRecord = (NodeEntryRecord) entry;
- Integer roleId = nodeEntryRecord.getRole();
- NodeEntry nodeEntry = new NodeEntry(roleId);
- nodeEntry.setLastUsed(nodeEntryRecord.getLastUsed());
- if (nodeEntryRecord.getActive()) {
- //if active at the time of save, make the last used time the save time
- nodeEntry.setLastUsed(saved);
- }
-
- String hostname =
- SliderUtils.sequenceToString(nodeEntryRecord.getHost());
- NodeInstance instance = history.getOrCreateNodeInstance(hostname);
- instance.set(roleId, nodeEntry);
+ loadedRoleHistory.add(nodeEntryRecord);
}
} catch (EOFException e) {
EOFException ex = new EOFException(
@@ -264,7 +247,7 @@ public class RoleHistoryWriter {
log.warn("mismatch between no of records saved {} and number read {}",
footer.getCount(), records);
}
- return records;
+ return loadedRoleHistory;
} finally {
in.close();
}
@@ -275,45 +258,27 @@ public class RoleHistoryWriter {
* Read a role history from a path in a filesystem
* @param fs filesystem
* @param path path to the file
- * @param roleHistory history to build
- * @return the number of records read
+ * @return the records read
* @throws IOException any problem
*/
- public int read(FileSystem fs, Path path, RoleHistory roleHistory) throws
- IOException,
- BadConfigException {
+ public LoadedRoleHistory read(FileSystem fs, Path path)
+ throws IOException, BadConfigException {
FSDataInputStream instream = fs.open(path);
- return read(instream, roleHistory);
- }
-
- /**
- * Read a role history from local file
- * @param file path to the file
- * @param roleHistory history to build
- * @return the number of records read
- * @throws IOException any problem
- */
- public int read(File file, RoleHistory roleHistory) throws
- IOException,
- BadConfigException {
-
-
- return read(new FileInputStream(file), roleHistory);
+ return read(instream);
}
/**
* Read from a resource in the classpath -used for testing
* @param resource resource
* @param roleHistory history to build
- * @return the number of records read
+ * @return the records read
* @throws IOException any problem
*/
- public int read(String resource, RoleHistory roleHistory) throws
+ public LoadedRoleHistory read(String resource) throws
IOException,
BadConfigException {
- return read(this.getClass().getClassLoader().getResourceAsStream(resource),
- roleHistory);
+ return read(this.getClass().getClassLoader().getResourceAsStream(resource));
}
@@ -366,28 +331,30 @@ public class RoleHistoryWriter {
/**
* Iterate through the paths until one can be loaded
- * @param roleHistory role history
* @param paths paths to load
- * @return the path of any loaded history -or null if all failed to load
+ * @return the loaded history including the path -or null if all failed to load
*/
- public Path attemptToReadHistory(RoleHistory roleHistory, FileSystem fileSystem, List<Path> paths)
+ public LoadedRoleHistory attemptToReadHistory(FileSystem fileSystem,
+ List<Path> paths)
throws BadConfigException {
ListIterator<Path> pathIterator = paths.listIterator();
boolean success = false;
Path path = null;
+ LoadedRoleHistory history = null;
while (!success && pathIterator.hasNext()) {
path = pathIterator.next();
try {
- read(fileSystem, path, roleHistory);
+ history = read(fileSystem, path);
//success
success = true;
+ history.setPath(path);
} catch (IOException e) {
log.info("Failed to read {}", path, e);
} catch (AvroTypeException e) {
log.warn("Failed to parse {}", path, e);
}
}
- return success ? path : null;
+ return history;
}
/**
@@ -395,16 +362,14 @@ public class RoleHistoryWriter {
* file is downgraded to a log and the next older path attempted instead
* @param fs filesystem
* @param dir dir to load from
- * @param roleHistory role history to build up
- * @return the path loaded
+ * @return the history loaded, including the path
* @throws IOException if indexing the history directory fails.
*/
- public Path loadFromHistoryDir(FileSystem fs, Path dir,
- RoleHistory roleHistory)
+ public LoadedRoleHistory loadFromHistoryDir(FileSystem fs, Path dir)
throws IOException, BadConfigException {
assert fs != null: "null filesystem";
List<Path> entries = findAllHistoryEntries(fs, dir, false);
- return attemptToReadHistory(roleHistory, fs, entries);
+ return attemptToReadHistory(fs, entries);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f10bef8a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
index b4f3978..015c6a3 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
@@ -23,6 +23,7 @@ import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.slider.api.ResourceKeys
+import org.apache.slider.common.SliderExitCodes
import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.core.exceptions.BadConfigException
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
@@ -30,6 +31,7 @@ import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector
+import org.apache.slider.server.avro.LoadedRoleHistory
import org.apache.slider.server.avro.RoleHistoryWriter
import org.junit.Test
@@ -159,7 +161,8 @@ class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest
cd.components["HistorySaveFlexLoad"] = opts
appState.updateResourceDefinitions(cd.confTree);
createAndStartNodes();
- historyWriter.read(fs, history, appState.roleHistory)
+ def loadedRoleHistory = historyWriter.read(fs, history)
+ assert 0 == appState.roleHistory.rebuild(loadedRoleHistory)
}
@Test
@@ -191,12 +194,8 @@ class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest
null, null,
new MostRecentContainerReleaseSelector())
// on this read there won't be the right number of roles
- try {
- historyWriter.read(fs, history, appState.roleHistory)
- fail("expected an exception")
- } catch (IOException e) {
- assert e.toString().contains("Number of roles")
- }
+ def loadedRoleHistory = historyWriter.read(fs, history)
+ assert 0 == appState.roleHistory.rebuild(loadedRoleHistory)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f10bef8a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
index 83cf2c6..c81c686 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
@@ -22,20 +22,34 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.FSDataOutputStream
import org.apache.hadoop.fs.Path
+import org.apache.slider.providers.PlacementPolicy
+import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.state.NodeEntry
import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.RoleHistory
+import org.apache.slider.server.avro.LoadedRoleHistory
import org.apache.slider.server.avro.RoleHistoryWriter
import org.junit.Test
@Slf4j
-//@CompileStatic
+@CompileStatic
class TestRoleHistoryRW extends BaseMockAppStateTest {
static long time = System.currentTimeMillis();
-
+ public static final String HISTORY_V1_6_ROLE = "org/apache/slider/server/avro/history-v01-6-role.json"
+ public static final String HISTORY_V1_3_ROLE = "org/apache/slider/server/avro/history-v01-3-role.json"
+
+
+ static final ProviderRole PROVIDER_ROLE3 = new ProviderRole(
+ "role3",
+ 3,
+ PlacementPolicy.STRICT,
+ 3,
+ 3)
+
@Override
String getTestName() {
return "TestHistoryRW"
@@ -48,7 +62,7 @@ class TestRoleHistoryRW extends BaseMockAppStateTest {
Path history = roleHistory.saveHistory(time++)
assert fs.isFile(history)
RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- historyWriter.read(fs, history, roleHistory)
+ historyWriter.read(fs, history)
}
@Test
@@ -65,7 +79,10 @@ class TestRoleHistoryRW extends BaseMockAppStateTest {
RoleHistoryWriter historyWriter = new RoleHistoryWriter();
RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
- assert 0 < historyWriter.read(fs, history, rh2)
+
+ def loadedRoleHistory = historyWriter.read(fs, history)
+ assert 0 < loadedRoleHistory.size()
+ rh2.rebuild(loadedRoleHistory)
NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
assert ni2 != null
NodeEntry ne2 = ni2.get(0)
@@ -101,8 +118,9 @@ class TestRoleHistoryRW extends BaseMockAppStateTest {
log.info("testWriteReadActiveData in $history")
RoleHistoryWriter historyWriter = new RoleHistoryWriter();
RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
-
- assert 3 == historyWriter.read(fs, history, rh2)
+ def loadedRoleHistory = historyWriter.read(fs, history)
+ assert 3 == loadedRoleHistory.size()
+ rh2.rebuild(loadedRoleHistory)
rh2.dump()
assert rh2.clusterSize == 2;
@@ -256,16 +274,52 @@ class TestRoleHistoryRW extends BaseMockAppStateTest {
}
/**
- * Test that a v1 JSON file can be read. This is the one without any name
+ * Test that a v1 JSON file can be read. Here the number of roles
+ * matches the current state.
* @throws Throwable
*/
@Test
- public void testReloadDataV1() throws Throwable {
- String source = "org/apache/slider/server/avro/history-v01-4-role.json"
+ public void testReloadDataV1_3_role() throws Throwable {
+ String source = HISTORY_V1_3_ROLE
RoleHistoryWriter writer = new RoleHistoryWriter()
+
+ def loadedRoleHistory = writer.read(source)
+ assert 4 == loadedRoleHistory.size()
RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
- assert 4 == writer.read(source, roleHistory)
+ assert 0 == roleHistory.rebuild(loadedRoleHistory)
+ }
+
+ /**
+ * Test that a v1 JSON file can be read. Here more roles than expected
+ * @throws Throwable
+ */
+ @Test
+ public void testReloadDataV1_6_role() throws Throwable {
+ String source = HISTORY_V1_6_ROLE
+ RoleHistoryWriter writer = new RoleHistoryWriter()
+
+ def loadedRoleHistory = writer.read(source)
+ assert 6 == loadedRoleHistory.size()
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ assert 3 == roleHistory.rebuild(loadedRoleHistory)
+ }
+
+ /**
+ * Test that a v1 JSON file can be read. Here the number of roles
+ * is less than the current state.
+ * @throws Throwable
+ */
+ @Test
+ public void testReload_less_roles() throws Throwable {
+ String source = HISTORY_V1_3_ROLE
+ RoleHistoryWriter writer = new RoleHistoryWriter()
+ def loadedRoleHistory = writer.read(source)
+ assert 4 == loadedRoleHistory.size()
+ def expandedRoles = new ArrayList(MockFactory.ROLES)
+ expandedRoles << PROVIDER_ROLE3
+ RoleHistory roleHistory = new RoleHistory(expandedRoles)
+ assert 0 == roleHistory.rebuild(loadedRoleHistory)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f10bef8a/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-3-role.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-3-role.json b/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-3-role.json
new file mode 100644
index 0000000..ceab0a5
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-3-role.json
@@ -0,0 +1,6 @@
+{"entry":{"org.apache.slider.server.avro.RoleHistoryHeader":{"version":1,"saved":1415296260647,"savedx":"149863b1a27","savedate":"6 Nov 2014 17:51:00 GMT","roles":3}}}
+{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":1,"active":false,"last_used":0}}}
+{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":2,"active":false,"last_used":0}}}
+{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":0,"active":false,"last_used":0}}}
+{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.86","role":2,"active":true,"last_used":0}}}
+{"entry":{"org.apache.slider.server.avro.RoleHistoryFooter":{"count":4}}}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f10bef8a/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-4-role.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-4-role.json b/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-4-role.json
deleted file mode 100644
index 565e998..0000000
--- a/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-4-role.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{"entry":{"org.apache.slider.server.avro.RoleHistoryHeader":{"version":1,"saved":1415296260647,"savedx":"149863b1a27","savedate":"6 Nov 2014 17:51:00 GMT","roles":6}}}
-{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":1,"active":false,"last_used":0}}}
-{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":2,"active":false,"last_used":0}}}
-{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":3,"active":false,"last_used":0}}}
-{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":4,"active":true,"last_used":0}}}
-{"entry":{"org.apache.slider.server.avro.RoleHistoryFooter":{"count":4}}}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f10bef8a/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-6-role.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-6-role.json b/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-6-role.json
new file mode 100644
index 0000000..f1c53d5
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-6-role.json
@@ -0,0 +1,8 @@
+{"entry":{"org.apache.slider.server.avro.RoleHistoryHeader":{"version":1,"saved":1415296260647,"savedx":"149863b1a27","savedate":"6 Nov 2014 17:51:00 GMT","roles":6}}}
+{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":1,"active":false,"last_used":0}}}
+{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":2,"active":false,"last_used":0}}}
+{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":0,"active":false,"last_used":0}}}
+{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.86","role":4,"active":true,"last_used":0}}}
+{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.86","role":5,"active":true,"last_used":0}}}
+{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.86","role":6,"active":true,"last_used":0}}}
+{"entry":{"org.apache.slider.server.avro.RoleHistoryFooter":{"count":6}}}
\ No newline at end of file