You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/04/09 14:35:45 UTC
[1/7] incubator-slider git commit: SLIDER-600: better rolehistory r/w
Repository: incubator-slider
Updated Branches:
refs/heads/develop 8dc94a721 -> 963602489
SLIDER-600: better rolehistory r/w
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/fd44d1cf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/fd44d1cf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/fd44d1cf
Branch: refs/heads/develop
Commit: fd44d1cfef0357e4e4abedc2e9dd27cc3807b8cd
Parents: f5f837c
Author: Steve Loughran <st...@apache.org>
Authored: Thu Mar 26 18:01:42 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Mar 26 18:01:42 2015 +0000
----------------------------------------------------------------------
.../slider/server/avro/RoleHistoryRecord.avsc | 4 +++
.../server/appmaster/state/RoleHistory.java | 14 ++++++++++
.../slider/server/avro/RoleHistoryWriter.java | 28 +++++++++++++++-----
.../model/history/TestRoleHistoryRW.groovy | 13 +++++++++
.../slider/server/avro/history-v01-4-role.json | 6 +++++
5 files changed, 58 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fd44d1cf/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc
----------------------------------------------------------------------
diff --git a/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc b/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc
index 0380664..0e42e13 100644
--- a/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc
+++ b/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc
@@ -30,6 +30,10 @@
"type": "int"
},
{
+ "name": "rolename",
+ "type": "string"
+ },
+ {
"name": "active",
"type": "boolean"
},
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fd44d1cf/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index c2a741c..c0de492 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -166,6 +166,20 @@ public class RoleHistory {
}
/**
+ * Lookup a role by ID
+ * @param roleId role Id
+ * @return role or null if not found
+ */
+ public ProviderRole lookupRole(int roleId) {
+ for (ProviderRole role : providerRoles) {
+ if (role.id == roleId) {
+ return role;
+ }
+ }
+ return null;
+ }
+
+ /**
* Clear the lists of available nodes
*/
private synchronized void resetAvailableNodeLists() {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fd44d1cf/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java b/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
index 031632b..b938f30 100644
--- a/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
+++ b/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.PathFilter;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.providers.ProviderRole;
import org.apache.slider.server.appmaster.state.NodeEntry;
import org.apache.slider.server.appmaster.state.NodeInstance;
import org.apache.slider.server.appmaster.state.RoleHistory;
@@ -106,10 +107,12 @@ public class RoleHistoryWriter {
Collection<NodeInstance> instances = history.cloneNodemap().values();
for (NodeInstance instance : instances) {
for (int role = 0; role < roles; role++) {
+ ProviderRole providerRole = history.lookupRole(role);
+ String rolename = providerRole != null ? providerRole.name : "";
NodeEntry nodeEntry = instance.get(role);
if (nodeEntry != null) {
- NodeEntryRecord ner = build(nodeEntry, role, instance.hostname);
+ NodeEntryRecord ner = build(nodeEntry, role, "", instance.hostname);
record = new RoleHistoryRecord(ner);
writer.write(record, encoder);
count++;
@@ -121,7 +124,6 @@ public class RoleHistoryWriter {
footer.setCount(count);
writer.write(new RoleHistoryRecord(footer), encoder);
encoder.flush();
- out.close();
return count;
} finally {
out.close();
@@ -140,8 +142,12 @@ public class RoleHistoryWriter {
* @return no of records written
* @throws IOException IO failures
*/
- public long write(FileSystem fs, Path path, boolean overwrite,
- RoleHistory history, long savetime) throws IOException {
+ public long write(FileSystem fs,
+ Path path,
+ boolean overwrite,
+ RoleHistory history,
+ long savetime)
+ throws IOException {
FSDataOutputStream out = fs.create(path, overwrite);
return write(out, history, savetime);
}
@@ -159,10 +165,18 @@ public class RoleHistoryWriter {
Path path = new Path(historyPath, filename);
return path;
}
-
- private NodeEntryRecord build(NodeEntry entry, int role, String hostname) {
+
+ /**
+ * Build a {@link NodeEntryRecord} from a node entry; include whether
+ * the node is in use and when it was last used.
+ * @param entry entry count
+ * @param role role index
+ * @param rolename
+ *@param hostname name @return
+ */
+ private NodeEntryRecord build(NodeEntry entry, int role, String rolename, String hostname) {
NodeEntryRecord record = new NodeEntryRecord(
- hostname, role, entry.getLive() > 0, entry.getLastUsed()
+ hostname, role, rolename, entry.getLive() > 0, entry.getLastUsed()
);
return record;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fd44d1cf/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
index 5575076..83cf2c6 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
@@ -255,4 +255,17 @@ class TestRoleHistoryRW extends BaseMockAppStateTest {
assert fs.exists(badfile )
}
+ /**
+ * Test that a v1 JSON file can be read. This is the one without any name
+ * @throws Throwable
+ */
+ @Test
+ public void testReloadDataV1() throws Throwable {
+ String source = "org/apache/slider/server/avro/history-v01-4-role.json"
+ RoleHistoryWriter writer = new RoleHistoryWriter()
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ assert 4 == writer.read(source, roleHistory)
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fd44d1cf/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-4-role.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-4-role.json b/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-4-role.json
new file mode 100644
index 0000000..565e998
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-4-role.json
@@ -0,0 +1,6 @@
+{"entry":{"org.apache.slider.server.avro.RoleHistoryHeader":{"version":1,"saved":1415296260647,"savedx":"149863b1a27","savedate":"6 Nov 2014 17:51:00 GMT","roles":6}}}
+{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":1,"active":false,"last_used":0}}}
+{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":2,"active":false,"last_used":0}}}
+{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":3,"active":false,"last_used":0}}}
+{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":4,"active":true,"last_used":0}}}
+{"entry":{"org.apache.slider.server.avro.RoleHistoryFooter":{"count":4}}}
\ No newline at end of file
[7/7] incubator-slider git commit: Merge branch
'feature/SLIDER-600-robust-history-reader' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-600-robust-history-reader' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/96360248
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/96360248
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/96360248
Branch: refs/heads/develop
Commit: 963602489dfeaebf01712d343da3b94d3aaa1fa7
Parents: 8dc94a7 c425524
Author: Steve Loughran <st...@apache.org>
Authored: Thu Apr 9 13:35:28 2015 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Apr 9 13:35:28 2015 +0100
----------------------------------------------------------------------
.../slider/server/avro/RoleHistoryRecord.avsc | 17 +-
.../server/appmaster/state/RoleHistory.java | 94 ++++++++--
.../slider/server/avro/LoadedRoleHistory.java | 81 ++++++++
.../slider/server/avro/RoleHistoryWriter.java | 183 ++++++++++---------
.../TestMockAppStateFlexDynamicRoles.groovy | 13 +-
.../model/history/TestRoleHistoryRW.groovy | 79 +++++++-
.../slider/server/avro/history-v01-3-role.json | 6 +
.../slider/server/avro/history-v01-6-role.json | 8 +
8 files changed, 370 insertions(+), 111 deletions(-)
----------------------------------------------------------------------
[2/7] incubator-slider git commit: Merge branch 'develop' into
feature/SLIDER-600-robust-history-reader
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-600-robust-history-reader
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0ac74d5c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0ac74d5c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0ac74d5c
Branch: refs/heads/develop
Commit: 0ac74d5cda17fcc6712dfe8cb723426e596869f7
Parents: fd44d1c 566d120
Author: Steve Loughran <st...@apache.org>
Authored: Tue Apr 7 11:43:06 2015 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Apr 7 11:43:06 2015 +0100
----------------------------------------------------------------------
app-packages/accumulo/appConfig-default.json | 2 +
.../accumulo/appConfig-secured-default.json | 3 +-
.../accumulo/appConfig-ssl-default.json | 2 +
.../accumulo/configuration/accumulo-env.xml | 15 +
.../accumulo/package/files/accumulo-slider.py | 18 +-
.../accumulo/package/scripts/accumulo_client.py | 47 +-
.../accumulo/package/scripts/client_params.py | 3 +-
app-packages/accumulo/package/scripts/params.py | 3 +-
.../funtest/accumulo/AccumuloBasicIT.groovy | 10 +-
.../funtest/accumulo/AccumuloScriptIT.groovy | 2 -
.../app-pkg-template/appConfig-default.json | 1 +
.../app-pkg-template/package/scripts/params.py | 1 +
.../command-logger/slider-pkg/appConfig.json | 1 +
.../slider-pkg/package/scripts/params.py | 1 +
app-packages/hbase-win/appConfig-default.json | 1 +
.../hbase-win/package/scripts/params.py | 1 +
app-packages/hbase/appConfig-default.json | 1 +
.../hbase/appConfig-phoenix-default.json | 1 +
.../hbase/appConfig-secured-default.json | 1 +
app-packages/hbase/hbase-slider.py | 28 +-
app-packages/hbase/package/scripts/params.py | 1 +
.../test/resources/appConfig_monitor_ssl.json | 1 +
.../memcached-win/appConfig-default.json | 1 +
.../memcached-win/package/scripts/params.py | 1 +
app-packages/memcached/appConfig-default.json | 1 +
.../memcached/package/scripts/params.py | 1 +
app-packages/storm-win/appConfig-default.json | 1 +
.../storm-win/package/scripts/params.py | 1 +
app-packages/storm/appConfig-default.json | 1 +
.../storm/appConfig-secured-default.json | 1 +
app-packages/storm/package/scripts/params.py | 1 +
pom.xml | 18 +-
.../org/apache/slider/api/proto/Messages.java | 435 ++++++++++++++++---
.../slider/api/proto/RestTypeMarshalling.java | 4 +
.../slider/api/types/ContainerInformation.java | 1 +
.../org/apache/slider/client/SliderClient.java | 112 +++--
.../org/apache/slider/common/SliderKeys.java | 5 +
.../slider/common/params/ActionPackageArgs.java | 7 +
.../apache/slider/common/params/Arguments.java | 1 +
.../slider/common/tools/CoreFileSystem.java | 10 +-
.../apache/slider/common/tools/SliderUtils.java | 24 +-
.../apache/slider/core/conf/AggregateConf.java | 13 +
.../core/registry/docstore/ConfigFormat.java | 1 +
.../PublishedConfigurationOutputter.java | 29 +-
.../providers/agent/AgentClientProvider.java | 16 +-
.../providers/agent/AgentProviderService.java | 8 +-
.../server/appmaster/RoleLaunchService.java | 3 +
.../server/appmaster/SliderAppMaster.java | 2 +-
.../server/appmaster/state/RoleInstance.java | 9 +
.../src/main/proto/SliderClusterMessages.proto | 4 +-
.../actions/TestActionInstallPackage.groovy | 5 +-
.../slider/agent/actions/TestActionList.groovy | 104 +++++
.../agent/actions/TestActionPackage.groovy | 7 +-
.../standalone/TestBuildStandaloneAM.groovy | 3 +-
.../client/TestPackageCommandOptions.groovy | 250 +++++++++++
.../registry/TestRegistryRestMarshalling.groovy | 1 -
.../apache/slider/test/SliderTestUtils.groovy | 12 +-
.../agent/TestAgentClientProvider2.java | 47 +-
slider-core/src/test/resources/log4j.properties | 1 +
59 files changed, 1093 insertions(+), 192 deletions(-)
----------------------------------------------------------------------
[6/7] incubator-slider git commit: SLIDER-600 javadocs and IDE hinted
warnings in RoleHistoryWriter
Posted by st...@apache.org.
SLIDER-600 javadocs and IDE hinted warnings in RoleHistoryWriter
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c4255245
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c4255245
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c4255245
Branch: refs/heads/develop
Commit: c4255245d27ab2a1043f84e304c16bb78bb97b32
Parents: 9dfb52d
Author: Steve Loughran <st...@apache.org>
Authored: Thu Apr 9 13:34:48 2015 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Apr 9 13:34:48 2015 +0100
----------------------------------------------------------------------
.../slider/server/avro/RoleHistoryWriter.java | 23 ++++++++------------
1 file changed, 9 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c4255245/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java b/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
index a6ab679..b76f228 100644
--- a/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
+++ b/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
@@ -139,9 +139,9 @@ public class RoleHistoryWriter {
}
/**
- * Create the role record
+ * Create the rolemap record
* @param history history
- * @return a record to place at the head of the file
+ * @return a record to insert into the file
*/
private RoleHistoryRecord createRolemapRecord(RoleHistory history) {
RoleHistoryMapping entry = new RoleHistoryMapping();
@@ -151,8 +151,7 @@ public class RoleHistoryWriter {
}
/**
- * Write write the file
- *
+ * Write the history information to a file
*
* @param fs filesystem
* @param path path
@@ -191,8 +190,8 @@ public class RoleHistoryWriter {
* the node is in use and when it was last used.
* @param entry entry count
* @param role role index
- * @param rolename
- *@param hostname name @return
+ * @param hostname name
+ * @return the record
*/
private NodeEntryRecord build(NodeEntry entry, int role, String hostname) {
NodeEntryRecord record = new NodeEntryRecord(
@@ -217,7 +216,7 @@ public class RoleHistoryWriter {
new SpecificDatumReader<>(RoleHistoryRecord.class);
Decoder decoder =
DecoderFactory.get().jsonDecoder(RoleHistoryRecord.getClassSchema(),
- in);
+ in);
//read header : no entry -> EOF
RoleHistoryRecord record = reader.read(null, decoder);
@@ -226,7 +225,6 @@ public class RoleHistoryWriter {
throw new IOException("Role History Header not found at start of file");
}
RoleHistoryHeader header = (RoleHistoryHeader) entry;
- Long saved = header.getSaved();
if (header.getVersion() != ROLE_HISTORY_VERSION) {
throw new IOException(
String.format("Can't read role file version %04x -need %04x",
@@ -308,13 +306,11 @@ public class RoleHistoryWriter {
/**
* Read from a resource in the classpath -used for testing
* @param resource resource
- * @param roleHistory history to build
* @return the records read
* @throws IOException any problem
*/
- public LoadedRoleHistory read(String resource) throws
- IOException,
- BadConfigException {
+ public LoadedRoleHistory read(String resource)
+ throws IOException, BadConfigException {
return read(this.getClass().getClassLoader().getResourceAsStream(resource));
}
@@ -377,10 +373,9 @@ public class RoleHistoryWriter {
throws BadConfigException {
ListIterator<Path> pathIterator = paths.listIterator();
boolean success = false;
- Path path = null;
LoadedRoleHistory history = null;
while (!success && pathIterator.hasNext()) {
- path = pathIterator.next();
+ Path path = pathIterator.next();
try {
history = read(fileSystem, path);
//success
[3/7] incubator-slider git commit: SLIDER-600: history entries of
unknown roles are now discarded
Posted by st...@apache.org.
SLIDER-600: history entries of unknown roles are now discarded
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f10bef8a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f10bef8a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f10bef8a
Branch: refs/heads/develop
Commit: f10bef8ad7bf88e47ab4ff987ca21d42fa42029e
Parents: 0ac74d5
Author: Steve Loughran <st...@apache.org>
Authored: Thu Apr 9 11:38:38 2015 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Apr 9 11:38:38 2015 +0100
----------------------------------------------------------------------
.../slider/server/avro/RoleHistoryRecord.avsc | 21 +++--
.../server/appmaster/state/RoleHistory.java | 68 ++++++++++++---
.../slider/server/avro/LoadedRoleHistory.java | 63 ++++++++++++++
.../slider/server/avro/RoleHistoryWriter.java | 87 ++++++--------------
.../TestMockAppStateFlexDynamicRoles.groovy | 13 ++-
.../model/history/TestRoleHistoryRW.groovy | 74 ++++++++++++++---
.../slider/server/avro/history-v01-3-role.json | 6 ++
.../slider/server/avro/history-v01-4-role.json | 6 --
.../slider/server/avro/history-v01-6-role.json | 8 ++
9 files changed, 244 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f10bef8a/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc
----------------------------------------------------------------------
diff --git a/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc b/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc
index 0e42e13..778a728 100644
--- a/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc
+++ b/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc
@@ -30,10 +30,6 @@
"type": "int"
},
{
- "name": "rolename",
- "type": "string"
- },
- {
"name": "active",
"type": "boolean"
},
@@ -74,6 +70,20 @@
},
{
"type": "record",
+ "name": "RoleHistoryMapping",
+ "namespace": "org.apache.slider.server.avro",
+ "fields": [
+ {
+ "name": "rolemap",
+ "type": {
+ "type": "map",
+ "values": "long"
+ }
+ }
+ ]
+ },
+ {
+ "type": "record",
"name": "RoleHistoryFooter",
"namespace": "org.apache.slider.server.avro",
"fields": [
@@ -94,7 +104,8 @@
"type": [
"org.apache.slider.server.avro.NodeEntryRecord",
"org.apache.slider.server.avro.RoleHistoryHeader",
- "org.apache.slider.server.avro.RoleHistoryFooter"
+ "org.apache.slider.server.avro.RoleHistoryFooter",
+ "org.apache.slider.server.avro.RoleHistoryMapping"
]
}
]
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f10bef8a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index c0de492..e95e2a6 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -29,6 +29,8 @@ import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
+import org.apache.slider.server.avro.LoadedRoleHistory;
+import org.apache.slider.server.avro.NodeEntryRecord;
import org.apache.slider.server.avro.RoleHistoryHeader;
import org.apache.slider.server.avro.RoleHistoryWriter;
import org.slf4j.Logger;
@@ -189,23 +191,61 @@ public class RoleHistory {
/**
* Reset the variables -this does not adjust the fixed attributes
* of the history.
+ * <p>
* This intended for use by the RoleWriter logic.
+ * @throws BadConfigException if there is a problem rebuilding the state
*/
- public synchronized void prepareForReading(RoleHistoryHeader header) throws
- IOException,
- BadConfigException {
+ private void prepareForReading(RoleHistoryHeader header)
+ throws BadConfigException {
reset();
int roleCountInSource = header.getRoles();
if (roleCountInSource != roleSize) {
- throw new IOException("Number of roles in source " + roleCountInSource
- + " does not match the expected number of " +
- roleSize);
+ log.warn("Number of roles in source {}"
+ +" does not match the expected number of {}",
+ roleCountInSource,
+ roleSize);
}
//record when the data was loaded
setThawedDataTime(header.getSaved());
}
-
+
+ /**
+ * rebuild the placement history from the loaded role history
+ * @param loadedRoleHistory loaded history
+ * @return the number of entries discarded
+ * @throws BadConfigException if there is a problem rebuilding the state
+ */
+ @VisibleForTesting
+ public synchronized int rebuild(LoadedRoleHistory loadedRoleHistory) throws BadConfigException {
+ RoleHistoryHeader header = loadedRoleHistory.getHeader();
+ prepareForReading(header);
+ int discarded = 0;
+ Long saved = header.getSaved();
+ for (NodeEntryRecord nodeEntryRecord : loadedRoleHistory.records) {
+ Integer roleId = nodeEntryRecord.getRole();
+ NodeEntry nodeEntry = new NodeEntry(roleId);
+ nodeEntry.setLastUsed(nodeEntryRecord.getLastUsed());
+ if (nodeEntryRecord.getActive()) {
+ //if active at the time of save, make the last used time the save time
+ nodeEntry.setLastUsed(saved);
+ }
+ String hostname = SliderUtils.sequenceToString(nodeEntryRecord.getHost());
+ ProviderRole providerRole = lookupRole(roleId);
+ if (providerRole == null) {
+ // discarding entry
+ log.info("Discarding history entry with unknown role: {} on host {}",
+ roleId, hostname);
+ discarded ++;
+ } else {
+ NodeInstance instance = getOrCreateNodeInstance(hostname);
+ instance.set(roleId, nodeEntry);
+ }
+ }
+ return discarded;
+ }
+
+
public synchronized long getStartTime() {
return startTime;
}
@@ -396,24 +436,26 @@ public class RoleHistory {
* Handle the start process <i>after the history has been rebuilt</i>,
* and after any gc/purge
*/
- @VisibleForTesting
public synchronized boolean onThaw() throws BadConfigException {
assert filesystem != null;
assert historyPath != null;
boolean thawSuccessful = false;
//load in files from data dir
- Path loaded = null;
+
+ LoadedRoleHistory loadedRoleHistory = null;
try {
- loaded = historyWriter.loadFromHistoryDir(filesystem, historyPath, this);
+ loadedRoleHistory = historyWriter.loadFromHistoryDir(filesystem, historyPath);
} catch (IOException e) {
log.warn("Exception trying to load history from {}", historyPath, e);
}
- if (loaded != null) {
+ if (loadedRoleHistory != null) {
+ rebuild(loadedRoleHistory);
thawSuccessful = true;
- log.debug("loaded history from {}", loaded);
+ Path loadPath = loadedRoleHistory.getPath();;
+ log.debug("loaded history from {}", loadPath);
// delete any old entries
try {
- int count = historyWriter.purgeOlderHistoryEntries(filesystem, loaded);
+ int count = historyWriter.purgeOlderHistoryEntries(filesystem, loadPath);
log.debug("Deleted {} old history entries", count);
} catch (IOException e) {
log.info("Ignoring exception raised while trying to delete old entries",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f10bef8a/slider-core/src/main/java/org/apache/slider/server/avro/LoadedRoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/avro/LoadedRoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/avro/LoadedRoleHistory.java
new file mode 100644
index 0000000..866e716
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/avro/LoadedRoleHistory.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.avro;
+
+import org.apache.hadoop.fs.Path;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * The role history
+ */
+public class LoadedRoleHistory {
+
+ private RoleHistoryHeader header;
+
+ private Path path;
+
+ public final Map<String, Integer> roleMap = new HashMap<>();
+
+ public final List<NodeEntryRecord> records = new ArrayList<>();
+
+ public void add(NodeEntryRecord record) {
+ records.add(record);
+ }
+
+ public int size() {
+ return records.size();
+ }
+ public RoleHistoryHeader getHeader() {
+ return header;
+ }
+
+ public void setHeader(RoleHistoryHeader header) {
+ this.header = header;
+ }
+
+ public Path getPath() {
+ return path;
+ }
+
+ public void setPath(Path path) {
+ this.path = path;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f10bef8a/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java b/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
index b938f30..b638ae9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
+++ b/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
@@ -47,8 +47,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.EOFException;
-import java.io.File;
-import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
@@ -61,7 +59,7 @@ import java.util.ListIterator;
import java.util.Locale;
/**
- * Write out the role history to an output stream
+ * Write out the role history to an output stream.
*/
public class RoleHistoryWriter {
protected static final Logger log =
@@ -107,12 +105,10 @@ public class RoleHistoryWriter {
Collection<NodeInstance> instances = history.cloneNodemap().values();
for (NodeInstance instance : instances) {
for (int role = 0; role < roles; role++) {
- ProviderRole providerRole = history.lookupRole(role);
- String rolename = providerRole != null ? providerRole.name : "";
NodeEntry nodeEntry = instance.get(role);
if (nodeEntry != null) {
- NodeEntryRecord ner = build(nodeEntry, role, "", instance.hostname);
+ NodeEntryRecord ner = build(nodeEntry, role, instance.hostname);
record = new RoleHistoryRecord(ner);
writer.write(record, encoder);
count++;
@@ -174,9 +170,9 @@ public class RoleHistoryWriter {
* @param rolename
*@param hostname name @return
*/
- private NodeEntryRecord build(NodeEntry entry, int role, String rolename, String hostname) {
+ private NodeEntryRecord build(NodeEntry entry, int role, String hostname) {
NodeEntryRecord record = new NodeEntryRecord(
- hostname, role, rolename, entry.getLive() > 0, entry.getLastUsed()
+ hostname, role, entry.getLive() > 0, entry.getLastUsed()
);
return record;
}
@@ -185,16 +181,14 @@ public class RoleHistoryWriter {
* Read a history, returning one that is ready to have its onThaw()
* method called
* @param in input source
- * @param history a history set up with the expected roles;
- * this will be built up with a node map configured with the node instances
- * and entries loaded from the source
* @return no. of entries read
* @throws IOException problems
*/
- public int read(InputStream in, RoleHistory history) throws
+ public LoadedRoleHistory read(InputStream in) throws
IOException,
BadConfigException {
try {
+ LoadedRoleHistory loadedRoleHistory = new LoadedRoleHistory();
DatumReader<RoleHistoryRecord> reader =
new SpecificDatumReader<>(RoleHistoryRecord.class);
Decoder decoder =
@@ -215,7 +209,7 @@ public class RoleHistoryWriter {
header.getVersion(),
ROLE_HISTORY_VERSION));
}
- history.prepareForReading(header);
+ loadedRoleHistory.setHeader(header);
RoleHistoryFooter footer;
int records = 0;
//go through reading data
@@ -234,18 +228,7 @@ public class RoleHistoryWriter {
}
records++;
NodeEntryRecord nodeEntryRecord = (NodeEntryRecord) entry;
- Integer roleId = nodeEntryRecord.getRole();
- NodeEntry nodeEntry = new NodeEntry(roleId);
- nodeEntry.setLastUsed(nodeEntryRecord.getLastUsed());
- if (nodeEntryRecord.getActive()) {
- //if active at the time of save, make the last used time the save time
- nodeEntry.setLastUsed(saved);
- }
-
- String hostname =
- SliderUtils.sequenceToString(nodeEntryRecord.getHost());
- NodeInstance instance = history.getOrCreateNodeInstance(hostname);
- instance.set(roleId, nodeEntry);
+ loadedRoleHistory.add(nodeEntryRecord);
}
} catch (EOFException e) {
EOFException ex = new EOFException(
@@ -264,7 +247,7 @@ public class RoleHistoryWriter {
log.warn("mismatch between no of records saved {} and number read {}",
footer.getCount(), records);
}
- return records;
+ return loadedRoleHistory;
} finally {
in.close();
}
@@ -275,45 +258,27 @@ public class RoleHistoryWriter {
* Read a role history from a path in a filesystem
* @param fs filesystem
* @param path path to the file
- * @param roleHistory history to build
- * @return the number of records read
+ * @return the records read
* @throws IOException any problem
*/
- public int read(FileSystem fs, Path path, RoleHistory roleHistory) throws
- IOException,
- BadConfigException {
+ public LoadedRoleHistory read(FileSystem fs, Path path)
+ throws IOException, BadConfigException {
FSDataInputStream instream = fs.open(path);
- return read(instream, roleHistory);
- }
-
- /**
- * Read a role history from local file
- * @param file path to the file
- * @param roleHistory history to build
- * @return the number of records read
- * @throws IOException any problem
- */
- public int read(File file, RoleHistory roleHistory) throws
- IOException,
- BadConfigException {
-
-
- return read(new FileInputStream(file), roleHistory);
+ return read(instream);
}
/**
* Read from a resource in the classpath -used for testing
* @param resource resource
* @param roleHistory history to build
- * @return the number of records read
+ * @return the records read
* @throws IOException any problem
*/
- public int read(String resource, RoleHistory roleHistory) throws
+ public LoadedRoleHistory read(String resource) throws
IOException,
BadConfigException {
- return read(this.getClass().getClassLoader().getResourceAsStream(resource),
- roleHistory);
+ return read(this.getClass().getClassLoader().getResourceAsStream(resource));
}
@@ -366,28 +331,30 @@ public class RoleHistoryWriter {
/**
* Iterate through the paths until one can be loaded
- * @param roleHistory role history
* @param paths paths to load
- * @return the path of any loaded history -or null if all failed to load
+ * @return the loaded history including the path -or null if all failed to load
*/
- public Path attemptToReadHistory(RoleHistory roleHistory, FileSystem fileSystem, List<Path> paths)
+ public LoadedRoleHistory attemptToReadHistory(FileSystem fileSystem,
+ List<Path> paths)
throws BadConfigException {
ListIterator<Path> pathIterator = paths.listIterator();
boolean success = false;
Path path = null;
+ LoadedRoleHistory history = null;
while (!success && pathIterator.hasNext()) {
path = pathIterator.next();
try {
- read(fileSystem, path, roleHistory);
+ history = read(fileSystem, path);
//success
success = true;
+ history.setPath(path);
} catch (IOException e) {
log.info("Failed to read {}", path, e);
} catch (AvroTypeException e) {
log.warn("Failed to parse {}", path, e);
}
}
- return success ? path : null;
+ return history;
}
/**
@@ -395,16 +362,14 @@ public class RoleHistoryWriter {
* file is downgraded to a log and the next older path attempted instead
* @param fs filesystem
* @param dir dir to load from
- * @param roleHistory role history to build up
- * @return the path loaded
+ * @return the history loaded, including the path
* @throws IOException if indexing the history directory fails.
*/
- public Path loadFromHistoryDir(FileSystem fs, Path dir,
- RoleHistory roleHistory)
+ public LoadedRoleHistory loadFromHistoryDir(FileSystem fs, Path dir)
throws IOException, BadConfigException {
assert fs != null: "null filesystem";
List<Path> entries = findAllHistoryEntries(fs, dir, false);
- return attemptToReadHistory(roleHistory, fs, entries);
+ return attemptToReadHistory(fs, entries);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f10bef8a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
index b4f3978..015c6a3 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
@@ -23,6 +23,7 @@ import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.slider.api.ResourceKeys
+import org.apache.slider.common.SliderExitCodes
import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.core.exceptions.BadConfigException
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
@@ -30,6 +31,7 @@ import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector
+import org.apache.slider.server.avro.LoadedRoleHistory
import org.apache.slider.server.avro.RoleHistoryWriter
import org.junit.Test
@@ -159,7 +161,8 @@ class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest
cd.components["HistorySaveFlexLoad"] = opts
appState.updateResourceDefinitions(cd.confTree);
createAndStartNodes();
- historyWriter.read(fs, history, appState.roleHistory)
+ def loadedRoleHistory = historyWriter.read(fs, history)
+ assert 0 == appState.roleHistory.rebuild(loadedRoleHistory)
}
@Test
@@ -191,12 +194,8 @@ class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest
null, null,
new MostRecentContainerReleaseSelector())
// on this read there won't be the right number of roles
- try {
- historyWriter.read(fs, history, appState.roleHistory)
- fail("expected an exception")
- } catch (IOException e) {
- assert e.toString().contains("Number of roles")
- }
+ def loadedRoleHistory = historyWriter.read(fs, history)
+ assert 0 == appState.roleHistory.rebuild(loadedRoleHistory)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f10bef8a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
index 83cf2c6..c81c686 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
@@ -22,20 +22,34 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.FSDataOutputStream
import org.apache.hadoop.fs.Path
+import org.apache.slider.providers.PlacementPolicy
+import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.state.NodeEntry
import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.RoleHistory
+import org.apache.slider.server.avro.LoadedRoleHistory
import org.apache.slider.server.avro.RoleHistoryWriter
import org.junit.Test
@Slf4j
-//@CompileStatic
+@CompileStatic
class TestRoleHistoryRW extends BaseMockAppStateTest {
static long time = System.currentTimeMillis();
-
+ public static final String HISTORY_V1_6_ROLE = "org/apache/slider/server/avro/history-v01-6-role.json"
+ public static final String HISTORY_V1_3_ROLE = "org/apache/slider/server/avro/history-v01-3-role.json"
+
+
+ static final ProviderRole PROVIDER_ROLE3 = new ProviderRole(
+ "role3",
+ 3,
+ PlacementPolicy.STRICT,
+ 3,
+ 3)
+
@Override
String getTestName() {
return "TestHistoryRW"
@@ -48,7 +62,7 @@ class TestRoleHistoryRW extends BaseMockAppStateTest {
Path history = roleHistory.saveHistory(time++)
assert fs.isFile(history)
RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- historyWriter.read(fs, history, roleHistory)
+ historyWriter.read(fs, history)
}
@Test
@@ -65,7 +79,10 @@ class TestRoleHistoryRW extends BaseMockAppStateTest {
RoleHistoryWriter historyWriter = new RoleHistoryWriter();
RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
- assert 0 < historyWriter.read(fs, history, rh2)
+
+ def loadedRoleHistory = historyWriter.read(fs, history)
+ assert 0 < loadedRoleHistory.size()
+ rh2.rebuild(loadedRoleHistory)
NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
assert ni2 != null
NodeEntry ne2 = ni2.get(0)
@@ -101,8 +118,9 @@ class TestRoleHistoryRW extends BaseMockAppStateTest {
log.info("testWriteReadActiveData in $history")
RoleHistoryWriter historyWriter = new RoleHistoryWriter();
RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
-
- assert 3 == historyWriter.read(fs, history, rh2)
+ def loadedRoleHistory = historyWriter.read(fs, history)
+ assert 3 == loadedRoleHistory.size()
+ rh2.rebuild(loadedRoleHistory)
rh2.dump()
assert rh2.clusterSize == 2;
@@ -256,16 +274,52 @@ class TestRoleHistoryRW extends BaseMockAppStateTest {
}
/**
- * Test that a v1 JSON file can be read. This is the one without any name
+ * Test that a v1 JSON file can be read. Here the number of roles
+ * matches the current state.
* @throws Throwable
*/
@Test
- public void testReloadDataV1() throws Throwable {
- String source = "org/apache/slider/server/avro/history-v01-4-role.json"
+ public void testReloadDataV1_3_role() throws Throwable {
+ String source = HISTORY_V1_3_ROLE
RoleHistoryWriter writer = new RoleHistoryWriter()
+
+ def loadedRoleHistory = writer.read(source)
+ assert 4 == loadedRoleHistory.size()
RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
- assert 4 == writer.read(source, roleHistory)
+ assert 0 == roleHistory.rebuild(loadedRoleHistory)
+ }
+
+ /**
+ * Test that a v1 JSON file can be read. Here more roles than expected
+ * @throws Throwable
+ */
+ @Test
+ public void testReloadDataV1_6_role() throws Throwable {
+ String source = HISTORY_V1_6_ROLE
+ RoleHistoryWriter writer = new RoleHistoryWriter()
+
+ def loadedRoleHistory = writer.read(source)
+ assert 6 == loadedRoleHistory.size()
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ assert 3 == roleHistory.rebuild(loadedRoleHistory)
+ }
+
+ /**
+ * Test that a v1 JSON file can be read. Here the number of roles
+ * is less than the current state.
+ * @throws Throwable
+ */
+ @Test
+ public void testReload_less_roles() throws Throwable {
+ String source = HISTORY_V1_3_ROLE
+ RoleHistoryWriter writer = new RoleHistoryWriter()
+ def loadedRoleHistory = writer.read(source)
+ assert 4 == loadedRoleHistory.size()
+ def expandedRoles = new ArrayList(MockFactory.ROLES)
+ expandedRoles << PROVIDER_ROLE3
+ RoleHistory roleHistory = new RoleHistory(expandedRoles)
+ assert 0 == roleHistory.rebuild(loadedRoleHistory)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f10bef8a/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-3-role.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-3-role.json b/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-3-role.json
new file mode 100644
index 0000000..ceab0a5
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-3-role.json
@@ -0,0 +1,6 @@
+{"entry":{"org.apache.slider.server.avro.RoleHistoryHeader":{"version":1,"saved":1415296260647,"savedx":"149863b1a27","savedate":"6 Nov 2014 17:51:00 GMT","roles":3}}}
+{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":1,"active":false,"last_used":0}}}
+{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":2,"active":false,"last_used":0}}}
+{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":0,"active":false,"last_used":0}}}
+{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.86","role":2,"active":true,"last_used":0}}}
+{"entry":{"org.apache.slider.server.avro.RoleHistoryFooter":{"count":4}}}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f10bef8a/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-4-role.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-4-role.json b/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-4-role.json
deleted file mode 100644
index 565e998..0000000
--- a/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-4-role.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{"entry":{"org.apache.slider.server.avro.RoleHistoryHeader":{"version":1,"saved":1415296260647,"savedx":"149863b1a27","savedate":"6 Nov 2014 17:51:00 GMT","roles":6}}}
-{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":1,"active":false,"last_used":0}}}
-{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":2,"active":false,"last_used":0}}}
-{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":3,"active":false,"last_used":0}}}
-{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":4,"active":true,"last_used":0}}}
-{"entry":{"org.apache.slider.server.avro.RoleHistoryFooter":{"count":4}}}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f10bef8a/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-6-role.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-6-role.json b/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-6-role.json
new file mode 100644
index 0000000..f1c53d5
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-6-role.json
@@ -0,0 +1,8 @@
+{"entry":{"org.apache.slider.server.avro.RoleHistoryHeader":{"version":1,"saved":1415296260647,"savedx":"149863b1a27","savedate":"6 Nov 2014 17:51:00 GMT","roles":6}}}
+{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":1,"active":false,"last_used":0}}}
+{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":2,"active":false,"last_used":0}}}
+{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":0,"active":false,"last_used":0}}}
+{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.86","role":4,"active":true,"last_used":0}}}
+{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.86","role":5,"active":true,"last_used":0}}}
+{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.86","role":6,"active":true,"last_used":0}}}
+{"entry":{"org.apache.slider.server.avro.RoleHistoryFooter":{"count":6}}}
\ No newline at end of file
[4/7] incubator-slider git commit: SLIDER-600: reread map information
from history;
currently no use is made of it. History reader also now ignores any extra
records in the history file, to make it resilient to change, especially
version-rollback
Posted by st...@apache.org.
SLIDER-600: reread map information from history; currently no use is made of it. History reader also now ignores any extra records in the history file, to make it resilient to change, especially version-rollback
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b2f060b0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b2f060b0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b2f060b0
Branch: refs/heads/develop
Commit: b2f060b083f048f6e7ab191701425899bace14c1
Parents: f10bef8
Author: Steve Loughran <st...@apache.org>
Authored: Thu Apr 9 13:15:17 2015 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Apr 9 13:15:17 2015 +0100
----------------------------------------------------------------------
.../slider/server/avro/RoleHistoryRecord.avsc | 4 +-
.../server/appmaster/state/RoleHistory.java | 12 ++++
.../slider/server/avro/LoadedRoleHistory.java | 18 +++++
.../slider/server/avro/RoleHistoryWriter.java | 72 +++++++++++++++-----
4 files changed, 87 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2f060b0/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc
----------------------------------------------------------------------
diff --git a/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc b/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc
index 778a728..3667c01 100644
--- a/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc
+++ b/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc
@@ -77,8 +77,8 @@
"name": "rolemap",
"type": {
"type": "map",
- "values": "long"
- }
+ "values": "int"
+ }
}
]
},
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2f060b0/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index e95e2a6..bb482af 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -895,6 +895,18 @@ public class RoleHistory {
}
/**
+ * Build the mapping entry for persisting to the role history
+ * @return a mapping object
+ */
+ public synchronized Map<CharSequence, Integer> buildMappingForHistoryFile() {
+ Map<CharSequence, Integer> mapping = new HashMap<>(getRoleSize());
+ for (ProviderRole role : providerRoles) {
+ mapping.put(role.name, role.id);
+ }
+ return mapping;
+ }
+
+ /**
* Get a clone of the available list
* @param role role index
* @return a clone of the list
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2f060b0/slider-core/src/main/java/org/apache/slider/server/avro/LoadedRoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/avro/LoadedRoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/avro/LoadedRoleHistory.java
index 866e716..3947064 100644
--- a/slider-core/src/main/java/org/apache/slider/server/avro/LoadedRoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/avro/LoadedRoleHistory.java
@@ -19,6 +19,7 @@
package org.apache.slider.server.avro;
import org.apache.hadoop.fs.Path;
+import org.apache.slider.common.tools.SliderUtils;
import java.util.ArrayList;
import java.util.HashMap;
@@ -38,10 +39,18 @@ public class LoadedRoleHistory {
public final List<NodeEntryRecord> records = new ArrayList<>();
+ /**
+ * Add a record
+ * @param record
+ */
public void add(NodeEntryRecord record) {
records.add(record);
}
+ /**
+ * Number of loaded records
+ * @return
+ */
public int size() {
return records.size();
}
@@ -60,4 +69,13 @@ public class LoadedRoleHistory {
public void setPath(Path path) {
this.path = path;
}
+
+ public void buildMapping(Map<CharSequence, Integer> source) {
+ roleMap.clear();
+ for (Map.Entry<CharSequence, Integer> entry : source.entrySet()) {
+ roleMap.put(SliderUtils.sequenceToString(entry.getKey()),
+ entry.getValue());
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2f060b0/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java b/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
index b638ae9..64929b4 100644
--- a/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
+++ b/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
@@ -39,7 +39,6 @@ import org.apache.hadoop.fs.PathFilter;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.providers.ProviderRole;
import org.apache.slider.server.appmaster.state.NodeEntry;
import org.apache.slider.server.appmaster.state.NodeInstance;
import org.apache.slider.server.appmaster.state.RoleHistory;
@@ -57,6 +56,7 @@ import java.util.Collections;
import java.util.List;
import java.util.ListIterator;
import java.util.Locale;
+import java.util.Map;
/**
* Write out the role history to an output stream.
@@ -89,17 +89,13 @@ public class RoleHistoryWriter {
DatumWriter<RoleHistoryRecord> writer =
new SpecificDatumWriter<>(RoleHistoryRecord.class);
+ RoleHistoryRecord record = createHeaderRecord(savetime, history);
int roles = history.getRoleSize();
- RoleHistoryHeader header = new RoleHistoryHeader();
- header.setVersion(ROLE_HISTORY_VERSION);
- header.setSaved(savetime);
- header.setSavedx(Long.toHexString(savetime));
- header.setSavedate(SliderUtils.toGMTString(savetime));
- header.setRoles(roles);
- RoleHistoryRecord record = new RoleHistoryRecord(header);
Schema schema = record.getSchema();
Encoder encoder = EncoderFactory.get().jsonEncoder(schema, out);
writer.write(record, encoder);
+ // now write the rolemap record
+ writer.write(createRolemapRecord(history), encoder);
long count = 0;
//now for every role history entry, write out its record
Collection<NodeInstance> instances = history.cloneNodemap().values();
@@ -127,6 +123,34 @@ public class RoleHistoryWriter {
}
/**
+ * Create the header record
+ * @param savetime time of save
+ * @param history history
+ * @return a record to place at the head of the file
+ */
+ private RoleHistoryRecord createHeaderRecord(long savetime, RoleHistory history) {
+ RoleHistoryHeader header = new RoleHistoryHeader();
+ header.setVersion(ROLE_HISTORY_VERSION);
+ header.setSaved(savetime);
+ header.setSavedx(Long.toHexString(savetime));
+ header.setSavedate(SliderUtils.toGMTString(savetime));
+ header.setRoles(history.getRoleSize());
+ return new RoleHistoryRecord(header);
+ }
+
+ /**
+ * Create the role record
+ * @param history history
+ * @return a record to place at the head of the file
+ */
+ private RoleHistoryRecord createRolemapRecord(RoleHistory history) {
+ RoleHistoryMapping entry = new RoleHistoryMapping();
+ Map<CharSequence, Integer> mapping = history.buildMappingForHistoryFile();
+ entry.setRolemap(mapping);
+ return new RoleHistoryRecord(entry);
+ }
+
+ /**
* Write write the file
*
*
@@ -210,25 +234,38 @@ public class RoleHistoryWriter {
ROLE_HISTORY_VERSION));
}
loadedRoleHistory.setHeader(header);
- RoleHistoryFooter footer;
+ RoleHistoryFooter footer = null;
int records = 0;
//go through reading data
try {
- while (true) {
+ while (footer == null) {
record = reader.read(null, decoder);
entry = record.getEntry();
if (entry instanceof RoleHistoryHeader) {
throw new IOException("Duplicate Role History Header found");
- }
- if (entry instanceof RoleHistoryFooter) {
+ } else if (entry instanceof RoleHistoryMapping) {
+ // role history mapping entry
+ if (!loadedRoleHistory.roleMap.isEmpty()) {
+ // duplicate role maps are viewed as something to warn over, rather than fail
+ log.warn("Duplicate role map; ignoring");
+ } else {
+ RoleHistoryMapping historyMapping = (RoleHistoryMapping) entry;
+ loadedRoleHistory.buildMapping(historyMapping.getRolemap());
+ }
+ } else if (entry instanceof NodeEntryRecord) {
+ // normal record
+ records++;
+ NodeEntryRecord nodeEntryRecord = (NodeEntryRecord) entry;
+ loadedRoleHistory.add(nodeEntryRecord);
+ } else if (entry instanceof RoleHistoryFooter) {
//tail end of the file
footer = (RoleHistoryFooter) entry;
- break;
+ } else {
+ // this is to handle future versions, such as when rolling back
+ // from a later version of slider
+ log.warn("Discarding unknown record {}", entry);
}
- records++;
- NodeEntryRecord nodeEntryRecord = (NodeEntryRecord) entry;
- loadedRoleHistory.add(nodeEntryRecord);
}
} catch (EOFException e) {
EOFException ex = new EOFException(
@@ -236,7 +273,8 @@ public class RoleHistoryWriter {
ex.initCause(e);
throw ex;
}
- //at this point there should be no data left.
+ // at this point there should be no data left.
+ // check by reading and expecting a -1
if (in.read() > 0) {
// footer is in stream before the last record
throw new EOFException(
[5/7] incubator-slider git commit: SLIDER-600: catch any exception
during loads, such as class casts
Posted by st...@apache.org.
SLIDER-600: catch any exception during loads, such as class casts
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9dfb52df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9dfb52df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9dfb52df
Branch: refs/heads/develop
Commit: 9dfb52df30f4b2954e826d89c9093b191829c3df
Parents: b2f060b
Author: Steve Loughran <st...@apache.org>
Authored: Thu Apr 9 13:34:17 2015 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Apr 9 13:34:17 2015 +0100
----------------------------------------------------------------------
.../java/org/apache/slider/server/avro/RoleHistoryWriter.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9dfb52df/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java b/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
index 64929b4..a6ab679 100644
--- a/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
+++ b/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
@@ -390,6 +390,9 @@ public class RoleHistoryWriter {
log.info("Failed to read {}", path, e);
} catch (AvroTypeException e) {
log.warn("Failed to parse {}", path, e);
+ } catch (Exception e) {
+ // low level event logged @ warn level
+ log.warn("Exception while reading {}", path, e);
}
}
return history;