You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/08/06 20:26:18 UTC
[3/8] git commit: SLIDER-296 removing use of RoleHistory/nodemap
target release selection logic
SLIDER-296 removing use of RoleHistory/nodemap target release selection logic
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8cecce18
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8cecce18
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8cecce18
Branch: refs/heads/develop
Commit: 8cecce1821733c33de3cf9723f9a96aee411305d
Parents: 5ce953c
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 6 17:11:39 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 6 17:11:39 2014 +0100
----------------------------------------------------------------------
.../server/appmaster/state/NodeInstance.java | 12 +-
.../server/appmaster/state/RoleHistory.java | 14 +
.../history/TestFindNodesForNewInstances.groovy | 128 ---------
.../history/TestFindNodesForRelease.groovy | 146 -----------
.../model/history/TestHistoryRW.groovy | 258 -------------------
.../model/history/TestHistoryRWOrdering.groovy | 145 -----------
.../model/history/TestNIComparators.groovy | 77 ------
.../TestOutstandingRequestTracker.groovy | 73 ------
.../TestRoleHistoryContainerEvents.groovy | 17 +-
...stRoleHistoryFindNodesForNewInstances.groovy | 128 +++++++++
.../TestRoleHistoryFindNodesForRelease.groovy | 146 +++++++++++
.../history/TestRoleHistoryNIComparators.groovy | 77 ++++++
...tRoleHistoryOutstandingRequestTracker.groovy | 73 ++++++
.../model/history/TestRoleHistoryRW.groovy | 258 +++++++++++++++++++
.../history/TestRoleHistoryRWOrdering.groovy | 145 +++++++++++
15 files changed, 861 insertions(+), 836 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
index 06375fb..fad60f2 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
@@ -75,7 +75,7 @@ public class NodeInstance {
}
/**
- * Cout the number of active role instances on this node
+ * Count the number of active role instances on this node
* @param role role index
* @return 0 if there are none, otherwise the #of nodes that are running and
* not being released already.
@@ -84,6 +84,16 @@ public class NodeInstance {
NodeEntry nodeEntry = get(role);
return (nodeEntry != null ) ? nodeEntry.getActive() : 0;
}
+
+ /**
+ * Count the number of live role instances on this node
+ * @param role role index
+ * @return 0 if there are none, otherwise the #of nodes that are running
+ */
+ public int getLiveRoleInstances(int role) {
+ NodeEntry nodeEntry = get(role);
+ return (nodeEntry != null ) ? nodeEntry.getLive() : 0;
+ }
/**
* Get the entry for a role -and remove it if present
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index de7e9d8..dbb104c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -529,6 +529,20 @@ public class RoleHistory {
return nodemap.findNodesForRelease(role, count);
}
+
+ /**
+ * Get the list of active nodes ... walks the node map so
+ * is O(nodes)
+ * @param role role index
+ * @return a possibly empty list of nodes with an instance of that node
+ */
+ public synchronized List<NodeInstance> listActiveNodes(int role) {
+ return nodemap.listActiveNodes(role);
+ }
+
+
+
+
/**
* Get the node entry of a container
* @param container container to look up
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForNewInstances.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForNewInstances.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForNewInstances.groovy
deleted file mode 100644
index dab03f5..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForNewInstances.groovy
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.history
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.slider.providers.ProviderRole
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockFactory
-import org.apache.slider.server.appmaster.state.NodeInstance
-import org.apache.slider.server.appmaster.state.RoleHistory
-import org.apache.slider.server.appmaster.state.RoleStatus
-import org.junit.Before
-import org.junit.Test
-
-/**
- * Testing finding nodes for new instances.
- * These tests validate the (currently) suboptimal
- * behavior of not listing any known nodes when there
- * are none in the available list -even if there are nodes
- * known to be not running live instances in the cluster.
- */
-@Slf4j
-@CompileStatic
-class TestFindNodesForNewInstances extends BaseMockAppStateTest {
-
- @Override
- String getTestName() {
- return "TestFindNodesForNewInstances"
- }
-
- NodeInstance age1Active4 = nodeInstance(1, 4, 0, 0)
- NodeInstance age2Active2 = nodeInstance(2, 2, 0, 1)
- NodeInstance age3Active0 = nodeInstance(3, 0, 0, 0)
- NodeInstance age4Active1 = nodeInstance(4, 1, 0, 0)
- NodeInstance age2Active0 = nodeInstance(2, 0, 0, 0)
- NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT)
-
- List<NodeInstance> nodes = [age2Active2, age2Active0, age4Active1, age1Active4, age3Active0]
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
-
- String roleName = "test"
- RoleStatus roleStat = new RoleStatus(new ProviderRole(roleName, 0))
- RoleStatus roleStat2 = new RoleStatus(new ProviderRole(roleName, 2))
-
- @Before
- public void setupNodeMap() {
- roleHistory.insert(nodes)
- roleHistory.buildAvailableNodeLists();
- }
-
-
- public List<NodeInstance> findNodes(int count, RoleStatus roleStatus = roleStat) {
- List < NodeInstance > found = [];
- for (int i = 0; i < count; i++) {
- NodeInstance f = roleHistory.findNodeForNewInstance(roleStatus)
- if (f) {
- found << f
- };
- }
- return found
- }
-
- @Test
- public void testFind1NodeR0() throws Throwable {
- NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
- log.info("found: $found")
- assert [age3Active0].contains(found)
- }
-
- @Test
- public void testFind2NodeR0() throws Throwable {
- NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
- log.info("found: $found")
- assert [age2Active0, age3Active0].contains(found)
- NodeInstance found2 = roleHistory.findNodeForNewInstance(roleStat)
- log.info("found: $found2")
- assert [age2Active0, age3Active0].contains(found2)
- assert found != found2;
- }
- @Test
- public void testFind3NodeR0ReturnsNull() throws Throwable {
- assert 2== findNodes(2).size()
- NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
- assert found == null;
- }
-
- @Test
- public void testFindNodesOneEntry() throws Throwable {
- List<NodeInstance> nodes = findNodes(4, roleStat2)
- assert 0 == nodes.size()
- }
-
- @Test
- public void testFindNodesIndependent() throws Throwable {
- assert 2 == findNodes(2).size()
- roleHistory.dump()
- assert 0 == findNodes(3, roleStat2).size()
- }
-
- @Test
- public void testFindNodesFallsBackWhenUsed() throws Throwable {
- // mark age2 and active 0 as busy, expect a null back
- age2Active0.get(0).onStartCompleted()
- assert age2Active0.getActiveRoleInstances(0) != 0
- age3Active0.get(0).onStartCompleted()
- assert age3Active0.getActiveRoleInstances(0) != 0
- NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
- log.info(found ?.toFullString())
- assert found == null
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForRelease.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForRelease.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForRelease.groovy
deleted file mode 100644
index 92915dd..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForRelease.groovy
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.history
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockFactory
-import org.apache.slider.server.appmaster.state.NodeInstance
-import org.apache.slider.server.appmaster.state.NodeMap
-import org.junit.Before
-import org.junit.Test
-
-@Slf4j
-@CompileStatic
-class TestFindNodesForRelease extends BaseMockAppStateTest {
-
-
- @Override
- String getTestName() {
- return "TestFindNodesForRelease"
- }
- NodeInstance age1Active4 = nodeInstance(1, 4, 0, 0)
- NodeInstance age2Active2 = nodeInstance(2, 2, 0, 0)
- NodeInstance age3Active0 = nodeInstance(3, 0, 0, 0)
- NodeInstance age4Active1 = nodeInstance(4, 1, 0, 0)
- NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT)
-
- List<NodeInstance> nodes = [age2Active2, age4Active1, age1Active4, age3Active0]
- NodeMap nodeMap = new NodeMap(MockFactory.ROLE_COUNT);
-
-
- @Before
- public void setupNodeMap() {
- nodeMap.insert(nodes)
- }
-
- private void assertReleased(
- int count,
- List<NodeInstance> expected,
- int role = 0) {
- List<NodeInstance> released = nodeMap.findNodesForRelease(role, count)
- assertListEquals(released, expected)
- }
- private void assertReleased(
- List<NodeInstance> expected,
- int role = 0) {
- List<NodeInstance> released = nodeMap.findNodesForRelease(role, expected.size())
- assertListEquals(released, expected)
- }
-
- @Test
- public void testListActiveNodes() throws Throwable {
- assertListEquals(nodeMap.listActiveNodes(0),
- [age1Active4,age2Active2, age4Active1])
- }
-
- @Test
- public void testReleaseMinus1() throws Throwable {
- try {
- nodeMap.findNodesForRelease(0, -1)
- fail("Expected an exception")
- } catch (IllegalArgumentException e) {
- }
- }
- @Test
- public void testReleaseO() throws Throwable {
- assertReleased(0, [])
- }
-
- @Test
- public void testRelease1() throws Throwable {
- assertReleased(1, [age1Active4])
- }
-
- @Test
- public void testRelease2() throws Throwable {
- assertReleased(2, [age1Active4, age1Active4])
- }
-
- @Test
- public void testRelease3() throws Throwable {
- assertReleased(3, [age1Active4, age1Active4, age1Active4 ])
- }
-
- @Test
- public void testRelease4() throws Throwable {
- assertReleased(4, [age1Active4, age1Active4, age1Active4 , age2Active2])
- }
-
- @Test
- public void testRelease5() throws Throwable {
- assertReleased([age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1])
- }
-
- @Test
- public void testRelease6() throws Throwable {
- assertReleased(
- [age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1, age1Active4])
- }
-
- @Test
- public void testRelease7() throws Throwable {
- assertReleased(
- [age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1,
- age1Active4, age2Active2])
- }
-
- @Test
- public void testRelease8() throws Throwable {
- assertReleased(8,
- [age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1,
- age1Active4, age2Active2])
- }
-
- @Test
- public void testPurgeInactiveTime3() throws Throwable {
- assert nodeMap.purgeUnusedEntries(3) == 0;
- }
-
- @Test
- public void testPurgeInactiveTime4() throws Throwable {
- assert nodeMap.purgeUnusedEntries(4) == 1;
- }
- @Test
- public void testPurgeInactiveTime5() throws Throwable {
- assert nodeMap.purgeUnusedEntries(5) == 1;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRW.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRW.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRW.groovy
deleted file mode 100644
index b646661..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRW.groovy
+++ /dev/null
@@ -1,258 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.history
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.fs.FSDataOutputStream
-import org.apache.hadoop.fs.Path
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockFactory
-import org.apache.slider.server.appmaster.state.NodeEntry
-import org.apache.slider.server.appmaster.state.NodeInstance
-import org.apache.slider.server.appmaster.state.RoleHistory
-import org.apache.slider.server.avro.RoleHistoryWriter
-import org.junit.Test
-
-@Slf4j
-@CompileStatic
-class TestHistoryRW extends BaseMockAppStateTest {
-
- static long time = System.currentTimeMillis();
-
- @Override
- String getTestName() {
- return "TestHistoryRW"
- }
-
- @Test
- public void testWriteReadEmpty() throws Throwable {
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
- roleHistory.onStart(fs, historyPath)
- Path history = roleHistory.saveHistory(time++)
- assert fs.isFile(history)
- RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- historyWriter.read(fs, history, roleHistory)
- }
-
- @Test
- public void testWriteReadData() throws Throwable {
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
- assert !roleHistory.onStart(fs, historyPath)
- String addr = "localhost"
- NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
- NodeEntry ne1 = instance.getOrCreate(0)
- ne1.lastUsed = 0xf00d
-
- Path history = roleHistory.saveHistory(time++)
- assert fs.isFile(history)
- RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
-
- assert 0 < historyWriter.read(fs, history, rh2)
- NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
- assert ni2 != null
- NodeEntry ne2 = ni2.get(0)
- assert ne2 !=null
- assert ne2.lastUsed == ne1.lastUsed
- }
-
- @Test
- public void testWriteReadActiveData() throws Throwable {
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
- roleHistory.onStart(fs, historyPath)
- String addr = "localhost"
- String addr2 = "rack1server5"
- NodeInstance localhost = roleHistory.getOrCreateNodeInstance(addr)
- NodeEntry orig1 = localhost.getOrCreate(0)
- orig1.lastUsed = 0x10
- NodeInstance rack1server5 = roleHistory.getOrCreateNodeInstance(addr2)
- NodeEntry orig2 = rack1server5.getOrCreate(1)
- orig2.live = 3
- assert !orig2.available
- NodeEntry orig3 = localhost.getOrCreate(1)
- orig3.lastUsed = 0x20
- orig3.live = 1
- assert !orig3.available
- orig3.release()
- assert orig3.available
- roleHistory.dump()
-
- long savetime = 0x0001000
- Path history = roleHistory.saveHistory(savetime)
- assert fs.isFile(history)
- describe("Loaded")
- log.info("testWriteReadActiveData in $history")
- RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
-
- assert 3 == historyWriter.read(fs, history, rh2)
- rh2.dump()
-
- assert rh2.clusterSize == 2;
- NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
- assert ni2 != null
- NodeEntry loadedNE = ni2.get(0)
- assert loadedNE.lastUsed == orig1.lastUsed
- NodeInstance ni2b = rh2.getExistingNodeInstance(addr2)
- assert ni2b != null
- NodeEntry loadedNE2 = ni2b.get(1)
- assert loadedNE2 != null
- assert loadedNE2.lastUsed == savetime
- assert rh2.thawedDataTime == savetime
-
- // now thaw it
- rh2.buildAvailableNodeLists();
- describe("thawing")
- rh2.dump();
- List<NodeInstance> available0 = rh2.cloneAvailableList(0)
- assert available0.size() == 1
-
- NodeInstance entry = available0.get(0)
- assert entry.hostname == "localhost"
- assert entry == localhost
- List<NodeInstance> available1 = rh2.cloneAvailableList(1)
- assert available1.size() == 2
- //and verify that even if last used was set, the save time is picked up
- assert entry.get(1).lastUsed == roleHistory.saveTime
-
- }
-
- @Test
- public void testWriteThaw() throws Throwable {
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
- assert !roleHistory.onStart(fs, historyPath)
- String addr = "localhost"
- NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
- NodeEntry ne1 = instance.getOrCreate(0)
- ne1.lastUsed = 0xf00d
-
- Path history = roleHistory.saveHistory(time++)
- long savetime =roleHistory.saveTime;
- assert fs.isFile(history)
- RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
- assert rh2.onStart(fs, historyPath)
- NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
- assert ni2 != null
- NodeEntry ne2 = ni2.get(0)
- assert ne2 != null
- assert ne2.lastUsed == ne1.lastUsed
- assert rh2.thawedDataTime == savetime
- }
-
-
- @Test
- public void testPurgeOlderEntries() throws Throwable {
- RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- time = 1;
- Path file1 = touch(historyWriter, time++)
- Path file2 = touch(historyWriter, time++)
- Path file3 = touch(historyWriter, time++)
- Path file4 = touch(historyWriter, time++)
- Path file5 = touch(historyWriter, time++)
- Path file6 = touch(historyWriter, time++)
-
- assert historyWriter.purgeOlderHistoryEntries(fs, file1) == 0
- assert historyWriter.purgeOlderHistoryEntries(fs, file2) == 1
- assert historyWriter.purgeOlderHistoryEntries(fs, file2) == 0
- assert historyWriter.purgeOlderHistoryEntries(fs, file5) == 3
- assert historyWriter.purgeOlderHistoryEntries(fs, file6) == 1
- try {
- // make an impossible assertion that will fail if the method
- // actually completes
- assert -1 == historyWriter.purgeOlderHistoryEntries(fs, file1)
- } catch (FileNotFoundException ignored) {
- // expected
- }
-
- }
-
- public Path touch(RoleHistoryWriter historyWriter, long time){
- Path path = historyWriter.createHistoryFilename(historyPath, time);
- FSDataOutputStream out = fs.create(path);
- out.close()
- return path
- }
-
- @Test
- public void testSkipEmptyFileOnRead() throws Throwable {
- describe "verify that empty histories are skipped on read; old histories purged"
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
- roleHistory.onStart(fs, historyPath)
- time = 0
- Path oldhistory = roleHistory.saveHistory(time++)
-
- String addr = "localhost"
- NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
- NodeEntry ne1 = instance.getOrCreate(0)
- ne1.lastUsed = 0xf00d
-
- Path goodhistory = roleHistory.saveHistory(time++)
-
- RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- Path touched = touch(historyWriter, time++)
-
- RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
- assert rh2.onStart(fs, historyPath)
- NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
- assert ni2 != null
-
- //and assert the older file got purged
- assert !fs.exists(oldhistory)
- assert fs.exists(goodhistory)
- assert fs.exists(touched )
- }
-
- @Test
- public void testSkipBrokenFileOnRead() throws Throwable {
- describe "verify that empty histories are skipped on read; old histories purged"
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
- roleHistory.onStart(fs, historyPath)
- time = 0
- Path oldhistory = roleHistory.saveHistory(time++)
-
- String addr = "localhost"
- NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
- NodeEntry ne1 = instance.getOrCreate(0)
- ne1.lastUsed = 0xf00d
-
- Path goodhistory = roleHistory.saveHistory(time++)
-
- RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- Path badfile = historyWriter.createHistoryFilename(historyPath, time++)
- FSDataOutputStream out = fs.create(badfile)
- out.writeBytes("{broken:true}")
- out.close()
-
- RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
- describe("IGNORE STACK TRACE BELOW")
-
- assert rh2.onStart(fs, historyPath)
-
- describe( "IGNORE STACK TRACE ABOVE")
- NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
- assert ni2 != null
-
- //and assert the older file got purged
- assert !fs.exists(oldhistory)
- assert fs.exists(goodhistory)
- assert fs.exists(badfile )
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRWOrdering.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRWOrdering.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRWOrdering.groovy
deleted file mode 100644
index 6ec046c..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRWOrdering.groovy
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.history
-
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.fs.Path
-import org.apache.slider.common.SliderKeys
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockFactory
-import org.apache.slider.server.appmaster.state.NodeEntry
-import org.apache.slider.server.appmaster.state.NodeInstance
-import org.apache.slider.server.appmaster.state.RoleHistory
-import org.apache.slider.server.avro.RoleHistoryWriter
-import org.junit.Test
-
-import java.util.regex.Matcher
-import java.util.regex.Pattern
-
-@Slf4j
-class TestHistoryRWOrdering extends BaseMockAppStateTest {
-
- def paths = pathlist(
- [
- "hdfs://localhost/history-0406c.json",
- "hdfs://localhost/history-5fffa.json",
- "hdfs://localhost/history-0001a.json",
- "hdfs://localhost/history-0001f.json",
- ]
- )
- Path h_0406c = paths[0]
- Path h_5fffa = paths[1]
- Path h_0001a = paths[3]
-
-
- List<Path> pathlist(List<String> pathnames) {
- def result = []
- pathnames.each { result << new Path(new URI(it as String)) }
- result
- }
-
- @Override
- String getTestName() {
- return "TestHistoryRWOrdering"
- }
-
-
- /**
- * This tests regexp pattern matching. It uses the current time so isn't
- * repeatable -but it does test a wider range of values in the process
- * @throws Throwable
- */
- @Test
- public void testPatternRoundTrip() throws Throwable {
- describe "test pattern matching of names"
- long value=System.currentTimeMillis()
- String name = String.format(SliderKeys.HISTORY_FILENAME_CREATION_PATTERN,value)
- String matchpattern = SliderKeys.HISTORY_FILENAME_MATCH_PATTERN
- Pattern pattern = Pattern.compile(matchpattern)
- Matcher matcher = pattern.matcher(name);
- if (!matcher.find()) {
- throw new Exception("No match for pattern $matchpattern in $name")
- }
- }
-
-
- @Test
- public void testWriteSequenceReadData() throws Throwable {
- describe "test that if multiple entries are written, the newest is picked up"
- long time = System.currentTimeMillis();
-
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
- assert !roleHistory.onStart(fs, historyPath)
- String addr = "localhost"
- NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
- NodeEntry ne1 = instance.getOrCreate(0)
- ne1.lastUsed = 0xf00d
-
- Path history1 = roleHistory.saveHistory(time++)
- Path history2 = roleHistory.saveHistory(time++)
- Path history3 = roleHistory.saveHistory(time++)
-
- //inject a later file with a different name
- sliderFileSystem.cat(new Path(historyPath, "file.json"), true, "hello, world")
-
-
- RoleHistoryWriter historyWriter = new RoleHistoryWriter();
-
- List<Path> entries = historyWriter.findAllHistoryEntries(
- fs,
- historyPath,
- false)
- assert entries.size() == 3
- assert entries[0] == history3
- assert entries[1] == history2
- assert entries[2] == history1
- }
-
- @Test
- public void testPathStructure() throws Throwable {
- assert h_5fffa.getName() == "history-5fffa.json"
- }
-
- @Test
- public void testPathnameComparator() throws Throwable {
-
- def newerName = new RoleHistoryWriter.NewerFilesFirst()
-
- log.info("$h_5fffa name is ${h_5fffa.getName()}")
- log.info("$h_0406c name is ${h_0406c.getName()}")
- assert newerName.compare(h_5fffa, h_5fffa) == 0
- assert newerName.compare(h_5fffa, h_0406c) < 0
- assert newerName.compare(h_5fffa, h_0001a) < 0
- assert newerName.compare(h_0001a, h_5fffa) > 0
-
- }
-
- @Test
- public void testPathSort() throws Throwable {
- def paths2 = new ArrayList(paths)
- RoleHistoryWriter.sortHistoryPaths(paths2)
- assertListEquals(paths2,
- [
- paths[1],
- paths[0],
- paths[3],
- paths[2]
- ])
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestNIComparators.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestNIComparators.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestNIComparators.groovy
deleted file mode 100644
index 77119d5..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestNIComparators.groovy
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.history
-
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockFactory
-import org.apache.slider.server.appmaster.state.NodeInstance
-import org.junit.Test
-
-/**
- * Unit test to verify the comparators sort as expected
- */
-class TestNIComparators extends BaseMockAppStateTest {
-
- NodeInstance age1Active4 = nodeInstance(1000, 4, 0, 0)
- NodeInstance age2Active2 = nodeInstance(1001, 2, 0, 0)
- NodeInstance age3Active0 = nodeInstance(1002, 0, 0, 0)
- NodeInstance age4Active1 = nodeInstance(1005, 0, 0, 0)
- NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT)
-
- List<NodeInstance> nodes = [age2Active2, age4Active1, age1Active4, age3Active0]
-
- @Override
- String getTestName() {
- return "TestNIComparators"
- }
-
- @Test
- public void testNewerThan() throws Throwable {
-
- Collections.sort(nodes, new NodeInstance.newerThan(0))
- assertListEquals(nodes,
- [age4Active1, age3Active0, age2Active2, age1Active4])
- }
-
- @Test
- public void testNewerThanNoRole() throws Throwable {
-
- nodes << empty
- Collections.sort(nodes, new NodeInstance.newerThan(0))
- assertListEquals(nodes,
- [age4Active1, age3Active0, age2Active2, age1Active4, empty])
- }
-
- @Test
- public void testMoreActiveThan() throws Throwable {
-
- Collections.sort(nodes, new NodeInstance.moreActiveThan(0))
- assertListEquals(nodes,
- [age1Active4, age2Active2, age4Active1, age3Active0],)
- }
-
- @Test
- public void testMoreActiveThanEmpty() throws Throwable {
- nodes << empty
- Collections.sort(nodes, new NodeInstance.moreActiveThan(0))
- assertListEquals(nodes,
- [age1Active4, age2Active2, age4Active1, age3Active0, empty])
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestOutstandingRequestTracker.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestOutstandingRequestTracker.groovy
deleted file mode 100644
index 8d1f4b0..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestOutstandingRequestTracker.groovy
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.history
-
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.state.NodeInstance
-import org.apache.slider.server.appmaster.state.OutstandingRequest
-import org.apache.slider.server.appmaster.state.OutstandingRequestTracker
-import org.junit.Test
-
-class TestOutstandingRequestTracker extends BaseMockAppStateTest {
-
- NodeInstance host1 = new NodeInstance("host1", 3)
- NodeInstance host2 = new NodeInstance("host2", 3)
-
- OutstandingRequestTracker tracker = new OutstandingRequestTracker()
-
- @Override
- String getTestName() {
- return "TestOutstandingRequestTracker"
- }
-
- @Test
- public void testAddRetrieveEntry() throws Throwable {
- OutstandingRequest request = tracker.addRequest(host1, 0)
- assert tracker.lookup(0, "host1").equals(request)
- assert tracker.remove(request).equals(request)
- assert !tracker.lookup(0, "host1")
- }
-
- @Test
- public void testAddCompleteEntry() throws Throwable {
- tracker.addRequest(host1, 0)
- tracker.addRequest(host2, 0)
- tracker.addRequest(host1, 1)
- assert tracker.onContainerAllocated(1, "host1")
- assert !tracker.lookup(1, "host1")
- assert tracker.lookup(0, "host1")
- }
-
- @Test
- public void testCancelEntries() throws Throwable {
- OutstandingRequest r1 = tracker.addRequest(host1, 0)
- OutstandingRequest r2 = tracker.addRequest(host2, 0)
- OutstandingRequest r3 = tracker.addRequest(host1, 1)
- List<NodeInstance> canceled = tracker.cancelOutstandingRequests(0)
- assert canceled.size() == 2
- assert canceled.contains(host1)
- assert canceled.contains(host2)
- assert tracker.lookup(1, "host1")
- assert !tracker.lookup(0, "host1")
- canceled = tracker.cancelOutstandingRequests(0)
- assert canceled.size() == 0
- assert tracker.cancelOutstandingRequests(1).size() == 1
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
index 1a71aed..340e72d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
@@ -114,12 +114,12 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest {
RoleStatus roleStatus = new RoleStatus(provRole)
//verify it is empty
- assert roleHistory.findNodesForRelease(role, 1).isEmpty()
+ assert roleHistory.listActiveNodes(role).empty
AMRMClient.ContainerRequest request =
roleHistory.requestNode(roleStatus, resource);
- List<String> nodes = request.getNodes()
+ List<String> nodes = request.nodes
assert nodes == null
//pick an idle host
@@ -128,7 +128,7 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest {
//build a container
MockContainer container = factory.newContainer()
container.nodeId = new MockNodeId(hostname, 0)
- container.priority = request.getPriority()
+ container.priority = request.priority
roleHistory.onContainerAssigned(container);
NodeMap nodemap = roleHistory.cloneNodemap();
@@ -147,10 +147,11 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest {
assert roleEntry.live == 1
// now pick that instance to destroy
+ List<NodeInstance> activeNodes = roleHistory.listActiveNodes(role)
+
- List<NodeInstance> forRelease = roleHistory.findNodesForRelease(role, 1)
- assert forRelease.size() == 1
- NodeInstance target = forRelease[0]
+ assert activeNodes.size() == 1
+ NodeInstance target = activeNodes[0]
assert target == allocated
roleHistory.onContainerReleaseSubmitted(container);
assert roleEntry.releasing == 1
@@ -164,13 +165,13 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest {
assert roleEntry.active == 0
// verify it is empty
- assert roleHistory.findNodesForRelease(role, 1).isEmpty()
+ assert roleHistory.listActiveNodes(role).empty
// ask for a container and expect to get the recently released one
AMRMClient.ContainerRequest request2 =
roleHistory.requestNode(roleStatus, resource);
- List<String> nodes2 = request2.getNodes()
+ List<String> nodes2 = request2.nodes
assert nodes2 != null
String hostname2 = nodes2[0]
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy
new file mode 100644
index 0000000..79cd348
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.providers.ProviderRole
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.RoleHistory
+import org.apache.slider.server.appmaster.state.RoleStatus
+import org.junit.Before
+import org.junit.Test
+
+/**
+ * Testing finding nodes for new instances.
+ * These tests validate the (currently) suboptimal
+ * behavior of not listing any known nodes when there
+ * are none in the available list -even if there are nodes
+ * known to be not running live instances in the cluster.
+ */
+@Slf4j
+@CompileStatic
+class TestRoleHistoryFindNodesForNewInstances extends BaseMockAppStateTest {
+
+ @Override
+ String getTestName() {
+ return "TestFindNodesForNewInstances"
+ }
+
+ NodeInstance age1Active4 = nodeInstance(1, 4, 0, 0)
+ NodeInstance age2Active2 = nodeInstance(2, 2, 0, 1)
+ NodeInstance age3Active0 = nodeInstance(3, 0, 0, 0)
+ NodeInstance age4Active1 = nodeInstance(4, 1, 0, 0)
+ NodeInstance age2Active0 = nodeInstance(2, 0, 0, 0)
+ NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT)
+
+ List<NodeInstance> nodes = [age2Active2, age2Active0, age4Active1, age1Active4, age3Active0]
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+
+ String roleName = "test"
+ RoleStatus roleStat = new RoleStatus(new ProviderRole(roleName, 0))
+ RoleStatus roleStat2 = new RoleStatus(new ProviderRole(roleName, 2))
+
+ @Before
+ public void setupNodeMap() {
+ roleHistory.insert(nodes)
+ roleHistory.buildAvailableNodeLists();
+ }
+
+
+ public List<NodeInstance> findNodes(int count, RoleStatus roleStatus = roleStat) {
+ List < NodeInstance > found = [];
+ for (int i = 0; i < count; i++) {
+ NodeInstance f = roleHistory.findNodeForNewInstance(roleStatus)
+ if (f) {
+ found << f
+ };
+ }
+ return found
+ }
+
+ @Test
+ public void testFind1NodeR0() throws Throwable {
+ NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
+ log.info("found: $found")
+ assert [age3Active0].contains(found)
+ }
+
+ @Test
+ public void testFind2NodeR0() throws Throwable {
+ NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
+ log.info("found: $found")
+ assert [age2Active0, age3Active0].contains(found)
+ NodeInstance found2 = roleHistory.findNodeForNewInstance(roleStat)
+ log.info("found: $found2")
+ assert [age2Active0, age3Active0].contains(found2)
+ assert found != found2;
+ }
+ @Test
+ public void testFind3NodeR0ReturnsNull() throws Throwable {
+ assert 2== findNodes(2).size()
+ NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
+ assert found == null;
+ }
+
+ @Test
+ public void testFindNodesOneEntry() throws Throwable {
+ List<NodeInstance> nodes = findNodes(4, roleStat2)
+ assert 0 == nodes.size()
+ }
+
+ @Test
+ public void testFindNodesIndependent() throws Throwable {
+ assert 2 == findNodes(2).size()
+ roleHistory.dump()
+ assert 0 == findNodes(3, roleStat2).size()
+ }
+
+ @Test
+ public void testFindNodesFallsBackWhenUsed() throws Throwable {
+ // mark age2 and active 0 as busy, expect a null back
+ age2Active0.get(0).onStartCompleted()
+ assert age2Active0.getActiveRoleInstances(0) != 0
+ age3Active0.get(0).onStartCompleted()
+ assert age3Active0.getActiveRoleInstances(0) != 0
+ NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
+ log.info(found ?.toFullString())
+ assert found == null
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForRelease.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForRelease.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForRelease.groovy
new file mode 100644
index 0000000..64db899
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForRelease.groovy
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.NodeMap
+import org.junit.Before
+import org.junit.Test
+
+@Slf4j
+@CompileStatic
+class TestRoleHistoryFindNodesForRelease extends BaseMockAppStateTest {
+
+
+ @Override
+ String getTestName() {
+ return "TestFindNodesForRelease"
+ }
+ NodeInstance age1Active4 = nodeInstance(1, 4, 0, 0)
+ NodeInstance age2Active2 = nodeInstance(2, 2, 0, 0)
+ NodeInstance age3Active0 = nodeInstance(3, 0, 0, 0)
+ NodeInstance age4Active1 = nodeInstance(4, 1, 0, 0)
+ NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT)
+
+ List<NodeInstance> nodes = [age2Active2, age4Active1, age1Active4, age3Active0]
+ NodeMap nodeMap = new NodeMap(MockFactory.ROLE_COUNT);
+
+
+ @Before
+ public void setupNodeMap() {
+ nodeMap.insert(nodes)
+ }
+
+ private void assertReleased(
+ int count,
+ List<NodeInstance> expected,
+ int role = 0) {
+ List<NodeInstance> released = nodeMap.findNodesForRelease(role, count)
+ assertListEquals(released, expected)
+ }
+ private void assertReleased(
+ List<NodeInstance> expected,
+ int role = 0) {
+ List<NodeInstance> released = nodeMap.findNodesForRelease(role, expected.size())
+ assertListEquals(released, expected)
+ }
+
+ @Test
+ public void testListActiveNodes() throws Throwable {
+ assertListEquals(nodeMap.listActiveNodes(0),
+ [age1Active4,age2Active2, age4Active1])
+ }
+
+ @Test
+ public void testReleaseMinus1() throws Throwable {
+ try {
+ nodeMap.findNodesForRelease(0, -1)
+ fail("Expected an exception")
+ } catch (IllegalArgumentException e) {
+ }
+ }
+ @Test
+ public void testReleaseO() throws Throwable {
+ assertReleased(0, [])
+ }
+
+ @Test
+ public void testRelease1() throws Throwable {
+ assertReleased(1, [age1Active4])
+ }
+
+ @Test
+ public void testRelease2() throws Throwable {
+ assertReleased(2, [age1Active4, age1Active4])
+ }
+
+ @Test
+ public void testRelease3() throws Throwable {
+ assertReleased(3, [age1Active4, age1Active4, age1Active4 ])
+ }
+
+ @Test
+ public void testRelease4() throws Throwable {
+ assertReleased(4, [age1Active4, age1Active4, age1Active4 , age2Active2])
+ }
+
+ @Test
+ public void testRelease5() throws Throwable {
+ assertReleased([age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1])
+ }
+
+ @Test
+ public void testRelease6() throws Throwable {
+ assertReleased(
+ [age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1, age1Active4])
+ }
+
+ @Test
+ public void testRelease7() throws Throwable {
+ assertReleased(
+ [age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1,
+ age1Active4, age2Active2])
+ }
+
+ @Test
+ public void testRelease8() throws Throwable {
+ assertReleased(8,
+ [age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1,
+ age1Active4, age2Active2])
+ }
+
+ @Test
+ public void testPurgeInactiveTime3() throws Throwable {
+ assert nodeMap.purgeUnusedEntries(3) == 0;
+ }
+
+ @Test
+ public void testPurgeInactiveTime4() throws Throwable {
+ assert nodeMap.purgeUnusedEntries(4) == 1;
+ }
+ @Test
+ public void testPurgeInactiveTime5() throws Throwable {
+ assert nodeMap.purgeUnusedEntries(5) == 1;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy
new file mode 100644
index 0000000..612cce8
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history
+
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.junit.Test
+
+/**
+ * Unit test to verify the comparators sort as expected
+ */
+class TestRoleHistoryNIComparators extends BaseMockAppStateTest {
+
+ NodeInstance age1Active4 = nodeInstance(1000, 4, 0, 0)
+ NodeInstance age2Active2 = nodeInstance(1001, 2, 0, 0)
+ NodeInstance age3Active0 = nodeInstance(1002, 0, 0, 0)
+ NodeInstance age4Active1 = nodeInstance(1005, 0, 0, 0)
+ NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT)
+
+ List<NodeInstance> nodes = [age2Active2, age4Active1, age1Active4, age3Active0]
+
+ @Override
+ String getTestName() {
+ return "TestNIComparators"
+ }
+
+ @Test
+ public void testNewerThan() throws Throwable {
+
+ Collections.sort(nodes, new NodeInstance.newerThan(0))
+ assertListEquals(nodes,
+ [age4Active1, age3Active0, age2Active2, age1Active4])
+ }
+
+ @Test
+ public void testNewerThanNoRole() throws Throwable {
+
+ nodes << empty
+ Collections.sort(nodes, new NodeInstance.newerThan(0))
+ assertListEquals(nodes,
+ [age4Active1, age3Active0, age2Active2, age1Active4, empty])
+ }
+
+ @Test
+ public void testMoreActiveThan() throws Throwable {
+
+ Collections.sort(nodes, new NodeInstance.moreActiveThan(0))
+ assertListEquals(nodes,
+ [age1Active4, age2Active2, age4Active1, age3Active0],)
+ }
+
+ @Test
+ public void testMoreActiveThanEmpty() throws Throwable {
+ nodes << empty
+ Collections.sort(nodes, new NodeInstance.moreActiveThan(0))
+ assertListEquals(nodes,
+ [age1Active4, age2Active2, age4Active1, age3Active0, empty])
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
new file mode 100644
index 0000000..7085678
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history
+
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.OutstandingRequest
+import org.apache.slider.server.appmaster.state.OutstandingRequestTracker
+import org.junit.Test
+
+class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
+
+ NodeInstance host1 = new NodeInstance("host1", 3)
+ NodeInstance host2 = new NodeInstance("host2", 3)
+
+ OutstandingRequestTracker tracker = new OutstandingRequestTracker()
+
+ @Override
+ String getTestName() {
+ return "TestOutstandingRequestTracker"
+ }
+
+ @Test
+ public void testAddRetrieveEntry() throws Throwable {
+ OutstandingRequest request = tracker.addRequest(host1, 0)
+ assert tracker.lookup(0, "host1").equals(request)
+ assert tracker.remove(request).equals(request)
+ assert !tracker.lookup(0, "host1")
+ }
+
+ @Test
+ public void testAddCompleteEntry() throws Throwable {
+ tracker.addRequest(host1, 0)
+ tracker.addRequest(host2, 0)
+ tracker.addRequest(host1, 1)
+ assert tracker.onContainerAllocated(1, "host1")
+ assert !tracker.lookup(1, "host1")
+ assert tracker.lookup(0, "host1")
+ }
+
+ @Test
+ public void testCancelEntries() throws Throwable {
+ OutstandingRequest r1 = tracker.addRequest(host1, 0)
+ OutstandingRequest r2 = tracker.addRequest(host2, 0)
+ OutstandingRequest r3 = tracker.addRequest(host1, 1)
+ List<NodeInstance> canceled = tracker.cancelOutstandingRequests(0)
+ assert canceled.size() == 2
+ assert canceled.contains(host1)
+ assert canceled.contains(host2)
+ assert tracker.lookup(1, "host1")
+ assert !tracker.lookup(0, "host1")
+ canceled = tracker.cancelOutstandingRequests(0)
+ assert canceled.size() == 0
+ assert tracker.cancelOutstandingRequests(1).size() == 1
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
new file mode 100644
index 0000000..0f4a4ca
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.FSDataOutputStream
+import org.apache.hadoop.fs.Path
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.state.NodeEntry
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.RoleHistory
+import org.apache.slider.server.avro.RoleHistoryWriter
+import org.junit.Test
+
+@Slf4j
+@CompileStatic
+class TestRoleHistoryRW extends BaseMockAppStateTest {
+
+ static long time = System.currentTimeMillis();
+
+ @Override
+ String getTestName() {
+ return "TestHistoryRW"
+ }
+
+ @Test
+ public void testWriteReadEmpty() throws Throwable {
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ roleHistory.onStart(fs, historyPath)
+ Path history = roleHistory.saveHistory(time++)
+ assert fs.isFile(history)
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ historyWriter.read(fs, history, roleHistory)
+ }
+
+ @Test
+ public void testWriteReadData() throws Throwable {
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ assert !roleHistory.onStart(fs, historyPath)
+ String addr = "localhost"
+ NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
+ NodeEntry ne1 = instance.getOrCreate(0)
+ ne1.lastUsed = 0xf00d
+
+ Path history = roleHistory.saveHistory(time++)
+ assert fs.isFile(history)
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
+
+ assert 0 < historyWriter.read(fs, history, rh2)
+ NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
+ assert ni2 != null
+ NodeEntry ne2 = ni2.get(0)
+ assert ne2 !=null
+ assert ne2.lastUsed == ne1.lastUsed
+ }
+
+ @Test
+ public void testWriteReadActiveData() throws Throwable {
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ roleHistory.onStart(fs, historyPath)
+ String addr = "localhost"
+ String addr2 = "rack1server5"
+ NodeInstance localhost = roleHistory.getOrCreateNodeInstance(addr)
+ NodeEntry orig1 = localhost.getOrCreate(0)
+ orig1.lastUsed = 0x10
+ NodeInstance rack1server5 = roleHistory.getOrCreateNodeInstance(addr2)
+ NodeEntry orig2 = rack1server5.getOrCreate(1)
+ orig2.live = 3
+ assert !orig2.available
+ NodeEntry orig3 = localhost.getOrCreate(1)
+ orig3.lastUsed = 0x20
+ orig3.live = 1
+ assert !orig3.available
+ orig3.release()
+ assert orig3.available
+ roleHistory.dump()
+
+ long savetime = 0x0001000
+ Path history = roleHistory.saveHistory(savetime)
+ assert fs.isFile(history)
+ describe("Loaded")
+ log.info("testWriteReadActiveData in $history")
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
+
+ assert 3 == historyWriter.read(fs, history, rh2)
+ rh2.dump()
+
+ assert rh2.clusterSize == 2;
+ NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
+ assert ni2 != null
+ NodeEntry loadedNE = ni2.get(0)
+ assert loadedNE.lastUsed == orig1.lastUsed
+ NodeInstance ni2b = rh2.getExistingNodeInstance(addr2)
+ assert ni2b != null
+ NodeEntry loadedNE2 = ni2b.get(1)
+ assert loadedNE2 != null
+ assert loadedNE2.lastUsed == savetime
+ assert rh2.thawedDataTime == savetime
+
+ // now thaw it
+ rh2.buildAvailableNodeLists();
+ describe("thawing")
+ rh2.dump();
+ List<NodeInstance> available0 = rh2.cloneAvailableList(0)
+ assert available0.size() == 1
+
+ NodeInstance entry = available0.get(0)
+ assert entry.hostname == "localhost"
+ assert entry == localhost
+ List<NodeInstance> available1 = rh2.cloneAvailableList(1)
+ assert available1.size() == 2
+ //and verify that even if last used was set, the save time is picked up
+ assert entry.get(1).lastUsed == roleHistory.saveTime
+
+ }
+
+ @Test
+ public void testWriteThaw() throws Throwable {
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ assert !roleHistory.onStart(fs, historyPath)
+ String addr = "localhost"
+ NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
+ NodeEntry ne1 = instance.getOrCreate(0)
+ ne1.lastUsed = 0xf00d
+
+ Path history = roleHistory.saveHistory(time++)
+ long savetime =roleHistory.saveTime;
+ assert fs.isFile(history)
+ RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
+ assert rh2.onStart(fs, historyPath)
+ NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
+ assert ni2 != null
+ NodeEntry ne2 = ni2.get(0)
+ assert ne2 != null
+ assert ne2.lastUsed == ne1.lastUsed
+ assert rh2.thawedDataTime == savetime
+ }
+
+
+ @Test
+ public void testPurgeOlderEntries() throws Throwable {
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ time = 1;
+ Path file1 = touch(historyWriter, time++)
+ Path file2 = touch(historyWriter, time++)
+ Path file3 = touch(historyWriter, time++)
+ Path file4 = touch(historyWriter, time++)
+ Path file5 = touch(historyWriter, time++)
+ Path file6 = touch(historyWriter, time++)
+
+ assert historyWriter.purgeOlderHistoryEntries(fs, file1) == 0
+ assert historyWriter.purgeOlderHistoryEntries(fs, file2) == 1
+ assert historyWriter.purgeOlderHistoryEntries(fs, file2) == 0
+ assert historyWriter.purgeOlderHistoryEntries(fs, file5) == 3
+ assert historyWriter.purgeOlderHistoryEntries(fs, file6) == 1
+ try {
+ // make an impossible assertion that will fail if the method
+ // actually completes
+ assert -1 == historyWriter.purgeOlderHistoryEntries(fs, file1)
+ } catch (FileNotFoundException ignored) {
+ // expected
+ }
+
+ }
+
+ public Path touch(RoleHistoryWriter historyWriter, long time){
+ Path path = historyWriter.createHistoryFilename(historyPath, time);
+ FSDataOutputStream out = fs.create(path);
+ out.close()
+ return path
+ }
+
+ @Test
+ public void testSkipEmptyFileOnRead() throws Throwable {
+ describe "verify that empty histories are skipped on read; old histories purged"
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ roleHistory.onStart(fs, historyPath)
+ time = 0
+ Path oldhistory = roleHistory.saveHistory(time++)
+
+ String addr = "localhost"
+ NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
+ NodeEntry ne1 = instance.getOrCreate(0)
+ ne1.lastUsed = 0xf00d
+
+ Path goodhistory = roleHistory.saveHistory(time++)
+
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ Path touched = touch(historyWriter, time++)
+
+ RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
+ assert rh2.onStart(fs, historyPath)
+ NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
+ assert ni2 != null
+
+ //and assert the older file got purged
+ assert !fs.exists(oldhistory)
+ assert fs.exists(goodhistory)
+ assert fs.exists(touched )
+ }
+
+ @Test
+ public void testSkipBrokenFileOnRead() throws Throwable {
+ describe "verify that empty histories are skipped on read; old histories purged"
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ roleHistory.onStart(fs, historyPath)
+ time = 0
+ Path oldhistory = roleHistory.saveHistory(time++)
+
+ String addr = "localhost"
+ NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
+ NodeEntry ne1 = instance.getOrCreate(0)
+ ne1.lastUsed = 0xf00d
+
+ Path goodhistory = roleHistory.saveHistory(time++)
+
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ Path badfile = historyWriter.createHistoryFilename(historyPath, time++)
+ FSDataOutputStream out = fs.create(badfile)
+ out.writeBytes("{broken:true}")
+ out.close()
+
+ RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
+ describe("IGNORE STACK TRACE BELOW")
+
+ assert rh2.onStart(fs, historyPath)
+
+ describe( "IGNORE STACK TRACE ABOVE")
+ NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
+ assert ni2 != null
+
+ //and assert the older file got purged
+ assert !fs.exists(oldhistory)
+ assert fs.exists(goodhistory)
+ assert fs.exists(badfile )
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy
new file mode 100644
index 0000000..a0663e8
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.Path
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.state.NodeEntry
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.RoleHistory
+import org.apache.slider.server.avro.RoleHistoryWriter
+import org.junit.Test
+
+import java.util.regex.Matcher
+import java.util.regex.Pattern
+
+@Slf4j
+class TestRoleHistoryRWOrdering extends BaseMockAppStateTest {
+
+ def paths = pathlist(
+ [
+ "hdfs://localhost/history-0406c.json",
+ "hdfs://localhost/history-5fffa.json",
+ "hdfs://localhost/history-0001a.json",
+ "hdfs://localhost/history-0001f.json",
+ ]
+ )
+ Path h_0406c = paths[0]
+ Path h_5fffa = paths[1]
+ Path h_0001a = paths[3]
+
+
+ List<Path> pathlist(List<String> pathnames) {
+ def result = []
+ pathnames.each { result << new Path(new URI(it as String)) }
+ result
+ }
+
+ @Override
+ String getTestName() {
+ return "TestHistoryRWOrdering"
+ }
+
+
+ /**
+ * This tests regexp pattern matching. It uses the current time so isn't
+ * repeatable -but it does test a wider range of values in the process
+ * @throws Throwable
+ */
+ @Test
+ public void testPatternRoundTrip() throws Throwable {
+ describe "test pattern matching of names"
+ long value=System.currentTimeMillis()
+ String name = String.format(SliderKeys.HISTORY_FILENAME_CREATION_PATTERN,value)
+ String matchpattern = SliderKeys.HISTORY_FILENAME_MATCH_PATTERN
+ Pattern pattern = Pattern.compile(matchpattern)
+ Matcher matcher = pattern.matcher(name);
+ if (!matcher.find()) {
+ throw new Exception("No match for pattern $matchpattern in $name")
+ }
+ }
+
+
+ @Test
+ public void testWriteSequenceReadData() throws Throwable {
+ describe "test that if multiple entries are written, the newest is picked up"
+ long time = System.currentTimeMillis();
+
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ assert !roleHistory.onStart(fs, historyPath)
+ String addr = "localhost"
+ NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
+ NodeEntry ne1 = instance.getOrCreate(0)
+ ne1.lastUsed = 0xf00d
+
+ Path history1 = roleHistory.saveHistory(time++)
+ Path history2 = roleHistory.saveHistory(time++)
+ Path history3 = roleHistory.saveHistory(time++)
+
+ //inject a later file with a different name
+ sliderFileSystem.cat(new Path(historyPath, "file.json"), true, "hello, world")
+
+
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+
+ List<Path> entries = historyWriter.findAllHistoryEntries(
+ fs,
+ historyPath,
+ false)
+ assert entries.size() == 3
+ assert entries[0] == history3
+ assert entries[1] == history2
+ assert entries[2] == history1
+ }
+
+ @Test
+ public void testPathStructure() throws Throwable {
+ assert h_5fffa.getName() == "history-5fffa.json"
+ }
+
+ @Test
+ public void testPathnameComparator() throws Throwable {
+
+ def newerName = new RoleHistoryWriter.NewerFilesFirst()
+
+ log.info("$h_5fffa name is ${h_5fffa.getName()}")
+ log.info("$h_0406c name is ${h_0406c.getName()}")
+ assert newerName.compare(h_5fffa, h_5fffa) == 0
+ assert newerName.compare(h_5fffa, h_0406c) < 0
+ assert newerName.compare(h_5fffa, h_0001a) < 0
+ assert newerName.compare(h_0001a, h_5fffa) > 0
+
+ }
+
+ @Test
+ public void testPathSort() throws Throwable {
+ def paths2 = new ArrayList(paths)
+ RoleHistoryWriter.sortHistoryPaths(paths2)
+ assertListEquals(paths2,
+ [
+ paths[1],
+ paths[0],
+ paths[3],
+ paths[2]
+ ])
+ }
+}