You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2014/06/18 23:06:03 UTC
git commit: ACCUMULO-2922 unit test LargestFirstMemoryManager
Repository: accumulo
Updated Branches:
refs/heads/master 5f24240e4 -> ffd2626a8
ACCUMULO-2922 unit test LargestFirstMemoryManager
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ffd2626a
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ffd2626a
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ffd2626a
Branch: refs/heads/master
Commit: ffd2626a8cc4df476eef9bfcd613896dce165a4d
Parents: 5f24240
Author: Eric C. Newton <er...@gmail.com>
Authored: Wed Jun 18 17:04:25 2014 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Wed Jun 18 17:05:57 2014 -0400
----------------------------------------------------------------------
.../tabletserver/LargestFirstMemoryManager.java | 11 +-
.../LargestFirstMemoryManagerTest.java | 262 +++++++++++++++++++
2 files changed, 270 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ffd2626a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java b/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
index b891ad6..d06d61e 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
@@ -40,7 +40,8 @@ public class LargestFirstMemoryManager implements MemoryManager {
private static final long ZERO_TIME = System.currentTimeMillis();
private static final int TSERV_MINC_MAXCONCURRENT_NUMWAITING_MULTIPLIER = 2;
private static final double MAX_FLUSH_AT_ONCE_PERCENT = 0.20;
-
+
+
private long maxMemory = -1;
private int maxConcurrentMincs;
private int numWaitingMultiplier;
@@ -111,7 +112,7 @@ public class LargestFirstMemoryManager implements MemoryManager {
maxObserved = 0;
}
- private long getMinCIdleThreshold(KeyExtent extent) {
+ protected long getMinCIdleThreshold(KeyExtent extent) {
Text tableId = extent.getTableId();
if (!mincIdleThresholds.containsKey(tableId))
mincIdleThresholds.put(tableId, config.getTableConfiguration(tableId.toString()).getTimeInMillis(Property.TABLE_MINC_COMPACT_IDLETIME));
@@ -131,7 +132,7 @@ public class LargestFirstMemoryManager implements MemoryManager {
TreeMap<Long,TabletInfo> largestMemTablets = new LargestMap(maxMinCs);
final TreeMap<Long,TabletInfo> largestIdleMemTablets = new LargestMap(maxConcurrentMincs);
- final long now = System.currentTimeMillis();
+ final long now = currentTimeMillis();
long ingestMemory = 0;
long compactionMemory = 0;
@@ -222,6 +223,10 @@ public class LargestFirstMemoryManager implements MemoryManager {
return result;
}
+ protected long currentTimeMillis() {
+ return System.currentTimeMillis();
+ }
+
@Override
public void tabletClosed(KeyExtent extent) {}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ffd2626a/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java b/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java
new file mode 100644
index 0000000..a9c609b
--- /dev/null
+++ b/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java
@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.tabletserver;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class LargestFirstMemoryManagerTest {
+
+ private static final long ZERO = System.currentTimeMillis();
+ private static final long LATER = ZERO + 20 * 60 * 1000;
+ private static final long ONE_GIG = 1024*1024*1024;
+ private static final long HALF_GIG = ONE_GIG/2;
+ private static final long QGIG = ONE_GIG/4;
+ private static final long ONE_MINUTE = 60 * 1000;
+
+ @Test
+ public void test() throws Exception {
+ LargestFirstMemoryManagerUnderTest mgr = new LargestFirstMemoryManagerUnderTest();
+ Instance instance = new MockInstance();
+ ServerConfiguration config = new ServerConfiguration(instance);
+ mgr.init(config);
+ MemoryManagementActions result;
+ // nothing to do
+ result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, 1000, 0), t(k("y"), ZERO, 2000, 0)));
+ assertEquals(0, result.tabletsToMinorCompact.size());
+ // one tablet is really big
+ result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, ONE_GIG, 0), t(k("y"), ZERO, 2000, 0)));
+ assertEquals(1, result.tabletsToMinorCompact.size());
+ assertEquals(k("x"), result.tabletsToMinorCompact.get(0));
+ // one tablet is idle
+ mgr.currentTime = LATER;
+ result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, 1001, 0), t(k("y"), LATER, 2000, 0)));
+ assertEquals(1, result.tabletsToMinorCompact.size());
+ assertEquals(k("x"), result.tabletsToMinorCompact.get(0));
+ // one tablet is idle, but one is really big
+ result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, 1001, 0), t(k("y"), LATER, ONE_GIG, 0)));
+ assertEquals(1, result.tabletsToMinorCompact.size());
+ assertEquals(k("y"), result.tabletsToMinorCompact.get(0));
+ // lots of work to do
+ mgr = new LargestFirstMemoryManagerUnderTest();
+ mgr.init(config);
+ result = mgr.getMemoryManagementActions(tablets(
+ t(k("a"), ZERO, HALF_GIG, 0),
+ t(k("b"), ZERO, HALF_GIG+1, 0),
+ t(k("c"), ZERO, HALF_GIG+2, 0),
+ t(k("d"), ZERO, HALF_GIG+3, 0),
+ t(k("e"), ZERO, HALF_GIG+4, 0),
+ t(k("f"), ZERO, HALF_GIG+5, 0),
+ t(k("g"), ZERO, HALF_GIG+6, 0),
+ t(k("h"), ZERO, HALF_GIG+7, 0),
+ t(k("i"), ZERO, HALF_GIG+8, 0))
+ );
+ assertEquals(2, result.tabletsToMinorCompact.size());
+ assertEquals(k("i"), result.tabletsToMinorCompact.get(0));
+ assertEquals(k("h"), result.tabletsToMinorCompact.get(1));
+ // one finished, one in progress, one filled up
+ mgr = new LargestFirstMemoryManagerUnderTest();
+ mgr.init(config);
+ result = mgr.getMemoryManagementActions(tablets(
+ t(k("a"), ZERO, HALF_GIG, 0),
+ t(k("b"), ZERO, HALF_GIG+1, 0),
+ t(k("c"), ZERO, HALF_GIG+2, 0),
+ t(k("d"), ZERO, HALF_GIG+3, 0),
+ t(k("e"), ZERO, HALF_GIG+4, 0),
+ t(k("f"), ZERO, HALF_GIG+5, 0),
+ t(k("g"), ZERO, ONE_GIG, 0),
+ t(k("h"), ZERO, 0, HALF_GIG+7),
+ t(k("i"), ZERO, 0, 0))
+ );
+ assertEquals(1, result.tabletsToMinorCompact.size());
+ assertEquals(k("g"), result.tabletsToMinorCompact.get(0));
+ // memory is very full, lots of candidates
+ result = mgr.getMemoryManagementActions(tablets(
+ t(k("a"), ZERO, HALF_GIG, 0),
+ t(k("b"), ZERO, ONE_GIG+1, 0),
+ t(k("c"), ZERO, ONE_GIG+2, 0),
+ t(k("d"), ZERO, ONE_GIG+3, 0),
+ t(k("e"), ZERO, ONE_GIG+4, 0),
+ t(k("f"), ZERO, ONE_GIG+5, 0),
+ t(k("g"), ZERO, ONE_GIG+6, 0),
+ t(k("h"), ZERO, 0, 0),
+ t(k("i"), ZERO, 0, 0))
+ );
+ assertEquals(2, result.tabletsToMinorCompact.size());
+ assertEquals(k("g"), result.tabletsToMinorCompact.get(0));
+ assertEquals(k("f"), result.tabletsToMinorCompact.get(1));
+ // only have two compactors, still busy
+ result = mgr.getMemoryManagementActions(tablets(
+ t(k("a"), ZERO, HALF_GIG, 0),
+ t(k("b"), ZERO, ONE_GIG+1, 0),
+ t(k("c"), ZERO, ONE_GIG+2, 0),
+ t(k("d"), ZERO, ONE_GIG+3, 0),
+ t(k("e"), ZERO, ONE_GIG+4, 0),
+ t(k("f"), ZERO, ONE_GIG, ONE_GIG+5),
+ t(k("g"), ZERO, ONE_GIG, ONE_GIG+6),
+ t(k("h"), ZERO, 0, 0),
+ t(k("i"), ZERO, 0, 0))
+ );
+ assertEquals(0, result.tabletsToMinorCompact.size());
+ // finished one
+ result = mgr.getMemoryManagementActions(tablets(
+ t(k("a"), ZERO, HALF_GIG, 0),
+ t(k("b"), ZERO, ONE_GIG+1, 0),
+ t(k("c"), ZERO, ONE_GIG+2, 0),
+ t(k("d"), ZERO, ONE_GIG+3, 0),
+ t(k("e"), ZERO, ONE_GIG+4, 0),
+ t(k("f"), ZERO, ONE_GIG, ONE_GIG+5),
+ t(k("g"), ZERO, ONE_GIG, 0),
+ t(k("h"), ZERO, 0, 0),
+ t(k("i"), ZERO, 0, 0))
+ );
+ assertEquals(1, result.tabletsToMinorCompact.size());
+ assertEquals(k("e"), result.tabletsToMinorCompact.get(0));
+
+ // many are running: do nothing
+ mgr = new LargestFirstMemoryManagerUnderTest();
+ mgr.init(config);
+ result = mgr.getMemoryManagementActions(tablets(
+ t(k("a"), ZERO, HALF_GIG, 0),
+ t(k("b"), ZERO, HALF_GIG+1, 0),
+ t(k("c"), ZERO, HALF_GIG+2, 0),
+ t(k("d"), ZERO, 0, HALF_GIG),
+ t(k("e"), ZERO, 0, HALF_GIG),
+ t(k("f"), ZERO, 0, HALF_GIG),
+ t(k("g"), ZERO, 0, HALF_GIG),
+ t(k("i"), ZERO, 0, HALF_GIG),
+ t(k("j"), ZERO, 0, HALF_GIG),
+ t(k("k"), ZERO, 0, HALF_GIG),
+ t(k("l"), ZERO, 0, HALF_GIG),
+ t(k("m"), ZERO, 0, HALF_GIG)
+ ));
+ assertEquals(0, result.tabletsToMinorCompact.size());
+
+ // observe adjustment:
+ mgr = new LargestFirstMemoryManagerUnderTest();
+ mgr.init(config);
+ // compact the largest
+ result = mgr.getMemoryManagementActions(tablets(
+ t(k("a"), ZERO, QGIG, 0),
+ t(k("b"), ZERO, QGIG+1, 0),
+ t(k("c"), ZERO, QGIG+2, 0)));
+ assertEquals(1, result.tabletsToMinorCompact.size());
+ assertEquals(k("c"), result.tabletsToMinorCompact.get(0));
+ // show that it is compacting... do nothing
+ result = mgr.getMemoryManagementActions(tablets(
+ t(k("a"), ZERO, QGIG, 0),
+ t(k("b"), ZERO, QGIG+1, 0),
+ t(k("c"), ZERO, 0, QGIG+2)));
+ assertEquals(0, result.tabletsToMinorCompact.size());
+ // not going to bother compacting any more
+ mgr.currentTime += ONE_MINUTE;
+ result = mgr.getMemoryManagementActions(tablets(
+ t(k("a"), ZERO, QGIG, 0),
+ t(k("b"), ZERO, QGIG+1, 0),
+ t(k("c"), ZERO, 0, QGIG+2)));
+ assertEquals(0, result.tabletsToMinorCompact.size());
+ // now do nothing
+ mgr.currentTime += ONE_MINUTE;
+ result = mgr.getMemoryManagementActions(tablets(
+ t(k("a"), ZERO, QGIG, 0),
+ t(k("b"), ZERO, 0, 0),
+ t(k("c"), ZERO, 0, 0)));
+ assertEquals(0, result.tabletsToMinorCompact.size());
+ // on no! more data, this time we compact because we've adjusted
+ mgr.currentTime += ONE_MINUTE;
+ result = mgr.getMemoryManagementActions(tablets(
+ t(k("a"), ZERO, QGIG, 0),
+ t(k("b"), ZERO, QGIG+1, 0),
+ t(k("c"), ZERO, 0, 0)));
+ assertEquals(1, result.tabletsToMinorCompact.size());
+ assertEquals(k("b"), result.tabletsToMinorCompact.get(0));
+ }
+
+ private static class LargestFirstMemoryManagerUnderTest extends LargestFirstMemoryManager {
+
+ public long currentTime = ZERO;
+
+ @Override
+ protected long currentTimeMillis() {
+ return currentTime;
+ }
+
+ @Override
+ protected long getMinCIdleThreshold(KeyExtent extent) {
+ return 15 * 60 * 1000;
+ }
+
+ }
+
+ private static KeyExtent k(String endRow) {
+ return new KeyExtent(new Text("1"), new Text(endRow), null);
+ }
+
+ private static class TestTabletState implements TabletState {
+
+ private final KeyExtent extent;
+ private final long lastCommit;
+ private final long memSize;
+ private final long compactingSize;
+
+ TestTabletState(KeyExtent extent, long commit, long memsize, long compactingTableSize) {
+ this.extent = extent;
+ this.lastCommit = commit;
+ this.memSize = memsize;
+ this.compactingSize = compactingTableSize;
+ }
+
+ @Override
+ public KeyExtent getExtent() {
+ return extent;
+ }
+
+ @Override
+ public long getLastCommitTime() {
+ return lastCommit;
+ }
+
+ @Override
+ public long getMemTableSize() {
+ return memSize;
+ }
+
+ @Override
+ public long getMinorCompactingMemTableSize() {
+ return compactingSize;
+ }
+
+ }
+
+ private TabletState t(KeyExtent ke, long lastCommit, long memSize, long compactingSize) {
+ return new TestTabletState(ke, lastCommit, memSize, compactingSize);
+ }
+
+ private static List<TabletState> tablets(TabletState ... states) {
+ return Arrays.asList(states);
+ }
+
+}