You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2015/05/10 23:06:24 UTC
[6/9] accumulo git commit: ACCUMULO-3777 Add existence check to
MemoryManager
ACCUMULO-3777 Add existence check to MemoryManager
Somehow we got into a situation where we were checking tablets
for a deleted table. Add a quick check for existence before
running the extent through the memory manager.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/719aac43
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/719aac43
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/719aac43
Branch: refs/heads/master
Commit: 719aac433e0428f969130820427d62404d5642e2
Parents: 757c1cb
Author: Josh Elser <el...@apache.org>
Authored: Sun May 10 01:09:19 2015 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Sun May 10 01:09:19 2015 -0400
----------------------------------------------------------------------
.../tabletserver/LargestFirstMemoryManager.java | 13 ++++
.../LargestFirstMemoryManagerTest.java | 67 ++++++++++++++++++++
2 files changed, 80 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/719aac43/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java b/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
index 323e59d..a39c8b6 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
@@ -22,6 +22,8 @@ import java.util.List;
import java.util.Map.Entry;
import java.util.TreeMap;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.Tables;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.impl.KeyExtent;
import org.apache.accumulo.server.conf.ServerConfiguration;
@@ -143,11 +145,16 @@ public class LargestFirstMemoryManager implements MemoryManager {
return mincIdleThresholds.get(tableId);
}
+ boolean tableExists(Instance instance, String tableId) {
+ return Tables.exists(instance, tableId);
+ }
+
@Override
public MemoryManagementActions getMemoryManagementActions(List<TabletState> tablets) {
if (maxMemory < 0)
throw new IllegalStateException("need to initialize " + LargestFirstMemoryManager.class.getName());
+ final Instance instance = config.getInstance();
final int maxMinCs = maxConcurrentMincs * numWaitingMultiplier;
mincIdleThresholds.clear();
@@ -164,6 +171,12 @@ public class LargestFirstMemoryManager implements MemoryManager {
// find the largest and most idle tablets
for (TabletState ts : tablets) {
+ // Make sure that the table still exists
+ if (!tableExists(instance, ts.getExtent().getTableId().toString())) {
+ log.info("Ignoring extent for deleted table: " + ts.getExtent());
+ continue;
+ }
+
final long memTabletSize = ts.getMemTableSize();
final long minorCompactingSize = ts.getMinorCompactingMemTableSize();
final long idleTime = now - Math.max(ts.getLastCommitTime(), ZERO_TIME);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/719aac43/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java b/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java
index 7c15cfc..b08b980 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java
@@ -33,6 +33,8 @@ import org.apache.accumulo.server.conf.TableConfiguration;
import org.apache.hadoop.io.Text;
import org.junit.Test;
+import com.google.common.base.Function;
+
public class LargestFirstMemoryManagerTest {
private static final long ZERO = System.currentTimeMillis();
@@ -160,6 +162,52 @@ public class LargestFirstMemoryManagerTest {
assertEquals(k("b"), result.tabletsToMinorCompact.get(0));
}
+ @Test
+ public void testDeletedTable() throws Exception {
+ final String deletedTableId = "1";
+ Function<String,Boolean> existenceCheck = new Function<String,Boolean>() {
+ public Boolean apply(String tableId) {
+ return !deletedTableId.equals(tableId);
+ }
+ };
+ LargestFirstMemoryManagerWithExistenceCheck mgr = new LargestFirstMemoryManagerWithExistenceCheck(existenceCheck);
+ ServerConfiguration config = new ServerConfiguration() {
+ ServerConfigurationFactory delegate = new ServerConfigurationFactory(new MockInstance());
+
+ @Override
+ public AccumuloConfiguration getConfiguration() {
+ return DefaultConfiguration.getInstance();
+ }
+
+ @Override
+ public TableConfiguration getTableConfiguration(String tableId) {
+ return delegate.getTableConfiguration(tableId);
+ }
+
+ @Override
+ public TableConfiguration getTableConfiguration(KeyExtent extent) {
+ return delegate.getTableConfiguration(extent);
+ }
+
+ @Override
+ public NamespaceConfiguration getNamespaceConfiguration(String namespaceId) {
+ return delegate.getNamespaceConfiguration(namespaceId);
+ }
+
+ @Override
+ public Instance getInstance() {
+ return delegate.getInstance();
+ }
+ };
+ mgr.init(config);
+ MemoryManagementActions result;
+ // one tablet is really big and the other is for a nonexistent table
+ KeyExtent extent = new KeyExtent(new Text("2"), new Text("j"), null);
+ result = mgr.getMemoryManagementActions(tablets(t(extent, ZERO, ONE_GIG, 0), t(k("j"), ZERO, ONE_GIG, 0)));
+ assertEquals(1, result.tabletsToMinorCompact.size());
+ assertEquals(extent, result.tabletsToMinorCompact.get(0));
+ }
+
private static class LargestFirstMemoryManagerUnderTest extends LargestFirstMemoryManager {
public long currentTime = ZERO;
@@ -174,6 +222,25 @@ public class LargestFirstMemoryManagerTest {
return 15 * 60 * 1000;
}
+ @Override
+ boolean tableExists(Instance instance, String tableId) {
+ return true;
+ }
+ }
+
+ private static class LargestFirstMemoryManagerWithExistenceCheck extends LargestFirstMemoryManagerUnderTest {
+
+ Function<String,Boolean> existenceCheck;
+
+ public LargestFirstMemoryManagerWithExistenceCheck(Function<String,Boolean> existenceCheck) {
+ super();
+ this.existenceCheck = existenceCheck;
+ }
+
+ @Override
+ boolean tableExists(Instance instance, String tableId) {
+ return existenceCheck.apply(tableId);
+ }
}
private static KeyExtent k(String endRow) {