You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2013/08/10 00:23:26 UTC
git commit: ACCUMULO-1651 read log records from zookeeper,
the root table and the metadata table, added a test
Updated Branches:
refs/heads/master 6591b4e94 -> e0a37f808
ACCUMULO-1651 read log records from zookeeper, the root table and the metadata table, added a test
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e0a37f80
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e0a37f80
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e0a37f80
Branch: refs/heads/master
Commit: e0a37f808ad266740cec702279861ce28e991cc8
Parents: 6591b4e
Author: Eric Newton <ec...@apache.org>
Authored: Fri Aug 9 18:23:36 2013 -0400
Committer: Eric Newton <ec...@apache.org>
Committed: Fri Aug 9 18:23:36 2013 -0400
----------------------------------------------------------------------
.../accumulo/server/util/MetadataTableUtil.java | 15 ++++++----
.../test/functional/GarbageCollectorIT.java | 29 ++++++++++++++++++++
2 files changed, 39 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e0a37f80/server/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java b/server/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index 8bdd63e..8894ca7 100644
--- a/server/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -913,11 +913,13 @@ public class MetadataTableUtil {
static class LogEntryIterator implements Iterator<LogEntry> {
- Iterator<LogEntry> rootTabletEntries = null;
+ Iterator<LogEntry> zookeeperEntries = null;
+ Iterator<LogEntry> rootTableEntries = null;
Iterator<Entry<Key,Value>> metadataEntries = null;
LogEntryIterator(Credentials creds) throws IOException, KeeperException, InterruptedException {
- rootTabletEntries = getLogEntries(creds, RootTable.EXTENT).iterator();
+ zookeeperEntries = getLogEntries(creds, RootTable.EXTENT).iterator();
+ rootTableEntries = getLogEntries(creds, new KeyExtent(new Text(MetadataTable.ID), null, null)).iterator();
try {
Scanner scanner = HdfsZooInstance.getInstance().getConnector(creds.getPrincipal(), creds.getToken())
.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
@@ -932,13 +934,16 @@ public class MetadataTableUtil {
@Override
public boolean hasNext() {
- return rootTabletEntries.hasNext() || metadataEntries.hasNext();
+ return zookeeperEntries.hasNext() || rootTableEntries.hasNext() || metadataEntries.hasNext();
}
@Override
public LogEntry next() {
- if (rootTabletEntries.hasNext()) {
- return rootTabletEntries.next();
+ if (zookeeperEntries.hasNext()) {
+ return zookeeperEntries.next();
+ }
+ if (rootTableEntries.hasNext()) {
+ return rootTableEntries.next();
}
Entry<Key,Value> entry = metadataEntries.next();
return entryFromKeyValue(entry.getKey(), entry.getValue());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e0a37f80/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java b/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
index 5ce1fa6..217b978 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
@@ -20,21 +20,27 @@ import static org.junit.Assert.assertTrue;
import java.util.HashMap;
import java.util.Map;
+import java.util.Map.Entry;
import org.apache.accumulo.core.cli.BatchWriterOpts;
import org.apache.accumulo.core.cli.ScannerOpts;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.TablePermission;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.minicluster.MemoryUnit;
import org.apache.accumulo.minicluster.MiniAccumuloConfig;
+import org.apache.accumulo.minicluster.ProcessReference;
+import org.apache.accumulo.minicluster.ServerType;
import org.apache.accumulo.server.gc.SimpleGarbageCollector;
import org.apache.accumulo.test.TestIngest;
import org.apache.accumulo.test.VerifyIngest;
@@ -96,6 +102,29 @@ public class GarbageCollectorIT extends ConfigurableMacIT {
assertTrue(output.contains("delete candidates has exceeded"));
}
+ @Test(timeout = 2 * 60 * 1000)
+ public void dontGCRootLog() throws Exception {
+ // dirty !METADATA
+ Connector c = getConnector();
+ String table = makeTableName();
+ c.tableOperations().create(table);
+ // let gc run for a bit
+ Process gc = cluster.exec(SimpleGarbageCollector.class);
+ UtilWaitThread.sleep(20 * 1000);
+ gc.destroy();
+ // kill tservers
+ for (ProcessReference ref : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, ref);
+ }
+ // run recovery
+ cluster.start();
+ // did it recover?
+ Scanner scanner = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ for (@SuppressWarnings("unused") Entry<Key,Value> unused : scanner) {
+
+ }
+ }
+
private int countFiles() throws Exception {
FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
int result = 0;