You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2013/04/12 17:34:13 UTC
svn commit: r1467312 - in /accumulo/branches/1.5:
core/src/main/java/org/apache/accumulo/core/
server/src/main/java/org/apache/accumulo/server/constraints/
server/src/main/java/org/apache/accumulo/server/gc/
server/src/main/java/org/apache/accumulo/ser...
Author: ecn
Date: Fri Apr 12 15:34:12 2013
New Revision: 1467312
URL: http://svn.apache.org/r1467312
Log:
ACCUMULO-1264 put delete markers for the METADATA table into the root tablet
Modified:
accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/Constants.java
accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
accumulo/branches/1.5/server/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java
Modified: accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/Constants.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/Constants.java?rev=1467312&r1=1467311&r2=1467312&view=diff
==============================================================================
--- accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/Constants.java (original)
+++ accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/Constants.java Fri Apr 12 15:34:12 2013
@@ -97,7 +97,9 @@ public class Constants {
public static final Key METADATA_RESERVED_KEYSPACE_STOP_KEY = new Key(new Text(new byte[] {'~' + 1}));
public static final Range METADATA_RESERVED_KEYSPACE = new Range(METADATA_RESERVED_KEYSPACE_START_KEY, true, METADATA_RESERVED_KEYSPACE_STOP_KEY, false);
public static final String METADATA_DELETE_FLAG_PREFIX = "~del";
+ public static final String METADATA_DELETE_FLAG_FOR_METADATA_PREFIX = "!!" + METADATA_DELETE_FLAG_PREFIX;
public static final Range METADATA_DELETES_KEYSPACE = new Range(new Key(new Text(METADATA_DELETE_FLAG_PREFIX)), true, new Key(new Text("~dem")), false);
+ public static final Range METADATA_DELETES_FOR_METADATA_KEYSPACE = new Range(new Key(new Text(METADATA_DELETE_FLAG_FOR_METADATA_PREFIX)), true, new Key(new Text("!!~dem")), false);
public static final String METADATA_BLIP_FLAG_PREFIX = "~blip"; // BLIP = bulk load in progress
public static final Range METADATA_BLIP_KEYSPACE = new Range(new Key(new Text(METADATA_BLIP_FLAG_PREFIX)), true, new Key(new Text("~bliq")), false);
Modified: accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java?rev=1467312&r1=1467311&r2=1467312&view=diff
==============================================================================
--- accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java (original)
+++ accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java Fri Apr 12 15:34:12 2013
@@ -101,9 +101,11 @@ public class MetadataConstraints impleme
byte[] row = mutation.getRow();
- // always allow rows that fall within reserved area
+ // always allow rows that fall within reserved areas
if (row.length > 0 && row[0] == '~')
return null;
+ if (row.length > 2 && row[0] == '!' && row[1] == '!' && row[2] == '~')
+ return null;
for (byte b : row) {
if (b == ';') {
Modified: accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java?rev=1467312&r1=1467311&r2=1467312&view=diff
==============================================================================
--- accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java (original)
+++ accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java Fri Apr 12 15:34:12 2013
@@ -20,8 +20,12 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
@@ -438,23 +442,37 @@ public class SimpleGarbageCollector impl
}
return candidates;
}
+
+ checkForBulkProcessingFiles = false;
+ Range range = Constants.METADATA_DELETES_FOR_METADATA_KEYSPACE;
+ candidates.addAll(getBatch(Constants.METADATA_DELETE_FLAG_FOR_METADATA_PREFIX, range));
+ if (candidateMemExceeded)
+ return candidates;
- Scanner scanner = instance.getConnector(credentials.getPrincipal(), CredentialHelper.extractToken(credentials)).createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+ range = Constants.METADATA_DELETES_KEYSPACE;
+ candidates.addAll(getBatch(Constants.METADATA_DELETE_FLAG_PREFIX, range));
+ return candidates;
+ }
+ private Collection<String> getBatch(String prefix, Range range) throws Exception {
+ // want to ensure GC makes progress... if the 1st N deletes are stable and we keep processing them,
+ // then will never inspect deletes after N
if (continueKey != null) {
- // want to ensure GC makes progress... if the 1st N deletes are stable and we keep processing them, then will never inspect deletes after N
- scanner.setRange(new Range(continueKey, true, Constants.METADATA_DELETES_KEYSPACE.getEndKey(), Constants.METADATA_DELETES_KEYSPACE.isEndKeyInclusive()));
+ if (!range.contains(continueKey)) {
+ // continue key is for some other range
+ return Collections.emptyList();
+ }
+ range = new Range(continueKey, true, range.getEndKey(), range.isEndKeyInclusive());
continueKey = null;
- } else {
- // scan the reserved keyspace for deletes
- scanner.setRange(Constants.METADATA_DELETES_KEYSPACE);
}
+ Scanner scanner = instance.getConnector(credentials.getPrincipal(), CredentialHelper.extractToken(credentials)).createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+ scanner.setRange(range);
+ List<String> result = new ArrayList<String>();
// find candidates for deletion; chop off the prefix
- checkForBulkProcessingFiles = false;
for (Entry<Key,Value> entry : scanner) {
- String cand = entry.getKey().getRow().toString().substring(Constants.METADATA_DELETE_FLAG_PREFIX.length());
- candidates.add(cand);
+ String cand = entry.getKey().getRow().toString().substring(prefix.length());
+ result.add(cand);
checkForBulkProcessingFiles |= cand.toLowerCase(Locale.ENGLISH).contains(Constants.BULK_PREFIX);
if (almostOutOfMemory()) {
candidateMemExceeded = true;
@@ -464,7 +482,7 @@ public class SimpleGarbageCollector impl
}
}
- return candidates;
+ return result;
}
static public boolean almostOutOfMemory() {
@@ -567,7 +585,18 @@ public class SimpleGarbageCollector impl
}
}
}
-
+
+ final static String METADATA_TABLE_DIR = "/" + Constants.METADATA_TABLE_ID;
+ private static Mutation getMarkerDeleteMutation(final String delete) {
+ String prefix = Constants.METADATA_DELETE_FLAG_PREFIX;
+ if (delete.startsWith(METADATA_TABLE_DIR))
+ prefix = Constants.METADATA_DELETE_FLAG_FOR_METADATA_PREFIX;
+
+ Mutation m = new Mutation(new Text(prefix + delete));
+ m.putDelete(EMPTY_TEXT, EMPTY_TEXT);
+ return m;
+ }
+
/**
* This method attempts to do its best to remove files from the filesystem that have been confirmed for deletion.
*/
@@ -584,7 +613,6 @@ public class SimpleGarbageCollector impl
log.error("Unable to create writer to remove file from the !METADATA table", e);
}
}
-
// when deleting a dir and all files in that dir, only need to delete the dir
// the dir will sort right before the files... so remove the files in this case
// to minimize namenode ops
@@ -597,10 +625,8 @@ public class SimpleGarbageCollector impl
} else if (lastDir != null) {
if (delete.startsWith(lastDir)) {
log.debug("Ignoring " + delete + " because " + lastDir + " exist");
- Mutation m = new Mutation(new Text(Constants.METADATA_DELETE_FLAG_PREFIX + delete));
- m.putDelete(EMPTY_TEXT, EMPTY_TEXT);
try {
- writer.addMutation(m);
+ writer.addMutation(getMarkerDeleteMutation(delete));
} catch (MutationsRejectedException e) {
throw new RuntimeException(e);
}
@@ -623,10 +649,11 @@ public class SimpleGarbageCollector impl
public void run() {
boolean removeFlag;
- log.debug("Deleting " + ServerConstants.getTablesDir() + delete);
+ String fullPath = ServerConstants.getTablesDir() + delete;
+ log.debug("Deleting " + fullPath);
try {
- Path p = new Path(ServerConstants.getTablesDir() + delete);
+ Path p = new Path(fullPath);
if (moveToTrash(p) || fs.delete(p, true)) {
// delete succeeded, still want to delete
@@ -668,15 +695,14 @@ public class SimpleGarbageCollector impl
// proceed to clearing out the flags for successful deletes and
// non-existent files
if (removeFlag && finalWriter != null) {
- Mutation m = new Mutation(new Text(Constants.METADATA_DELETE_FLAG_PREFIX + delete));
- m.putDelete(EMPTY_TEXT, EMPTY_TEXT);
- finalWriter.addMutation(m);
+ finalWriter.addMutation(getMarkerDeleteMutation(delete));
}
} catch (Exception e) {
log.error(e, e);
}
}
+
};
deleteThreadPool.execute(deleteTask);
Modified: accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java?rev=1467312&r1=1467311&r2=1467312&view=diff
==============================================================================
--- accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java (original)
+++ accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java Fri Apr 12 15:34:12 2013
@@ -479,11 +479,15 @@ public class MetadataTable extends org.a
public static Mutation createDeleteMutation(String tableId, String pathToRemove) {
Mutation delFlag;
+ String prefix = Constants.METADATA_DELETE_FLAG_PREFIX;
+ if (tableId.equals(Constants.METADATA_TABLE_ID))
+ prefix = Constants.METADATA_DELETE_FLAG_FOR_METADATA_PREFIX;
+
if (pathToRemove.startsWith("../"))
- delFlag = new Mutation(new Text(Constants.METADATA_DELETE_FLAG_PREFIX + pathToRemove.substring(2)));
+ delFlag = new Mutation(new Text(prefix + pathToRemove.substring(2)));
else
- delFlag = new Mutation(new Text(Constants.METADATA_DELETE_FLAG_PREFIX + "/" + tableId + pathToRemove));
-
+ delFlag = new Mutation(new Text(prefix + "/" + tableId + pathToRemove));
+
delFlag.put(EMPTY_TEXT, EMPTY_TEXT, new Value(new byte[] {}));
return delFlag;
}
Modified: accumulo/branches/1.5/server/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/server/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java?rev=1467312&r1=1467311&r2=1467312&view=diff
==============================================================================
--- accumulo/branches/1.5/server/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java (original)
+++ accumulo/branches/1.5/server/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java Fri Apr 12 15:34:12 2013
@@ -21,7 +21,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.TreeMap;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.conf.AccumuloConfiguration;