You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2013/10/29 14:39:15 UTC
git commit: ACCUMULO-1773 properly handled exceptions in AGC
Updated Branches:
refs/heads/master c2dee4be5 -> cc7b7ef69
ACCUMULO-1773 properly handled exceptions in AGC
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/cc7b7ef6
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/cc7b7ef6
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/cc7b7ef6
Branch: refs/heads/master
Commit: cc7b7ef69b389010b6700a1723674ee6edf04a18
Parents: c2dee4b
Author: Keith Turner <kt...@apache.org>
Authored: Tue Oct 29 09:38:31 2013 -0400
Committer: Keith Turner <kt...@apache.org>
Committed: Tue Oct 29 09:38:44 2013 -0400
----------------------------------------------------------------------
.../server/gc/GarbageCollectionAlgorithm.java | 61 ++++++++++++--------
.../server/gc/GarbageCollectionEnvironment.java | 35 ++++++++---
.../server/gc/SimpleGarbageCollector.java | 21 +------
3 files changed, 68 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cc7b7ef6/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectionAlgorithm.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectionAlgorithm.java b/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectionAlgorithm.java
index 02018bc..325f1d9 100644
--- a/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectionAlgorithm.java
+++ b/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectionAlgorithm.java
@@ -119,7 +119,7 @@ public class GarbageCollectionAlgorithm {
return ret;
}
- protected void confirmDeletes(GarbageCollectionEnvironment gce, SortedMap<String,String> candidateMap) throws TableNotFoundException, AccumuloException,
+ private void confirmDeletes(GarbageCollectionEnvironment gce, SortedMap<String,String> candidateMap) throws TableNotFoundException, AccumuloException,
AccumuloSecurityException {
boolean checkForBulkProcessingFiles = false;
Iterator<String> relativePaths = candidateMap.keySet().iterator();
@@ -227,18 +227,46 @@ public class GarbageCollectionAlgorithm {
}
+ private List<String> getCandidates(GarbageCollectionEnvironment gce, String lastCandidate) throws TableNotFoundException, AccumuloException,
+ AccumuloSecurityException {
+ Span candidatesSpan = Trace.start("getCandidates");
+ List<String> candidates;
+ try {
+ candidates = gce.getCandidates(lastCandidate);
+ } finally {
+ candidatesSpan.stop();
+ }
+ return candidates;
+ }
+
+ private void confirmDeletesTrace(GarbageCollectionEnvironment gce, SortedMap<String,String> candidateMap) throws TableNotFoundException, AccumuloException,
+ AccumuloSecurityException {
+ Span confirmDeletesSpan = Trace.start("confirmDeletes");
+ try {
+ confirmDeletes(gce, candidateMap);
+ } finally {
+ confirmDeletesSpan.stop();
+ }
+ }
+
+ private void deleteConfirmed(GarbageCollectionEnvironment gce, SortedMap<String,String> candidateMap) throws IOException, AccumuloException,
+ AccumuloSecurityException, TableNotFoundException {
+ Span deleteSpan = Trace.start("deleteFiles");
+ try {
+ gce.delete(candidateMap);
+ } finally {
+ deleteSpan.stop();
+ }
+
+ cleanUpDeletedTableDirs(gce, candidateMap);
+ }
+
public void collect(GarbageCollectionEnvironment gce) throws TableNotFoundException, AccumuloException, AccumuloSecurityException, IOException {
String lastCandidate = "";
while (true) {
- Span candidatesSpan = Trace.start("getCandidates");
- List<String> candidates;
- try {
- candidates = gce.getCandidates(lastCandidate);
- } finally {
- candidatesSpan.stop();
- }
+ List<String> candidates = getCandidates(gce, lastCandidate);
if (candidates.size() == 0)
break;
@@ -250,23 +278,10 @@ public class GarbageCollectionAlgorithm {
SortedMap<String,String> candidateMap = makeRelative(candidates);
- Span confirmDeletesSpan = Trace.start("confirmDeletes");
- try {
- confirmDeletes(gce, candidateMap);
- } finally {
- confirmDeletesSpan.stop();
- }
+ confirmDeletesTrace(gce, candidateMap);
gce.incrementInUseStat(origSize - candidateMap.size());
- Span deleteSpan = Trace.start("deleteFiles");
- try {
- gce.delete(candidateMap);
- } finally {
- deleteSpan.stop();
- }
-
- cleanUpDeletedTableDirs(gce, candidateMap);
+ deleteConfirmed(gce, candidateMap);
}
}
-
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cc7b7ef6/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectionEnvironment.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectionEnvironment.java b/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectionEnvironment.java
index fa35ed3..1d39254 100644
--- a/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectionEnvironment.java
+++ b/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectionEnvironment.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.server.gc;
import java.io.IOException;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedMap;
@@ -28,6 +29,8 @@ import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
@@ -37,9 +40,11 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.Sc
public interface GarbageCollectionEnvironment {
/**
- * Return a list of paths to files which are candidates for deletion from a given table, {@link RootTable.NAME} or {@link MetadataTable.NAME}
- * @param continuePoint A row to resume from if a previous invocation was stopped due to finding an extremely large number
- * of candidates to remove which would have exceeded memory limitations
+ * Return a list of paths to files and dirs which are candidates for deletion from a given table, {@link RootTable.NAME} or {@link MetadataTable.NAME}
+ *
+ * @param continuePoint
+ * A row to resume from if a previous invocation was stopped due to finding an extremely large number of candidates to remove which would have
+ * exceeded memory limitations
* @return A collection of candidates files for deletion, may not be the complete collection of files for deletion at this point in time
* @throws TableNotFoundException
* @throws AccumuloException
@@ -49,6 +54,7 @@ public interface GarbageCollectionEnvironment {
/**
* Fetch a list of paths for all bulk loads in progress (blip) from a given table, {@link RootTable.NAME} or {@link MetadataTable.NAME}
+ *
* @return The list of files for each bulk load currently in progress.
* @throws TableNotFoundException
* @throws AccumuloException
@@ -58,6 +64,7 @@ public interface GarbageCollectionEnvironment {
/**
* Fetches the references to files, {@link DataFileColumnFamily.NAME} or {@link ScanFileColumnFamily.NAME}, from tablets
+ *
* @return An Iterator to the @{link Entry<Key,Value>}s which constitute a reference to a file.
* @throws TableNotFoundException
* @throws AccumuloException
@@ -67,33 +74,45 @@ public interface GarbageCollectionEnvironment {
/**
* Return the set of tableIDs for the given instance this GarbageCollector is running over
+ *
* @return The valueSet for the table name to table id map.
*/
Set<String> getTableIDs();
/**
* Delete the given files from the provided {@link Map} of relative path to absolute path for each file that should be deleted
- * @param candidateMap A Map from relative path to absolute path for files to be deleted.
+ *
+ * @param candidateMap
+ * A Map from relative path to absolute path for files to be deleted.
* @throws IOException
+ * @throws AccumuloSecurityException
+ * @throws AccumuloException
+ * @throws TableNotFoundException
*/
- void delete(SortedMap<String,String> candidateMap) throws IOException;
+ void delete(SortedMap<String,String> candidateMap) throws IOException, AccumuloException, AccumuloSecurityException, TableNotFoundException;
/**
* Delete a table's directory if it is empty.
- * @param tableID The id of the table whose directory we are to operate on
+ *
+ * @param tableID
+ * The id of the table whose directory we are to operate on
* @throws IOException
*/
void deleteTableDirIfEmpty(String tableID) throws IOException;
/**
* Increment the number of candidates for deletion for the current garbage collection run
- * @param i Value to increment the deletion candidates by
+ *
+ * @param i
+ * Value to increment the deletion candidates by
*/
void incrementCandidatesStat(long i);
/**
* Increment the number of files still in use for the current garbage collection run
- * @param i Value to increment the still-in-use count by.
+ *
+ * @param i
+ * Value to increment the still-in-use count by.
*/
void incrementInUseStat(long i);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cc7b7ef6/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java b/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
index 2619a1c..6dd9d2c 100644
--- a/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
+++ b/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
@@ -238,7 +238,7 @@ public class SimpleGarbageCollector implements Iface {
}
@Override
- public void delete(SortedMap<String,String> confirmedDeletes) throws IOException {
+ public void delete(SortedMap<String,String> confirmedDeletes) throws IOException, AccumuloException, AccumuloSecurityException, TableNotFoundException {
if (opts.safeMode) {
if (opts.verbose)
@@ -251,23 +251,8 @@ public class SimpleGarbageCollector implements Iface {
return;
}
- // create a batchwriter to remove the delete flags for successful
- // deletes; Need separate writer for the root tablet.
- BatchWriter writer = null;
- Connector c;
- try {
- c = instance.getConnector(SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken());
- writer = c.createBatchWriter(tableName, new BatchWriterConfig());
- } catch (AccumuloException e) {
- log.error("Unable to connect to Accumulo to write deletes", e);
- // TODO Throw exception or return?
- } catch (AccumuloSecurityException e) {
- log.error("Unable to connect to Accumulo to write deletes", e);
- // TODO Throw exception or return?
- } catch (TableNotFoundException e) {
- log.error("Unable to create writer to remove file from the " + e.getTableName() + " table", e);
- // TODO Throw exception or return?
- }
+ Connector c = instance.getConnector(SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken());
+ BatchWriter writer = c.createBatchWriter(tableName, new BatchWriterConfig());
// when deleting a dir and all files in that dir, only need to delete the dir
// the dir will sort right before the files... so remove the files in this case