You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2015/10/20 22:15:38 UTC
[1/3] accumulo git commit: ACCUMULO-4021 explicitly return a value
that marks the end of GC
Repository: accumulo
Updated Branches:
refs/heads/master 52ba24e32 -> 39f14472b
ACCUMULO-4021 explicitly return a value that marks the end of GC
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/deef5352
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/deef5352
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/deef5352
Branch: refs/heads/master
Commit: deef5352db0bec4b1578568c3da84220c43038a0
Parents: 5ca779a
Author: Eric C. Newton <er...@gmail.com>
Authored: Tue Oct 20 16:14:35 2015 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Tue Oct 20 16:14:35 2015 -0400
----------------------------------------------------------------------
.../accumulo/gc/GarbageCollectionAlgorithm.java | 16 +++++++++-------
.../accumulo/gc/GarbageCollectionEnvironment.java | 6 ++++--
.../apache/accumulo/gc/SimpleGarbageCollector.java | 10 ++++------
.../apache/accumulo/gc/GarbageCollectionTest.java | 5 ++---
4 files changed, 19 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/deef5352/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
index 0e016d5..a8cde31 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
@@ -113,7 +113,7 @@ public class GarbageCollectionAlgorithm {
try {
relPath = makeRelative(candidate, 0);
} catch (IllegalArgumentException iae) {
- log.warn("Ingoring invalid deletion candidate " + candidate);
+ log.warn("Ignoring invalid deletion candidate " + candidate);
continue;
}
ret.put(relPath, candidate);
@@ -230,16 +230,15 @@ public class GarbageCollectionAlgorithm {
}
- private List<String> getCandidates(GarbageCollectionEnvironment gce, String lastCandidate) throws TableNotFoundException, AccumuloException,
+ private boolean getCandidates(GarbageCollectionEnvironment gce, String lastCandidate, List<String> candidates)
+ throws TableNotFoundException, AccumuloException,
AccumuloSecurityException {
Span candidatesSpan = Trace.start("getCandidates");
- List<String> candidates;
try {
- candidates = gce.getCandidates(lastCandidate);
+ return gce.getCandidates(lastCandidate, candidates);
} finally {
candidatesSpan.stop();
}
- return candidates;
}
private void confirmDeletesTrace(GarbageCollectionEnvironment gce, SortedMap<String,String> candidateMap) throws TableNotFoundException, AccumuloException,
@@ -268,8 +267,11 @@ public class GarbageCollectionAlgorithm {
String lastCandidate = "";
- while (true) {
- List<String> candidates = getCandidates(gce, lastCandidate);
+ boolean outOfMemory = true;
+ while (outOfMemory) {
+ List<String> candidates = new ArrayList<String>();
+
+ outOfMemory = getCandidates(gce, lastCandidate, candidates);
if (candidates.size() == 0)
break;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/deef5352/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java
index cca2411..4eab25a 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java
@@ -45,9 +45,11 @@ public interface GarbageCollectionEnvironment {
* @param continuePoint
* A row to resume from if a previous invocation was stopped due to finding an extremely large number of candidates to remove which would have
* exceeded memory limitations
- * @return A collection of candidates files for deletion, may not be the complete collection of files for deletion at this point in time
+ * @param candidates
+ * A collection of candidates files for deletion, may not be the complete collection of files for deletion at this point in time
+ * @return true if the results are short due to insufficient memory, otherwise false
*/
- List<String> getCandidates(String continuePoint) throws TableNotFoundException, AccumuloException, AccumuloSecurityException;
+ boolean getCandidates(String continuePoint, List<String> candidates) throws TableNotFoundException, AccumuloException, AccumuloSecurityException;
/**
* Fetch a list of paths for all bulk loads in progress (blip) from a given table, {@link RootTable#NAME} or {@link MetadataTable#NAME}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/deef5352/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index 8761480..3ab95a7 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@ -19,7 +19,6 @@ package org.apache.accumulo.gc;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.UnknownHostException;
-import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -269,7 +268,7 @@ public class SimpleGarbageCollector implements Iface {
}
@Override
- public List<String> getCandidates(String continuePoint) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
+ public boolean getCandidates(String continuePoint, List<String> result) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
// want to ensure GC makes progress... if the 1st N deletes are stable and we keep processing them,
// then will never inspect deletes after N
Range range = MetadataSchema.DeletesSection.getRange();
@@ -280,19 +279,18 @@ public class SimpleGarbageCollector implements Iface {
Scanner scanner = instance.getConnector(credentials.getPrincipal(), credentials.getToken()).createScanner(tableName, Authorizations.EMPTY);
scanner.setRange(range);
- List<String> result = new ArrayList<String>();
+ result.clear();
// find candidates for deletion; chop off the prefix
for (Entry<Key,Value> entry : scanner) {
String cand = entry.getKey().getRow().toString().substring(MetadataSchema.DeletesSection.getRowPrefix().length());
result.add(cand);
if (almostOutOfMemory(Runtime.getRuntime())) {
log.info("List of delete candidates has exceeded the memory threshold. Attempting to delete what has been gathered so far.");
- break;
+ return true;
}
}
- return result;
-
+ return false;
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/deef5352/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
----------------------------------------------------------------------
diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
index 50c3957..2953d07 100644
--- a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
+++ b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
@@ -51,14 +51,13 @@ public class GarbageCollectionTest {
ArrayList<String> tablesDirsToDelete = new ArrayList<String>();
@Override
- public List<String> getCandidates(String continuePoint) {
+ public boolean getCandidates(String continuePoint, List<String> ret) {
Iterator<String> iter = candidates.tailSet(continuePoint, false).iterator();
- ArrayList<String> ret = new ArrayList<String>();
while (iter.hasNext() && ret.size() < 3) {
ret.add(iter.next());
}
- return ret;
+ return ret.size() == 3;
}
@Override
[3/3] accumulo git commit: Merge branch '1.7'
Posted by ec...@apache.org.
Merge branch '1.7'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/39f14472
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/39f14472
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/39f14472
Branch: refs/heads/master
Commit: 39f14472b109170c8434de0b7d4cadf7473dbd1f
Parents: 52ba24e 89d69f6
Author: Eric C. Newton <er...@gmail.com>
Authored: Tue Oct 20 16:15:07 2015 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Tue Oct 20 16:15:07 2015 -0400
----------------------------------------------------------------------
.../accumulo/gc/GarbageCollectionAlgorithm.java | 16 +++++++++-------
.../accumulo/gc/GarbageCollectionEnvironment.java | 6 ++++--
.../apache/accumulo/gc/SimpleGarbageCollector.java | 10 ++++------
.../apache/accumulo/gc/GarbageCollectionTest.java | 5 ++---
4 files changed, 19 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/39f14472/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
----------------------------------------------------------------------
[2/3] accumulo git commit: Merge branch '1.6' into 1.7
Posted by ec...@apache.org.
Merge branch '1.6' into 1.7
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/89d69f60
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/89d69f60
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/89d69f60
Branch: refs/heads/master
Commit: 89d69f60ce78f910158995fbe1c9eb89a47090e4
Parents: 1f28ee7 deef535
Author: Eric C. Newton <er...@gmail.com>
Authored: Tue Oct 20 16:14:56 2015 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Tue Oct 20 16:14:56 2015 -0400
----------------------------------------------------------------------
.../accumulo/gc/GarbageCollectionAlgorithm.java | 16 +++++++++-------
.../accumulo/gc/GarbageCollectionEnvironment.java | 6 ++++--
.../apache/accumulo/gc/SimpleGarbageCollector.java | 10 ++++------
.../apache/accumulo/gc/GarbageCollectionTest.java | 5 ++---
4 files changed, 19 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/89d69f60/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/89d69f60/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/89d69f60/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
----------------------------------------------------------------------
diff --cc server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index 037023a,3ab95a7..1daafcb
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@@ -248,9 -277,9 +247,9 @@@ public class SimpleGarbageCollector ext
range = new Range(new Key(continueRow).followingKey(PartialKey.ROW), true, range.getEndKey(), range.isEndKeyInclusive());
}
- Scanner scanner = instance.getConnector(credentials.getPrincipal(), credentials.getToken()).createScanner(tableName, Authorizations.EMPTY);
+ Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY);
scanner.setRange(range);
- List<String> result = new ArrayList<String>();
+ result.clear();
// find candidates for deletion; chop off the prefix
for (Entry<Key,Value> entry : scanner) {
String cand = entry.getKey().getRow().toString().substring(MetadataSchema.DeletesSection.getRowPrefix().length());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/89d69f60/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
----------------------------------------------------------------------
diff --cc server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
index 544bd69,2953d07..1548fa1
--- a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
+++ b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
@@@ -53,12 -49,10 +53,11 @@@ public class GarbageCollectionTest
ArrayList<String> deletes = new ArrayList<String>();
ArrayList<String> tablesDirsToDelete = new ArrayList<String>();
+ TreeMap<String,Status> filesToReplicate = new TreeMap<String,Status>();
@Override
- public List<String> getCandidates(String continuePoint) {
+ public boolean getCandidates(String continuePoint, List<String> ret) {
Iterator<String> iter = candidates.tailSet(continuePoint, false).iterator();
- ArrayList<String> ret = new ArrayList<String>();
while (iter.hasNext() && ret.size() < 3) {
ret.add(iter.next());
}