You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2018/05/30 12:40:01 UTC
[accumulo] branch master updated: #469 Fix bug with GC not scanning
root table for file refs (#503)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/master by this push:
new 780c4ca #469 Fix bug with GC not scanning root table for file refs (#503)
780c4ca is described below
commit 780c4cab06597547d028ed59abdaa967a1c7900a
Author: Keith Turner <ke...@deenlo.com>
AuthorDate: Wed May 30 08:39:42 2018 -0400
#469 Fix bug with GC not scanning root table for file refs (#503)
This bug was introduced in 9feb5e1. In addition to fixing the bug,
specifying the table was made mandatory for the metadata scanner
builder.
The bug was found by MetadataMaxFilesIT
---
.../core/metadata/schema/MetadataScanner.java | 20 ++++++++++++++++----
.../org/apache/accumulo/core/summary/Gatherer.java | 7 ++++---
.../apache/accumulo/gc/SimpleGarbageCollector.java | 4 ++--
.../accumulo/master/tableOps/bulkVer2/LoadFiles.java | 2 +-
.../master/tableOps/bulkVer2/PrepBulkImport.java | 6 +++---
5 files changed, 26 insertions(+), 13 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataScanner.java b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataScanner.java
index 6fc05bb..573d3c4 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataScanner.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataScanner.java
@@ -41,6 +41,7 @@ import org.apache.accumulo.core.client.impl.Table.ID;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.impl.KeyExtent;
import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.BulkFileColumnFamily;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ClonedColumnFamily;
@@ -62,10 +63,11 @@ public class MetadataScanner implements Iterable<TabletMetadata>, AutoCloseable
TableOptions from(Connector conn);
}
- public interface TableOptions extends RangeOptions {
- /**
- * Optionally set a table name, defaults to {@value MetadataTable#NAME}
- */
+ public interface TableOptions {
+ RangeOptions scanRootTable();
+
+ RangeOptions scanMetadataTable();
+
RangeOptions scanTable(String tableName);
}
@@ -310,6 +312,16 @@ public class MetadataScanner implements Iterable<TabletMetadata>, AutoCloseable
this.table = tableName;
return this;
}
+
+ @Override
+ public RangeOptions scanMetadataTable() {
+ return scanTable(MetadataTable.NAME);
+ }
+
+ @Override
+ public RangeOptions scanRootTable() {
+ return scanTable(RootTable.NAME);
+ }
}
private Scanner scanner;
diff --git a/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java b/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java
index 0799510..eaae24d 100644
--- a/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java
+++ b/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java
@@ -160,7 +160,7 @@ public class Gatherer {
Predicate<String> fileSelector)
throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
- Iterable<TabletMetadata> tmi = MetadataScanner.builder().from(ctx)
+ Iterable<TabletMetadata> tmi = MetadataScanner.builder().from(ctx).scanMetadataTable()
.overRange(tableId, startRow, endRow).fetchFiles().fetchLocation().fetchLast().fetchPrev()
.build();
@@ -522,8 +522,9 @@ public class Gatherer {
private int countFiles()
throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
// TODO use a batch scanner + iterator to parallelize counting files
- return MetadataScanner.builder().from(ctx).overRange(tableId, startRow, endRow).fetchFiles()
- .fetchPrev().build().stream().mapToInt(tm -> tm.getFiles().size()).sum();
+ return MetadataScanner.builder().from(ctx).scanMetadataTable()
+ .overRange(tableId, startRow, endRow).fetchFiles().fetchPrev().build().stream()
+ .mapToInt(tm -> tm.getFiles().size()).sum();
}
private class GatherRequest implements Supplier<SummaryCollection> {
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index 3c60861..6049b42 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@ -300,8 +300,8 @@ public class SimpleGarbageCollector extends AccumuloServerContext implements Ifa
throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
Stream<TabletMetadata> tabletStream = MetadataScanner.builder().from(getConnector())
- .overTabletRange().checkConsistency().fetchDir().fetchFiles().fetchScans().build()
- .stream();
+ .scanTable(tableName).overTabletRange().checkConsistency().fetchDir().fetchFiles()
+ .fetchScans().build().stream();
Stream<Reference> refStream = tabletStream.flatMap(tm -> {
Stream<Reference> refs = Stream.concat(tm.getFiles().stream(), tm.getScans().stream())
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java
index 1baac99..390a3ea 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java
@@ -108,7 +108,7 @@ class LoadFiles extends MasterRepo {
Text startRow = loadMapEntry.getKey().getPrevEndRow();
long timeInMillis = master.getConfiguration().getTimeInMillis(Property.MASTER_BULK_TIMEOUT);
- Iterator<TabletMetadata> tabletIter = MetadataScanner.builder().from(master)
+ Iterator<TabletMetadata> tabletIter = MetadataScanner.builder().from(master).scanMetadataTable()
.overRange(tableId, startRow, null).checkConsistency().fetchPrev().fetchLocation()
.fetchLoaded().build().iterator();
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/PrepBulkImport.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/PrepBulkImport.java
index 18ddab9..bca1389 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/PrepBulkImport.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/PrepBulkImport.java
@@ -166,9 +166,9 @@ public class PrepBulkImport extends MasterRepo {
Iterators.transform(lmi, entry -> entry.getKey());
TabletIterFactory tabletIterFactory = startRow -> {
- return MetadataScanner.builder().from(master).overRange(bulkInfo.tableId, startRow, null)
- .checkConsistency().fetchPrev().build().stream().map(TabletMetadata::getExtent)
- .iterator();
+ return MetadataScanner.builder().from(master).scanMetadataTable()
+ .overRange(bulkInfo.tableId, startRow, null).checkConsistency().fetchPrev().build()
+ .stream().map(TabletMetadata::getExtent).iterator();
};
checkForMerge(bulkInfo.tableId.canonicalID(),
--
To stop receiving notification emails like this one, please contact
kturner@apache.org.