[GitHub] keith-turner closed pull request #503: Fix bug with GC not scanning root table for file refs

[GitBox <gi...@apache.org>]

keith-turner closed pull request #503: Fix bug with GC not scanning root table for file refs
URL: https://github.com/apache/accumulo/pull/503
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataScanner.java b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataScanner.java
index 6fc05bb538..573d3c4633 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataScanner.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataScanner.java
@@ -41,6 +41,7 @@
 import org.apache.accumulo.core.data.Range;
 import org.apache.accumulo.core.data.impl.KeyExtent;
 import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.BulkFileColumnFamily;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ClonedColumnFamily;
@@ -62,10 +63,11 @@
     TableOptions from(Connector conn);
   }
 
-  public interface TableOptions extends RangeOptions {
-    /**
-     * Optionally set a table name, defaults to {@value MetadataTable#NAME}
-     */
+  public interface TableOptions {
+    RangeOptions scanRootTable();
+
+    RangeOptions scanMetadataTable();
+
     RangeOptions scanTable(String tableName);
   }
 
@@ -310,6 +312,16 @@ public RangeOptions scanTable(String tableName) {
       this.table = tableName;
       return this;
     }
+
+    @Override
+    public RangeOptions scanMetadataTable() {
+      return scanTable(MetadataTable.NAME);
+    }
+
+    @Override
+    public RangeOptions scanRootTable() {
+      return scanTable(RootTable.NAME);
+    }
   }
 
   private Scanner scanner;
diff --git a/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java b/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java
index 0799510f7a..eaae24d416 100644
--- a/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java
+++ b/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java
@@ -160,7 +160,7 @@ private TSummaryRequest getRequest() {
       Predicate<String> fileSelector)
       throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
 
-    Iterable<TabletMetadata> tmi = MetadataScanner.builder().from(ctx)
+    Iterable<TabletMetadata> tmi = MetadataScanner.builder().from(ctx).scanMetadataTable()
         .overRange(tableId, startRow, endRow).fetchFiles().fetchLocation().fetchLast().fetchPrev()
         .build();
 
@@ -522,8 +522,9 @@ public SummaryCollection get(long timeout, TimeUnit unit)
   private int countFiles()
       throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
     // TODO use a batch scanner + iterator to parallelize counting files
-    return MetadataScanner.builder().from(ctx).overRange(tableId, startRow, endRow).fetchFiles()
-        .fetchPrev().build().stream().mapToInt(tm -> tm.getFiles().size()).sum();
+    return MetadataScanner.builder().from(ctx).scanMetadataTable()
+        .overRange(tableId, startRow, endRow).fetchFiles().fetchPrev().build().stream()
+        .mapToInt(tm -> tm.getFiles().size()).sum();
   }
 
   private class GatherRequest implements Supplier<SummaryCollection> {
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index 3c60861e5a..6049b425cf 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@ -300,8 +300,8 @@ public boolean getCandidates(String continuePoint, List<String> result)
         throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
 
       Stream<TabletMetadata> tabletStream = MetadataScanner.builder().from(getConnector())
-          .overTabletRange().checkConsistency().fetchDir().fetchFiles().fetchScans().build()
-          .stream();
+          .scanTable(tableName).overTabletRange().checkConsistency().fetchDir().fetchFiles()
+          .fetchScans().build().stream();
 
       Stream<Reference> refStream = tabletStream.flatMap(tm -> {
         Stream<Reference> refs = Stream.concat(tm.getFiles().stream(), tm.getScans().stream())
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java
index 1baac994b4..390a3ea72c 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java
@@ -108,7 +108,7 @@ private long loadFiles(Table.ID tableId, Path bulkDir, LoadMappingIterator lmi,
     Text startRow = loadMapEntry.getKey().getPrevEndRow();
 
     long timeInMillis = master.getConfiguration().getTimeInMillis(Property.MASTER_BULK_TIMEOUT);
-    Iterator<TabletMetadata> tabletIter = MetadataScanner.builder().from(master)
+    Iterator<TabletMetadata> tabletIter = MetadataScanner.builder().from(master).scanMetadataTable()
         .overRange(tableId, startRow, null).checkConsistency().fetchPrev().fetchLocation()
         .fetchLoaded().build().iterator();
 
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/PrepBulkImport.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/PrepBulkImport.java
index 18ddab9f44..bca1389411 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/PrepBulkImport.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/PrepBulkImport.java
@@ -166,9 +166,9 @@ private void checkForMerge(final Master master) throws Exception {
       Iterators.transform(lmi, entry -> entry.getKey());
 
       TabletIterFactory tabletIterFactory = startRow -> {
-        return MetadataScanner.builder().from(master).overRange(bulkInfo.tableId, startRow, null)
-            .checkConsistency().fetchPrev().build().stream().map(TabletMetadata::getExtent)
-            .iterator();
+        return MetadataScanner.builder().from(master).scanMetadataTable()
+            .overRange(bulkInfo.tableId, startRow, null).checkConsistency().fetchPrev().build()
+            .stream().map(TabletMetadata::getExtent).iterator();
       };
 
       checkForMerge(bulkInfo.tableId.canonicalID(),


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services