You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2021/07/19 15:53:16 UTC
[accumulo] branch main updated: Make Ample only return exact
matches for KeyExtents (#2189)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 548b037 Make Ample only return exact matches for KeyExtents (#2189)
548b037 is described below
commit 548b037c5917dc1e693733dc3d89599369d79103
Author: Dom G <47...@users.noreply.github.com>
AuthorDate: Mon Jul 19 11:53:05 2021 -0400
Make Ample only return exact matches for KeyExtents (#2189)
---
.../core/metadata/schema/TabletsMetadata.java | 69 +++++++++++++++-------
.../accumulo/server/util/MetadataTableUtil.java | 5 +-
.../org/apache/accumulo/compactor/Compactor.java | 3 +-
.../org/apache/accumulo/tserver/TabletServer.java | 5 ++
.../org/apache/accumulo/tserver/tablet/Tablet.java | 5 +-
5 files changed, 58 insertions(+), 29 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
index 042786c..8ffa08e 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
@@ -36,6 +36,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Objects;
+import java.util.Set;
import java.util.function.Function;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
@@ -87,8 +88,9 @@ public class TabletsMetadata implements Iterable<TabletMetadata>, AutoCloseable
public static class Builder implements TableRangeOptions, TableOptions, RangeOptions, Options {
- private List<Text> families = new ArrayList<>();
- private List<ColumnFQ> qualifiers = new ArrayList<>();
+ private final List<Text> families = new ArrayList<>();
+ private final List<ColumnFQ> qualifiers = new ArrayList<>();
+ private Set<KeyExtent> extentsToFetch = null;
private Ample.DataLevel level;
private String table;
private Range range;
@@ -98,8 +100,8 @@ public class TabletsMetadata implements Iterable<TabletMetadata>, AutoCloseable
private boolean saveKeyValues;
private TableId tableId;
private ReadConsistency readConsistency = ReadConsistency.IMMEDIATE;
- private AccumuloClient _client;
- private Collection<KeyExtent> extents;
+ private final AccumuloClient _client;
+ private Collection<KeyExtent> extents = null;
Builder(AccumuloClient client) {
this._client = client;
@@ -132,19 +134,33 @@ public class TabletsMetadata implements Iterable<TabletMetadata>, AutoCloseable
var ranges = extents.stream().map(KeyExtent::toMetaRange).collect(toList());
scanner.setRanges(ranges);
+ boolean extentsPresent = extentsToFetch != null;
+
+ if (!fetchedCols.isEmpty() && extentsPresent)
+ fetch(ColumnType.PREV_ROW);
+
configureColumns(scanner);
IteratorSetting iterSetting = new IteratorSetting(100, WholeRowIterator.class);
scanner.addScanIterator(iterSetting);
- Iterable<TabletMetadata> tmi = () -> Iterators.transform(scanner.iterator(), entry -> {
- try {
- return TabletMetadata.convertRow(
- WholeRowIterator.decodeRow(entry.getKey(), entry.getValue()).entrySet().iterator(),
- fetchedCols, saveKeyValues);
- } catch (IOException e) {
- throw new UncheckedIOException(e);
+ Iterable<TabletMetadata> tmi = () -> {
+ Iterator<TabletMetadata> iter = Iterators.transform(scanner.iterator(), entry -> {
+ try {
+ return TabletMetadata.convertRow(WholeRowIterator
+ .decodeRow(entry.getKey(), entry.getValue()).entrySet().iterator(), fetchedCols,
+ saveKeyValues);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ });
+
+ if (extentsPresent) {
+ return Iterators.filter(iter,
+ tabletMetadata -> extentsToFetch.contains(tabletMetadata.getExtent()));
+ } else {
+ return iter;
}
- });
+ };
return new TabletsMetadata(scanner, tmi);
} catch (TableNotFoundException e) {
@@ -161,7 +177,9 @@ public class TabletsMetadata implements Iterable<TabletMetadata>, AutoCloseable
new IsolatedScanner(client.createScanner(resolvedTable, Authorizations.EMPTY));
scanner.setRange(range);
- if (checkConsistency && !fetchedCols.contains(ColumnType.PREV_ROW)) {
+ boolean extentsPresent = extentsToFetch != null;
+
+ if (!fetchedCols.isEmpty() && (checkConsistency || extentsPresent)) {
fetch(ColumnType.PREV_ROW);
}
@@ -172,8 +190,14 @@ public class TabletsMetadata implements Iterable<TabletMetadata>, AutoCloseable
synchronized (scanner) {
scanner.setRange(r);
RowIterator rowIter = new RowIterator(scanner);
- return Iterators.transform(rowIter,
+ Iterator<TabletMetadata> iter = Iterators.transform(rowIter,
ri -> TabletMetadata.convertRow(ri, fetchedCols, saveKeyValues));
+ if (extentsPresent) {
+ return Iterators.filter(iter,
+ tabletMetadata -> extentsToFetch.contains(tabletMetadata.getExtent()));
+ } else {
+ return iter;
+ }
}
};
@@ -291,6 +315,7 @@ public class TabletsMetadata implements Iterable<TabletMetadata>, AutoCloseable
public Options forTablet(KeyExtent extent) {
forTable(extent.tableId());
this.range = new Range(extent.toMetaRow());
+ this.extentsToFetch = Set.of(extent);
return this;
}
@@ -304,6 +329,7 @@ public class TabletsMetadata implements Iterable<TabletMetadata>, AutoCloseable
this.level = DataLevel.USER;
this.extents = extents;
+ this.extentsToFetch = Set.copyOf(extents);
return this;
}
@@ -376,14 +402,13 @@ public class TabletsMetadata implements Iterable<TabletMetadata>, AutoCloseable
/**
* Get the tablet metadata for this extents end row. This should only ever return a single
- * tablet. No checking is done for prev row, so it could differ.
+ * tablet where the end row and prev end row exactly match the given extent.
*/
Options forTablet(KeyExtent extent);
/**
- * Get the tablet metadata for the given extents. This will find tablets based on end row, so
- * it's possible the prev rows could differ for the tablets returned. If this matters, then it
- * must be checked.
+ * Get the tablet metadata for the given extents. This will only return tablets where the end
+ * row and prev end row exactly match the given extents.
*/
Options forTablets(Collection<KeyExtent> extents);
@@ -431,8 +456,8 @@ public class TabletsMetadata implements Iterable<TabletMetadata>, AutoCloseable
private static class TabletMetadataIterator implements Iterator<TabletMetadata> {
private boolean sawLast = false;
- private Iterator<TabletMetadata> iter;
- private Text endRow;
+ private final Iterator<TabletMetadata> iter;
+ private final Text endRow;
TabletMetadataIterator(Iterator<TabletMetadata> source, Text endRow) {
this.iter = source;
@@ -487,9 +512,9 @@ public class TabletsMetadata implements Iterable<TabletMetadata>, AutoCloseable
.convertToTabletMetadata();
}
- private ScannerBase scanner;
+ private final ScannerBase scanner;
- private Iterable<TabletMetadata> tablets;
+ private final Iterable<TabletMetadata> tablets;
private TabletsMetadata(TabletMetadata tm) {
this.scanner = null;
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index 709cc7f..591cf5a 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -418,8 +418,9 @@ public class MetadataTableUtil {
TabletMetadata tablet = context.getAmple().readTablet(extent, FILES, LOGS, PREV_ROW, DIR);
- if (!tablet.getExtent().equals(extent))
- throw new RuntimeException("Unexpected extent " + tablet.getExtent() + " expected " + extent);
+ if (tablet == null) {
+ throw new RuntimeException("Tablet " + extent + " not found in metadata");
+ }
result.addAll(tablet.getLogs());
diff --git a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
index 41865f0..b63e1bd 100644
--- a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
+++ b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
@@ -190,8 +190,7 @@ public class Compactor extends AbstractServer implements CompactorService.Iface
TabletMetadata tabletMeta =
getContext().getAmple().readTablet(extent, ColumnType.ECOMP, ColumnType.PREV_ROW);
- if (tabletMeta == null || !tabletMeta.getExtent().equals(extent)
- || !tabletMeta.getExternalCompactions().containsKey(ecid)) {
+ if (tabletMeta == null || !tabletMeta.getExternalCompactions().containsKey(ecid)) {
// table was deleted OR tablet was split or merged OR tablet no longer thinks compaction
// is running for some reason
LOG.info("Cancelling compaction {} that no longer has a metadata entry at {}", ecid,
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index c4bb1eb..1341838 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -926,6 +926,11 @@ public class TabletServer extends AbstractServer {
static boolean checkTabletMetadata(KeyExtent extent, TServerInstance instance,
TabletMetadata meta) throws AccumuloException {
+ if (meta == null) {
+ log.info("Not loading tablet {}, its metadata was not found.", extent);
+ return false;
+ }
+
if (!meta.sawPrevEndRow()) {
throw new AccumuloException("Metadata entry does not have prev row (" + meta.getTableId()
+ " " + meta.getEndRow() + ")");
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 522efff..ad93e70 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -1361,9 +1361,8 @@ public class Tablet {
var tabletMeta = context.getAmple().readTablet(extent, ColumnType.FILES, ColumnType.LOGS,
ColumnType.ECOMP, ColumnType.PREV_ROW);
- if (!tabletMeta.getExtent().equals(extent)) {
- String msg = "Closed tablet " + extent + " does not match extent in metadata table "
- + tabletMeta.getExtent();
+ if (tabletMeta == null) {
+ String msg = "Closed tablet " + extent + " not found in metadata";
log.error(msg);
throw new RuntimeException(msg);
}