You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by dl...@apache.org on 2023/05/02 17:24:10 UTC
[accumulo] branch elasticity updated: Set the TabletHostingGoal during tablet merge (#3335)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 1cd934e3e4 Set the TabletHostingGoal during tablet merge (#3335)
1cd934e3e4 is described below
commit 1cd934e3e45bfc85709f44787e09634723e53c2b
Author: Dave Marion <dl...@apache.org>
AuthorDate: Tue May 2 13:24:05 2023 -0400
Set the TabletHostingGoal during tablet merge (#3335)
* Set the TabletHostingGoal during tablet merge
Closes #3307
---
.../accumulo/manager/TabletGroupWatcher.java | 22 ++++++++++++++++
.../apache/accumulo/test/functional/MergeIT.java | 30 ++++++++++++++++++++++
2 files changed, 52 insertions(+)
diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
index a4a76083c9..431f097875 100644
--- a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
@@ -44,6 +44,8 @@ import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.RowIterator;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.TabletHostingGoal;
+import org.apache.accumulo.core.clientImpl.TabletHostingGoalUtil;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
@@ -71,6 +73,7 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.Cu
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.FutureLocationColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.HostingColumnFamily;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
import org.apache.accumulo.core.metadata.schema.MetadataTime;
@@ -726,6 +729,7 @@ abstract class TabletGroupWatcher extends AccumuloDaemonThread {
if (range.isMeta()) {
targetSystemTable = RootTable.NAME;
}
+ Set<TabletHostingGoal> goals = new HashSet<>();
AccumuloClient client = manager.getContext();
@@ -737,6 +741,7 @@ abstract class TabletGroupWatcher extends AccumuloDaemonThread {
TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
ServerColumnFamily.TIME_COLUMN.fetch(scanner);
ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner);
+ HostingColumnFamily.GOAL_COLUMN.fetch(scanner);
scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
Mutation m = new Mutation(stopRow);
MetadataTime maxLogicalTime = null;
@@ -756,6 +761,9 @@ abstract class TabletGroupWatcher extends AccumuloDaemonThread {
} else if (ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(key)) {
var allVolumesDir = new AllVolumesDirectory(range.tableId(), value.toString());
bw.addMutation(manager.getContext().getAmple().createDeleteMutation(allVolumesDir));
+ } else if (HostingColumnFamily.GOAL_COLUMN.hasColumns(key)) {
+ TabletHostingGoal thisGoal = TabletHostingGoalUtil.fromValue(value);
+ goals.add(thisGoal);
}
}
@@ -764,6 +772,7 @@ abstract class TabletGroupWatcher extends AccumuloDaemonThread {
scanner = client.createScanner(targetSystemTable, Authorizations.EMPTY);
scanner.setRange(new Range(stopRow));
ServerColumnFamily.TIME_COLUMN.fetch(scanner);
+ HostingColumnFamily.GOAL_COLUMN.fetch(scanner);
scanner.fetchColumnFamily(ExternalCompactionColumnFamily.NAME);
Set<String> extCompIds = new HashSet<>();
for (Entry<Key,Value> entry : scanner) {
@@ -772,6 +781,9 @@ abstract class TabletGroupWatcher extends AccumuloDaemonThread {
MetadataTime.parse(entry.getValue().toString()));
} else if (ExternalCompactionColumnFamily.NAME.equals(entry.getKey().getColumnFamily())) {
extCompIds.add(entry.getKey().getColumnQualifierData().toString());
+ } else if (HostingColumnFamily.GOAL_COLUMN.hasColumns(entry.getKey())) {
+ TabletHostingGoal thisGoal = TabletHostingGoalUtil.fromValue(entry.getValue());
+ goals.add(thisGoal);
}
}
@@ -782,6 +794,16 @@ abstract class TabletGroupWatcher extends AccumuloDaemonThread {
// delete any entries for external compactions
extCompIds.forEach(ecid -> m.putDelete(ExternalCompactionColumnFamily.STR_NAME, ecid));
+ // Set the TabletHostingGoal for this tablet based on the goals of the other tablets in
+ // the merge range. Always takes priority over never.
+ TabletHostingGoal mergeHostingGoal = TabletHostingGoal.ONDEMAND;
+ if (range.isMeta() || goals.contains(TabletHostingGoal.ALWAYS)) {
+ mergeHostingGoal = TabletHostingGoal.ALWAYS;
+ } else if (goals.contains(TabletHostingGoal.NEVER)) {
+ mergeHostingGoal = TabletHostingGoal.NEVER;
+ }
+ HostingColumnFamily.GOAL_COLUMN.put(m, TabletHostingGoalUtil.toValue(mergeHostingGoal));
+
if (!m.getUpdates().isEmpty()) {
bw.addMutation(m);
}
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java b/test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
index 82a7264207..0bfa4b545c 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
@@ -20,6 +20,7 @@ package org.apache.accumulo.test.functional;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
import java.time.Duration;
import java.util.Arrays;
@@ -34,16 +35,24 @@ import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.admin.NewTableConfiguration;
+import org.apache.accumulo.core.client.admin.TabletHostingGoal;
import org.apache.accumulo.core.client.admin.TimeType;
+import org.apache.accumulo.core.clientImpl.TabletHostingGoalUtil;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.HostingColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.util.Merge;
import org.apache.accumulo.harness.AccumuloClusterHarness;
import org.apache.hadoop.io.Text;
import org.junit.jupiter.api.Test;
+import com.google.common.collect.Iterables;
+
public class MergeIT extends AccumuloClusterHarness {
SortedSet<Text> splits(String[] points) {
@@ -72,9 +81,30 @@ public class MergeIT extends AccumuloClusterHarness {
bw.addMutation(m);
}
}
+ c.tableOperations().setTabletHostingGoal(tableName, new Range("d", "e"),
+ TabletHostingGoal.ALWAYS);
+ c.tableOperations().setTabletHostingGoal(tableName, new Range("e", "f"),
+ TabletHostingGoal.NEVER);
c.tableOperations().flush(tableName, null, null, true);
c.tableOperations().merge(tableName, new Text("c1"), new Text("f1"));
assertEquals(8, c.tableOperations().listSplits(tableName).size());
+ try (Scanner s = c.createScanner(MetadataTable.NAME)) {
+ String tid = c.tableOperations().tableIdMap().get(tableName);
+ s.setRange(new Range(tid + ";g"));
+ TabletColumnFamily.PREV_ROW_COLUMN.fetch(s);
+ HostingColumnFamily.GOAL_COLUMN.fetch(s);
+ assertEquals(2, Iterables.size(s));
+ for (Entry<Key,Value> rows : s) {
+ if (TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(rows.getKey())) {
+ assertEquals("c", TabletColumnFamily.decodePrevEndRow(rows.getValue()).toString());
+ } else if (HostingColumnFamily.GOAL_COLUMN.hasColumns(rows.getKey())) {
+ assertEquals(TabletHostingGoal.ALWAYS,
+ TabletHostingGoalUtil.fromValue(rows.getValue()));
+ } else {
+ fail("Unknown column");
+ }
+ }
+ }
}
}