You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by dl...@apache.org on 2023/05/02 17:24:10 UTC

[accumulo] branch elasticity updated: Set the TabletHostingGoal during tablet merge (#3335)

This is an automated email from the ASF dual-hosted git repository.

dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/elasticity by this push:
     new 1cd934e3e4 Set the TabletHostingGoal during tablet merge (#3335)
1cd934e3e4 is described below

commit 1cd934e3e45bfc85709f44787e09634723e53c2b
Author: Dave Marion <dl...@apache.org>
AuthorDate: Tue May 2 13:24:05 2023 -0400

    Set the TabletHostingGoal during tablet merge (#3335)
    
    * Set the TabletHostingGoal during tablet merge
    
    Closes #3307
---
 .../accumulo/manager/TabletGroupWatcher.java       | 22 ++++++++++++++++
 .../apache/accumulo/test/functional/MergeIT.java   | 30 ++++++++++++++++++++++
 2 files changed, 52 insertions(+)

diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
index a4a76083c9..431f097875 100644
--- a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
@@ -44,6 +44,8 @@ import org.apache.accumulo.core.client.MutationsRejectedException;
 import org.apache.accumulo.core.client.RowIterator;
 import org.apache.accumulo.core.client.Scanner;
 import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.TabletHostingGoal;
+import org.apache.accumulo.core.clientImpl.TabletHostingGoalUtil;
 import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Mutation;
@@ -71,6 +73,7 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.Cu
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.FutureLocationColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.HostingColumnFamily;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
 import org.apache.accumulo.core.metadata.schema.MetadataTime;
@@ -726,6 +729,7 @@ abstract class TabletGroupWatcher extends AccumuloDaemonThread {
     if (range.isMeta()) {
       targetSystemTable = RootTable.NAME;
     }
+    Set<TabletHostingGoal> goals = new HashSet<>();
 
     AccumuloClient client = manager.getContext();
 
@@ -737,6 +741,7 @@ abstract class TabletGroupWatcher extends AccumuloDaemonThread {
       TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
       ServerColumnFamily.TIME_COLUMN.fetch(scanner);
       ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner);
+      HostingColumnFamily.GOAL_COLUMN.fetch(scanner);
       scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
       Mutation m = new Mutation(stopRow);
       MetadataTime maxLogicalTime = null;
@@ -756,6 +761,9 @@ abstract class TabletGroupWatcher extends AccumuloDaemonThread {
         } else if (ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(key)) {
           var allVolumesDir = new AllVolumesDirectory(range.tableId(), value.toString());
           bw.addMutation(manager.getContext().getAmple().createDeleteMutation(allVolumesDir));
+        } else if (HostingColumnFamily.GOAL_COLUMN.hasColumns(key)) {
+          TabletHostingGoal thisGoal = TabletHostingGoalUtil.fromValue(value);
+          goals.add(thisGoal);
         }
       }
 
@@ -764,6 +772,7 @@ abstract class TabletGroupWatcher extends AccumuloDaemonThread {
       scanner = client.createScanner(targetSystemTable, Authorizations.EMPTY);
       scanner.setRange(new Range(stopRow));
       ServerColumnFamily.TIME_COLUMN.fetch(scanner);
+      HostingColumnFamily.GOAL_COLUMN.fetch(scanner);
       scanner.fetchColumnFamily(ExternalCompactionColumnFamily.NAME);
       Set<String> extCompIds = new HashSet<>();
       for (Entry<Key,Value> entry : scanner) {
@@ -772,6 +781,9 @@ abstract class TabletGroupWatcher extends AccumuloDaemonThread {
               MetadataTime.parse(entry.getValue().toString()));
         } else if (ExternalCompactionColumnFamily.NAME.equals(entry.getKey().getColumnFamily())) {
           extCompIds.add(entry.getKey().getColumnQualifierData().toString());
+        } else if (HostingColumnFamily.GOAL_COLUMN.hasColumns(entry.getKey())) {
+          TabletHostingGoal thisGoal = TabletHostingGoalUtil.fromValue(entry.getValue());
+          goals.add(thisGoal);
         }
       }
 
@@ -782,6 +794,16 @@ abstract class TabletGroupWatcher extends AccumuloDaemonThread {
       // delete any entries for external compactions
       extCompIds.forEach(ecid -> m.putDelete(ExternalCompactionColumnFamily.STR_NAME, ecid));
 
+      // Set the TabletHostingGoal for this tablet based on the goals of the other tablets in
+      // the merge range. Always takes priority over never.
+      TabletHostingGoal mergeHostingGoal = TabletHostingGoal.ONDEMAND;
+      if (range.isMeta() || goals.contains(TabletHostingGoal.ALWAYS)) {
+        mergeHostingGoal = TabletHostingGoal.ALWAYS;
+      } else if (goals.contains(TabletHostingGoal.NEVER)) {
+        mergeHostingGoal = TabletHostingGoal.NEVER;
+      }
+      HostingColumnFamily.GOAL_COLUMN.put(m, TabletHostingGoalUtil.toValue(mergeHostingGoal));
+
       if (!m.getUpdates().isEmpty()) {
         bw.addMutation(m);
       }
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java b/test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
index 82a7264207..0bfa4b545c 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
@@ -20,6 +20,7 @@ package org.apache.accumulo.test.functional;
 
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
 
 import java.time.Duration;
 import java.util.Arrays;
@@ -34,16 +35,24 @@ import org.apache.accumulo.core.client.AccumuloClient;
 import org.apache.accumulo.core.client.BatchWriter;
 import org.apache.accumulo.core.client.Scanner;
 import org.apache.accumulo.core.client.admin.NewTableConfiguration;
+import org.apache.accumulo.core.client.admin.TabletHostingGoal;
 import org.apache.accumulo.core.client.admin.TimeType;
+import org.apache.accumulo.core.clientImpl.TabletHostingGoalUtil;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
 import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.HostingColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.core.util.Merge;
 import org.apache.accumulo.harness.AccumuloClusterHarness;
 import org.apache.hadoop.io.Text;
 import org.junit.jupiter.api.Test;
 
+import com.google.common.collect.Iterables;
+
 public class MergeIT extends AccumuloClusterHarness {
 
   SortedSet<Text> splits(String[] points) {
@@ -72,9 +81,30 @@ public class MergeIT extends AccumuloClusterHarness {
           bw.addMutation(m);
         }
       }
+      c.tableOperations().setTabletHostingGoal(tableName, new Range("d", "e"),
+          TabletHostingGoal.ALWAYS);
+      c.tableOperations().setTabletHostingGoal(tableName, new Range("e", "f"),
+          TabletHostingGoal.NEVER);
       c.tableOperations().flush(tableName, null, null, true);
       c.tableOperations().merge(tableName, new Text("c1"), new Text("f1"));
       assertEquals(8, c.tableOperations().listSplits(tableName).size());
+      try (Scanner s = c.createScanner(MetadataTable.NAME)) {
+        String tid = c.tableOperations().tableIdMap().get(tableName);
+        s.setRange(new Range(tid + ";g"));
+        TabletColumnFamily.PREV_ROW_COLUMN.fetch(s);
+        HostingColumnFamily.GOAL_COLUMN.fetch(s);
+        assertEquals(2, Iterables.size(s));
+        for (Entry<Key,Value> rows : s) {
+          if (TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(rows.getKey())) {
+            assertEquals("c", TabletColumnFamily.decodePrevEndRow(rows.getValue()).toString());
+          } else if (HostingColumnFamily.GOAL_COLUMN.hasColumns(rows.getKey())) {
+            assertEquals(TabletHostingGoal.ALWAYS,
+                TabletHostingGoalUtil.fromValue(rows.getValue()));
+          } else {
+            fail("Unknown column");
+          }
+        }
+      }
     }
   }