You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by wc...@apache.org on 2025/08/05 19:34:53 UTC
(hbase) 02/08: HBASE-28505 Implement enforcement to require Date Tiered Compaction for Time Range Data Tiering (#5809)
This is an automated email from the ASF dual-hosted git repository.
wchevreuil pushed a commit to branch HBASE-28463
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 752568203812073aba1704bd5d31eeba0a989e86
Author: vinayak hegde <vi...@gmail.com>
AuthorDate: Fri Apr 12 14:54:37 2024 +0530
HBASE-28505 Implement enforcement to require Date Tiered Compaction for Time Range Data Tiering (#5809)
Signed-off-by: Wellington Chevreuil <wc...@apache.org>
---
.../hbase/regionserver/DateTieredStoreEngine.java | 3 ++
.../hadoop/hbase/util/TableDescriptorChecker.java | 36 +++++++++++++++++
.../hbase/client/TestIllegalTableDescriptor.java | 45 ++++++++++++++++++++++
3 files changed, 84 insertions(+)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java
index ded6564bce5..26437ab1124 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java
@@ -41,6 +41,9 @@ import org.apache.yetus.audience.InterfaceAudience;
@InterfaceAudience.Private
public class DateTieredStoreEngine extends StoreEngine<DefaultStoreFlusher,
DateTieredCompactionPolicy, DateTieredCompactor, DefaultStoreFileManager> {
+
+ public static final String DATE_TIERED_STORE_ENGINE = DateTieredStoreEngine.class.getName();
+
@Override
public boolean needsCompaction(List<HStoreFile> filesCompacting) {
return compactionPolicy.needsCompaction(storeFileManager.getStoreFiles(), filesCompacting);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/TableDescriptorChecker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/TableDescriptorChecker.java
index a826860aae4..409cc284dee 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/TableDescriptorChecker.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/TableDescriptorChecker.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hbase.util;
+import static org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine.DATE_TIERED_STORE_ENGINE;
+
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CompoundConfiguration;
@@ -29,10 +31,13 @@ import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.conf.ConfigKey;
import org.apache.hadoop.hbase.fs.ErasureCodingUtils;
+import org.apache.hadoop.hbase.regionserver.DataTieringManager;
+import org.apache.hadoop.hbase.regionserver.DataTieringType;
import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
import org.apache.hadoop.hbase.regionserver.HStore;
import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
import org.apache.hadoop.hbase.regionserver.RegionSplitPolicy;
+import org.apache.hadoop.hbase.regionserver.StoreEngine;
import org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy;
import org.apache.hadoop.hbase.regionserver.compactions.FIFOCompactionPolicy;
import org.apache.yetus.audience.InterfaceAudience;
@@ -201,6 +206,8 @@ public final class TableDescriptorChecker {
// check in-memory compaction
warnOrThrowExceptionForFailure(logWarn, hcd::getInMemoryCompaction);
+
+ checkDateTieredCompactionForTimeRangeDataTiering(conf, td);
}
}
@@ -220,6 +227,35 @@ public final class TableDescriptorChecker {
});
}
+ private static void checkDateTieredCompactionForTimeRangeDataTiering(final Configuration conf,
+ final TableDescriptor td) throws IOException {
+ // Table level configurations
+ checkDateTieredCompactionForTimeRangeDataTiering(conf);
+ for (ColumnFamilyDescriptor cfd : td.getColumnFamilies()) {
+ // Column family level configurations
+ Configuration cfdConf =
+ new CompoundConfiguration().add(conf).addStringMap(cfd.getConfiguration());
+ checkDateTieredCompactionForTimeRangeDataTiering(cfdConf);
+ }
+ }
+
+ private static void checkDateTieredCompactionForTimeRangeDataTiering(final Configuration conf)
+ throws IOException {
+ final String errorMessage =
+ "Time Range Data Tiering should be enabled with Date Tiered Compaction.";
+
+ warnOrThrowExceptionForFailure(false, () -> {
+
+ // Determine whether Date Tiered Compaction will be enabled when Time Range Data Tiering is
+ // enabled after the configuration change.
+ if (DataTieringType.TIME_RANGE.name().equals(conf.get(DataTieringManager.DATATIERING_KEY))) {
+ if (!DATE_TIERED_STORE_ENGINE.equals(conf.get(StoreEngine.STORE_ENGINE_CLASS_KEY))) {
+ throw new IllegalArgumentException(errorMessage);
+ }
+ }
+ });
+ }
+
private static void checkCompactionPolicy(final Configuration conf, final TableDescriptor td)
throws IOException {
warnOrThrowExceptionForFailure(false, () -> {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIllegalTableDescriptor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIllegalTableDescriptor.java
index a67c43d31fa..666cfbdf54a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIllegalTableDescriptor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIllegalTableDescriptor.java
@@ -31,6 +31,9 @@ import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.regionserver.DataTieringManager;
+import org.apache.hadoop.hbase.regionserver.DataTieringType;
+import org.apache.hadoop.hbase.regionserver.StoreEngine;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -187,6 +190,48 @@ public class TestIllegalTableDescriptor {
+ "cause very frequent flushing."));
}
+ @Test
+ public void testIllegalTableDescriptorWithDataTiering() throws IOException {
+ // table level configuration changes
+ TableDescriptorBuilder builder =
+ TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()));
+ ColumnFamilyDescriptorBuilder cfBuilder = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY);
+ builder.setColumnFamily(cfBuilder.build());
+
+ // First scenario: DataTieringType set to TIME_RANGE without DateTieredStoreEngine
+ builder.setValue(DataTieringManager.DATATIERING_KEY, DataTieringType.TIME_RANGE.name());
+ checkTableIsIllegal(builder.build());
+
+ // Second scenario: DataTieringType set to TIME_RANGE with DateTieredStoreEngine
+ builder.setValue(StoreEngine.STORE_ENGINE_CLASS_KEY,
+ "org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine");
+ checkTableIsLegal(builder.build());
+
+ // Third scenario: Disabling DateTieredStoreEngine while Time Range DataTiering is active
+ builder.setValue(StoreEngine.STORE_ENGINE_CLASS_KEY,
+ "org.apache.hadoop.hbase.regionserver.DefaultStoreEngine");
+ checkTableIsIllegal(builder.build());
+
+ // column family level configuration changes
+ builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()));
+ cfBuilder = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY);
+
+ // First scenario: DataTieringType set to TIME_RANGE without DateTieredStoreEngine
+ cfBuilder.setConfiguration(DataTieringManager.DATATIERING_KEY,
+ DataTieringType.TIME_RANGE.name());
+ checkTableIsIllegal(builder.setColumnFamily(cfBuilder.build()).build());
+
+ // Second scenario: DataTieringType set to TIME_RANGE with DateTieredStoreEngine
+ cfBuilder.setConfiguration(StoreEngine.STORE_ENGINE_CLASS_KEY,
+ "org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine");
+ checkTableIsLegal(builder.modifyColumnFamily(cfBuilder.build()).build());
+
+ // Third scenario: Disabling DateTieredStoreEngine while Time Range DataTiering is active
+ cfBuilder.setConfiguration(StoreEngine.STORE_ENGINE_CLASS_KEY,
+ "org.apache.hadoop.hbase.regionserver.DefaultStoreEngine");
+ checkTableIsIllegal(builder.modifyColumnFamily(cfBuilder.build()).build());
+ }
+
private void checkTableIsLegal(TableDescriptor tableDescriptor) throws IOException {
Admin admin = TEST_UTIL.getAdmin();
admin.createTable(tableDescriptor);