You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by wc...@apache.org on 2024/04/22 10:22:42 UTC
(hbase) branch HBASE-28463 updated (25f13b6f45d -> a2321ce9d82)
This is an automated email from the ASF dual-hosted git repository.
wchevreuil pushed a change to branch HBASE-28463
in repository https://gitbox.apache.org/repos/asf/hbase.git
discard 25f13b6f45d HBASE-28466 Integration of time-based priority logic of bucket cache in prefetch functionality of HBase (#5808)
omit b7bb8b9d27a HBASE-28505 Implement enforcement to require Date Tiered Compaction for Time Range Data Tiering (#5809)
omit fe285bcc2f6 HBASE-28465 Implementation of framework for time-based priority bucket-cache (#5793)
add 5a8ae130552 HBASE-28516 Bump idna from 2.8 to 3.7 in /dev-support/git-jira-release-audit (#5821)
add 4b7a39a229d HBASE-28504 Implement eviction logic for scanners in Rest APIs to prevent scanner leakage (#5802)
add 4ed17c7117a HBASE-28518 Allow specifying a filter for the REST multiget endpoint
add 4c29c5d86bd HBASE-28500 Rest Java client library assumes stateless servers (#5804)
add 16e9affca37 HBASE-28292 Make Delay prefetch property to be dynamically configured (#5605)
add d7f9ba65487 HBASE-28500 Rest Java client library assumes stateless servers (Addendum:fix spotbugs warning) (#5831)
add a34b4bc5171 HBASE-28470 Fix typo in Java method comment (#5786)
add bc37ce87b03 HBASE-28509 ScanResumer.resume would perform unnecessary scan when cl… (#5817)
add 3539581268f HBASE-28511 Update hbase-thirdparty to 4.1.7 (#5818)
add aaeef2db6ac HBASE-28215 CreateTableProcedure and DeleteTableProcedure should sleep a while before retrying (#5502)
add 8b5ccda02f8 HASE-28414 create-release should spotless:apply after making any file changes (#5824)
add c73f8b51cba HBASE-28497 Missing fields in Get.toJSON (#5800)
add 8936a19b933 Revert "HBASE-28215 CreateTableProcedure and DeleteTableProcedure should sleep a while before retrying (#5502)"
add 5a404c49504 HBASE-28150 CreateTableProcedure and DeleteTableProcedure should sleep a while before retrying (#5502)
new 481a77da447 HBASE-28465 Implementation of framework for time-based priority bucket-cache (#5793)
new c1ea6520f3c HBASE-28505 Implement enforcement to require Date Tiered Compaction for Time Range Data Tiering (#5809)
new a2321ce9d82 HBASE-28466 Integration of time-based priority logic of bucket cache in prefetch functionality of HBase (#5808)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (25f13b6f45d)
\
N -- N -- N refs/heads/HBASE-28463 (a2321ce9d82)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
dev-support/create-release/release-build.sh | 6 +-
dev-support/create-release/release-util.sh | 11 +++
.../git-jira-release-audit/requirements.txt | 2 +-
.../java/org/apache/hadoop/hbase/client/Admin.java | 2 +-
.../AsyncScanSingleRegionRpcRetryingCaller.java | 37 +++++++--
.../hbase/client/AsyncTableResultScanner.java | 24 +++++-
.../java/org/apache/hadoop/hbase/client/Get.java | 21 +++++
.../apache/hadoop/hbase/filter/ParseFilter.java | 2 +-
.../hadoop/hbase/security/EncryptionUtil.java | 4 +-
.../hadoop/hbase/shaded/protobuf/ProtobufUtil.java | 2 +-
.../apache/hadoop/hbase/client/TestOperation.java | 72 +++++++++++++++++
hbase-examples/pom.xml | 2 +-
hbase-protocol-shaded/pom.xml | 2 +-
hbase-rest/pom.xml | 4 +
.../org/apache/hadoop/hbase/rest/Constants.java | 9 ++-
.../apache/hadoop/hbase/rest/MultiRowResource.java | 31 +++++++-
.../apache/hadoop/hbase/rest/ScannerResource.java | 38 +++++++--
.../apache/hadoop/hbase/rest/TableResource.java | 29 +++++--
.../apache/hadoop/hbase/rest/client/Client.java | 49 ++++++++++--
.../hadoop/hbase/rest/TestMultiRowResource.java | 81 +++++++++++++++++---
.../apache/hadoop/hbase/rest/TestTableScan.java | 41 ++++++++--
.../hadoop/hbase/io/hfile/FixedFileTrailer.java | 2 +-
.../org/apache/hadoop/hbase/io/hfile/HFile.java | 2 +
.../hadoop/hbase/io/hfile/HFileReaderImpl.java | 9 +++
.../hadoop/hbase/io/hfile/PrefetchExecutor.java | 89 ++++++++++++++++++++--
.../master/procedure/CreateTableProcedure.java | 24 +++++-
.../master/procedure/DeleteTableProcedure.java | 24 +++++-
.../hadoop/hbase/regionserver/HRegionServer.java | 7 ++
.../regionserver/PrefetchExecutorNotifier.java | 75 ++++++++++++++++++
.../TestAsyncTableScannerCloseWhileSuspending.java | 54 +++++++++----
.../BadMasterObserverForCreateDeleteTable.java | 55 +++++++++++++
.../apache/hadoop/hbase/io/hfile/TestPrefetch.java | 56 +++++++++++++-
.../TestCreateDeleteTableProcedureWithRetry.java | 88 +++++++++++++++++++++
.../master/procedure/TestCreateTableProcedure.java | 4 +-
pom.xml | 8 +-
35 files changed, 871 insertions(+), 95 deletions(-)
create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/PrefetchExecutorNotifier.java
create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/BadMasterObserverForCreateDeleteTable.java
create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateDeleteTableProcedureWithRetry.java
(hbase) 01/03: HBASE-28465 Implementation of framework for time-based priority bucket-cache (#5793)
Posted by wc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
wchevreuil pushed a commit to branch HBASE-28463
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 481a77da447cdb13859d7e2279fe3eb519663c2d
Author: vinayak hegde <vi...@gmail.com>
AuthorDate: Mon Apr 8 20:54:19 2024 +0530
HBASE-28465 Implementation of framework for time-based priority bucket-cache (#5793)
Signed-off-by: Wellington Chevreuil <wc...@apache.org>
---
.../hbase/regionserver/DataTieringException.java | 27 ++
.../hbase/regionserver/DataTieringManager.java | 222 ++++++++++++
.../hadoop/hbase/regionserver/DataTieringType.java | 26 ++
.../hadoop/hbase/regionserver/HRegionServer.java | 1 +
.../hbase/regionserver/TestDataTieringManager.java | 389 +++++++++++++++++++++
5 files changed, 665 insertions(+)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringException.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringException.java
new file mode 100644
index 00000000000..8d356422f6e
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringException.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.yetus.audience.InterfaceAudience;
+
+@InterfaceAudience.Private
+public class DataTieringException extends Exception {
+ DataTieringException(String reason) {
+ super(reason);
+ }
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java
new file mode 100644
index 00000000000..0bc04ddc428
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.OptionalLong;
+import java.util.Set;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The DataTieringManager class categorizes data into hot data and cold data based on the specified
+ * {@link DataTieringType} when DataTiering is enabled. DataTiering is disabled by default with
+ * {@link DataTieringType} set to {@link DataTieringType#NONE}. The {@link DataTieringType}
+ * determines the logic for distinguishing data into hot or cold. By default, all data is considered
+ * as hot.
+ */
+@InterfaceAudience.Private
+public class DataTieringManager {
+ private static final Logger LOG = LoggerFactory.getLogger(DataTieringManager.class);
+ public static final String DATATIERING_KEY = "hbase.hstore.datatiering.type";
+ public static final String DATATIERING_HOT_DATA_AGE_KEY =
+ "hbase.hstore.datatiering.hot.age.millis";
+ public static final DataTieringType DEFAULT_DATATIERING = DataTieringType.NONE;
+ public static final long DEFAULT_DATATIERING_HOT_DATA_AGE = 7 * 24 * 60 * 60 * 1000; // 7 Days
+ private static DataTieringManager instance;
+ private final Map<String, HRegion> onlineRegions;
+
+ private DataTieringManager(Map<String, HRegion> onlineRegions) {
+ this.onlineRegions = onlineRegions;
+ }
+
+ /**
+ * Initializes the DataTieringManager instance with the provided map of online regions.
+ * @param onlineRegions A map containing online regions.
+ */
+ public static synchronized void instantiate(Map<String, HRegion> onlineRegions) {
+ if (instance == null) {
+ instance = new DataTieringManager(onlineRegions);
+ LOG.info("DataTieringManager instantiated successfully.");
+ } else {
+ LOG.warn("DataTieringManager is already instantiated.");
+ }
+ }
+
+ /**
+ * Retrieves the instance of DataTieringManager.
+ * @return The instance of DataTieringManager.
+ * @throws IllegalStateException if DataTieringManager has not been instantiated.
+ */
+ public static synchronized DataTieringManager getInstance() {
+ if (instance == null) {
+ throw new IllegalStateException(
+ "DataTieringManager has not been instantiated. Call instantiate() first.");
+ }
+ return instance;
+ }
+
+ /**
+ * Determines whether data tiering is enabled for the given block cache key.
+ * @param key the block cache key
+ * @return {@code true} if data tiering is enabled for the HFile associated with the key,
+ * {@code false} otherwise
+ * @throws DataTieringException if there is an error retrieving the HFile path or configuration
+ */
+ public boolean isDataTieringEnabled(BlockCacheKey key) throws DataTieringException {
+ Path hFilePath = key.getFilePath();
+ if (hFilePath == null) {
+ throw new DataTieringException("BlockCacheKey Doesn't Contain HFile Path");
+ }
+ return isDataTieringEnabled(hFilePath);
+ }
+
+ /**
+ * Determines whether data tiering is enabled for the given HFile path.
+ * @param hFilePath the path to the HFile
+ * @return {@code true} if data tiering is enabled, {@code false} otherwise
+ * @throws DataTieringException if there is an error retrieving the configuration
+ */
+ public boolean isDataTieringEnabled(Path hFilePath) throws DataTieringException {
+ Configuration configuration = getConfiguration(hFilePath);
+ DataTieringType dataTieringType = getDataTieringType(configuration);
+ return !dataTieringType.equals(DataTieringType.NONE);
+ }
+
+ /**
+ * Determines whether the data associated with the given block cache key is considered hot.
+ * @param key the block cache key
+ * @return {@code true} if the data is hot, {@code false} otherwise
+ * @throws DataTieringException if there is an error retrieving data tiering information or the
+ * HFile maximum timestamp
+ */
+ public boolean isHotData(BlockCacheKey key) throws DataTieringException {
+ Path hFilePath = key.getFilePath();
+ if (hFilePath == null) {
+ throw new DataTieringException("BlockCacheKey Doesn't Contain HFile Path");
+ }
+ return isHotData(hFilePath);
+ }
+
+ /**
+ * Determines whether the data in the HFile at the given path is considered hot based on the
+ * configured data tiering type and hot data age.
+ * @param hFilePath the path to the HFile
+ * @return {@code true} if the data is hot, {@code false} otherwise
+ * @throws DataTieringException if there is an error retrieving data tiering information or the
+ * HFile maximum timestamp
+ */
+ public boolean isHotData(Path hFilePath) throws DataTieringException {
+ Configuration configuration = getConfiguration(hFilePath);
+ DataTieringType dataTieringType = getDataTieringType(configuration);
+
+ if (dataTieringType.equals(DataTieringType.TIME_RANGE)) {
+ long hotDataAge = getDataTieringHotDataAge(configuration);
+
+ HStoreFile hStoreFile = getHStoreFile(hFilePath);
+ if (hStoreFile == null) {
+ LOG.error("HStoreFile corresponding to " + hFilePath + " doesn't exist");
+ return false;
+ }
+ OptionalLong maxTimestamp = hStoreFile.getMaximumTimestamp();
+ if (!maxTimestamp.isPresent()) {
+ throw new DataTieringException("Maximum timestamp not present for " + hFilePath);
+ }
+
+ long currentTimestamp = EnvironmentEdgeManager.getDelegate().currentTime();
+ long diff = currentTimestamp - maxTimestamp.getAsLong();
+ return diff <= hotDataAge;
+ }
+ // DataTieringType.NONE or other types are considered hot by default
+ return true;
+ }
+
+ /**
+ * Returns a set of cold data filenames from the given set of cached blocks. Cold data is
+ * determined by the configured data tiering type and hot data age.
+ * @param allCachedBlocks a set of all cached block cache keys
+ * @return a set of cold data filenames
+ * @throws DataTieringException if there is an error determining whether a block is hot
+ */
+ public Set<String> getColdDataFiles(Set<BlockCacheKey> allCachedBlocks)
+ throws DataTieringException {
+ Set<String> coldHFiles = new HashSet<>();
+ for (BlockCacheKey key : allCachedBlocks) {
+ if (coldHFiles.contains(key.getHfileName())) {
+ continue;
+ }
+ if (!isHotData(key)) {
+ coldHFiles.add(key.getHfileName());
+ }
+ }
+ return coldHFiles;
+ }
+
+ private HRegion getHRegion(Path hFilePath) throws DataTieringException {
+ if (hFilePath.getParent() == null || hFilePath.getParent().getParent() == null) {
+ throw new DataTieringException("Incorrect HFile Path: " + hFilePath);
+ }
+ String regionId = hFilePath.getParent().getParent().getName();
+ HRegion hRegion = this.onlineRegions.get(regionId);
+ if (hRegion == null) {
+ throw new DataTieringException("HRegion corresponding to " + hFilePath + " doesn't exist");
+ }
+ return hRegion;
+ }
+
+ private HStore getHStore(Path hFilePath) throws DataTieringException {
+ HRegion hRegion = getHRegion(hFilePath);
+ String columnFamily = hFilePath.getParent().getName();
+ HStore hStore = hRegion.getStore(Bytes.toBytes(columnFamily));
+ if (hStore == null) {
+ throw new DataTieringException("HStore corresponding to " + hFilePath + " doesn't exist");
+ }
+ return hStore;
+ }
+
+ private HStoreFile getHStoreFile(Path hFilePath) throws DataTieringException {
+ HStore hStore = getHStore(hFilePath);
+ for (HStoreFile file : hStore.getStorefiles()) {
+ if (file.getPath().equals(hFilePath)) {
+ return file;
+ }
+ }
+ return null;
+ }
+
+ private Configuration getConfiguration(Path hFilePath) throws DataTieringException {
+ HStore hStore = getHStore(hFilePath);
+ return hStore.getReadOnlyConfiguration();
+ }
+
+ private DataTieringType getDataTieringType(Configuration conf) {
+ return DataTieringType.valueOf(conf.get(DATATIERING_KEY, DEFAULT_DATATIERING.name()));
+ }
+
+ private long getDataTieringHotDataAge(Configuration conf) {
+ return Long.parseLong(
+ conf.get(DATATIERING_HOT_DATA_AGE_KEY, String.valueOf(DEFAULT_DATATIERING_HOT_DATA_AGE)));
+ }
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringType.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringType.java
new file mode 100644
index 00000000000..ee54576a648
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringType.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.yetus.audience.InterfaceAudience;
+
+@InterfaceAudience.Public
+public enum DataTieringType {
+ NONE,
+ TIME_RANGE
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 191d1ebc524..88679a6eb6c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -534,6 +534,7 @@ public class HRegionServer extends HBaseServerBase<RSRpcServices>
regionServerAccounting = new RegionServerAccounting(conf);
blockCache = BlockCacheFactory.createBlockCache(conf);
+ DataTieringManager.instantiate(onlineRegions);
mobFileCache = new MobFileCache(conf);
rsSnapshotVerifier = new RSSnapshotVerifier(conf);
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java
new file mode 100644
index 00000000000..afb5862a8a4
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java
@@ -0,0 +1,389 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtil;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.io.hfile.BlockCache;
+import org.apache.hadoop.hbase.io.hfile.BlockCacheFactory;
+import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
+import org.apache.hadoop.hbase.io.hfile.BlockType;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
+import org.apache.hadoop.hbase.testclassification.RegionServerTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * This class is used to test the functionality of the DataTieringManager.
+ *
+ * The mock online regions are stored in {@link TestDataTieringManager#testOnlineRegions}.
+ * For all tests, the setup of {@link TestDataTieringManager#testOnlineRegions} occurs only once.
+ * Please refer to {@link TestDataTieringManager#setupOnlineRegions()} for the structure.
+ * Additionally, a list of all store files is maintained in {@link TestDataTieringManager#hStoreFiles}.
+ * The characteristics of these store files are listed below:
+ * @formatter:off ## HStoreFile Information
+ *
+ * | HStoreFile | Region | Store | DataTiering | isHot |
+ * |------------------|--------------------|---------------------|-----------------------|-------|
+ * | hStoreFile0 | region1 | hStore11 | TIME_RANGE | true |
+ * | hStoreFile1 | region1 | hStore12 | NONE | true |
+ * | hStoreFile2 | region2 | hStore21 | TIME_RANGE | true |
+ * | hStoreFile3 | region2 | hStore22 | TIME_RANGE | false |
+ * @formatter:on
+ */
+
+@Category({ RegionServerTests.class, SmallTests.class })
+public class TestDataTieringManager {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestDataTieringManager.class);
+
+ private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
+ private static Configuration defaultConf;
+ private static FileSystem fs;
+ private static CacheConfig cacheConf;
+ private static Path testDir;
+ private static Map<String, HRegion> testOnlineRegions;
+
+ private static DataTieringManager dataTieringManager;
+ private static List<HStoreFile> hStoreFiles;
+
+ @BeforeClass
+ public static void setupBeforeClass() throws Exception {
+ testDir = TEST_UTIL.getDataTestDir(TestDataTieringManager.class.getSimpleName());
+ defaultConf = TEST_UTIL.getConfiguration();
+ fs = HFileSystem.get(defaultConf);
+ BlockCache blockCache = BlockCacheFactory.createBlockCache(defaultConf);
+ cacheConf = new CacheConfig(defaultConf, blockCache);
+ setupOnlineRegions();
+ DataTieringManager.instantiate(testOnlineRegions);
+ dataTieringManager = DataTieringManager.getInstance();
+ }
+
+ @FunctionalInterface
+ interface DataTieringMethodCallerWithPath {
+ boolean call(DataTieringManager manager, Path path) throws DataTieringException;
+ }
+
+ @FunctionalInterface
+ interface DataTieringMethodCallerWithKey {
+ boolean call(DataTieringManager manager, BlockCacheKey key) throws DataTieringException;
+ }
+
+ @Test
+ public void testDataTieringEnabledWithKey() {
+ DataTieringMethodCallerWithKey methodCallerWithKey = DataTieringManager::isDataTieringEnabled;
+
+ // Test with valid key
+ BlockCacheKey key = new BlockCacheKey(hStoreFiles.get(0).getPath(), 0, true, BlockType.DATA);
+ testDataTieringMethodWithKeyNoException(methodCallerWithKey, key, true);
+
+ // Test with another valid key
+ key = new BlockCacheKey(hStoreFiles.get(1).getPath(), 0, true, BlockType.DATA);
+ testDataTieringMethodWithKeyNoException(methodCallerWithKey, key, false);
+
+ // Test with valid key with no HFile Path
+ key = new BlockCacheKey(hStoreFiles.get(0).getPath().getName(), 0);
+ testDataTieringMethodWithKeyExpectingException(methodCallerWithKey, key,
+ new DataTieringException("BlockCacheKey Doesn't Contain HFile Path"));
+ }
+
+ @Test
+ public void testDataTieringEnabledWithPath() {
+ DataTieringMethodCallerWithPath methodCallerWithPath = DataTieringManager::isDataTieringEnabled;
+
+ // Test with valid path
+ Path hFilePath = hStoreFiles.get(1).getPath();
+ testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, false);
+
+ // Test with another valid path
+ hFilePath = hStoreFiles.get(3).getPath();
+ testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, true);
+
+ // Test with an incorrect path
+ hFilePath = new Path("incorrectPath");
+ testDataTieringMethodWithPathExpectingException(methodCallerWithPath, hFilePath,
+ new DataTieringException("Incorrect HFile Path: " + hFilePath));
+
+ // Test with a non-existing HRegion path
+ Path basePath = hStoreFiles.get(0).getPath().getParent().getParent().getParent();
+ hFilePath = new Path(basePath, "incorrectRegion/cf1/filename");
+ testDataTieringMethodWithPathExpectingException(methodCallerWithPath, hFilePath,
+ new DataTieringException("HRegion corresponding to " + hFilePath + " doesn't exist"));
+
+ // Test with a non-existing HStore path
+ basePath = hStoreFiles.get(0).getPath().getParent().getParent();
+ hFilePath = new Path(basePath, "incorrectCf/filename");
+ testDataTieringMethodWithPathExpectingException(methodCallerWithPath, hFilePath,
+ new DataTieringException("HStore corresponding to " + hFilePath + " doesn't exist"));
+ }
+
+ @Test
+ public void testHotDataWithKey() {
+ DataTieringMethodCallerWithKey methodCallerWithKey = DataTieringManager::isHotData;
+
+ // Test with valid key
+ BlockCacheKey key = new BlockCacheKey(hStoreFiles.get(0).getPath(), 0, true, BlockType.DATA);
+ testDataTieringMethodWithKeyNoException(methodCallerWithKey, key, true);
+
+ // Test with another valid key
+ key = new BlockCacheKey(hStoreFiles.get(3).getPath(), 0, true, BlockType.DATA);
+ testDataTieringMethodWithKeyNoException(methodCallerWithKey, key, false);
+ }
+
+ @Test
+ public void testHotDataWithPath() {
+ DataTieringMethodCallerWithPath methodCallerWithPath = DataTieringManager::isHotData;
+
+ // Test with valid path
+ Path hFilePath = hStoreFiles.get(2).getPath();
+ testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, true);
+
+ // Test with another valid path
+ hFilePath = hStoreFiles.get(3).getPath();
+ testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, false);
+
+ // Test with a filename where corresponding HStoreFile in not present
+ hFilePath = new Path(hStoreFiles.get(0).getPath().getParent(), "incorrectFileName");
+ testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, false);
+ }
+
+ @Test
+ public void testColdDataFiles() {
+ Set<BlockCacheKey> allCachedBlocks = new HashSet<>();
+ for (HStoreFile file : hStoreFiles) {
+ allCachedBlocks.add(new BlockCacheKey(file.getPath(), 0, true, BlockType.DATA));
+ }
+
+ // Verify hStoreFile3 is identified as cold data
+ DataTieringMethodCallerWithPath methodCallerWithPath = DataTieringManager::isHotData;
+ Path hFilePath = hStoreFiles.get(3).getPath();
+ testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, false);
+
+ // Verify all the other files in hStoreFiles are hot data
+ for (int i = 0; i < hStoreFiles.size() - 1; i++) {
+ hFilePath = hStoreFiles.get(i).getPath();
+ testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, true);
+ }
+
+ try {
+ Set<String> coldFilePaths = dataTieringManager.getColdDataFiles(allCachedBlocks);
+ assertEquals(1, coldFilePaths.size());
+ } catch (DataTieringException e) {
+ fail("Unexpected DataTieringException: " + e.getMessage());
+ }
+ }
+
+ private void testDataTieringMethodWithPath(DataTieringMethodCallerWithPath caller, Path path,
+ boolean expectedResult, DataTieringException exception) {
+ try {
+ boolean value = caller.call(dataTieringManager, path);
+ if (exception != null) {
+ fail("Expected DataTieringException to be thrown");
+ }
+ assertEquals(expectedResult, value);
+ } catch (DataTieringException e) {
+ if (exception == null) {
+ fail("Unexpected DataTieringException: " + e.getMessage());
+ }
+ assertEquals(exception.getMessage(), e.getMessage());
+ }
+ }
+
+ private void testDataTieringMethodWithKey(DataTieringMethodCallerWithKey caller,
+ BlockCacheKey key, boolean expectedResult, DataTieringException exception) {
+ try {
+ boolean value = caller.call(dataTieringManager, key);
+ if (exception != null) {
+ fail("Expected DataTieringException to be thrown");
+ }
+ assertEquals(expectedResult, value);
+ } catch (DataTieringException e) {
+ if (exception == null) {
+ fail("Unexpected DataTieringException: " + e.getMessage());
+ }
+ assertEquals(exception.getMessage(), e.getMessage());
+ }
+ }
+
+ private void testDataTieringMethodWithPathExpectingException(
+ DataTieringMethodCallerWithPath caller, Path path, DataTieringException exception) {
+ testDataTieringMethodWithPath(caller, path, false, exception);
+ }
+
+ private void testDataTieringMethodWithPathNoException(DataTieringMethodCallerWithPath caller,
+ Path path, boolean expectedResult) {
+ testDataTieringMethodWithPath(caller, path, expectedResult, null);
+ }
+
+ private void testDataTieringMethodWithKeyExpectingException(DataTieringMethodCallerWithKey caller,
+ BlockCacheKey key, DataTieringException exception) {
+ testDataTieringMethodWithKey(caller, key, false, exception);
+ }
+
+ private void testDataTieringMethodWithKeyNoException(DataTieringMethodCallerWithKey caller,
+ BlockCacheKey key, boolean expectedResult) {
+ testDataTieringMethodWithKey(caller, key, expectedResult, null);
+ }
+
+ private static void setupOnlineRegions() throws IOException {
+ testOnlineRegions = new HashMap<>();
+ hStoreFiles = new ArrayList<>();
+
+ long day = 24 * 60 * 60 * 1000;
+ long currentTime = System.currentTimeMillis();
+
+ HRegion region1 = createHRegion("table1");
+
+ HStore hStore11 = createHStore(region1, "cf1", getConfWithTimeRangeDataTieringEnabled(day));
+ hStoreFiles
+ .add(createHStoreFile(hStore11.getStoreContext().getFamilyStoreDirectoryPath(), currentTime));
+ hStore11.refreshStoreFiles();
+ HStore hStore12 = createHStore(region1, "cf2");
+ hStoreFiles.add(createHStoreFile(hStore12.getStoreContext().getFamilyStoreDirectoryPath(),
+ currentTime - day));
+ hStore12.refreshStoreFiles();
+
+ region1.stores.put(Bytes.toBytes("cf1"), hStore11);
+ region1.stores.put(Bytes.toBytes("cf2"), hStore12);
+
+ HRegion region2 =
+ createHRegion("table2", getConfWithTimeRangeDataTieringEnabled((long) (2.5 * day)));
+
+ HStore hStore21 = createHStore(region2, "cf1");
+ hStoreFiles.add(createHStoreFile(hStore21.getStoreContext().getFamilyStoreDirectoryPath(),
+ currentTime - 2 * day));
+ hStore21.refreshStoreFiles();
+ HStore hStore22 = createHStore(region2, "cf2");
+ hStoreFiles.add(createHStoreFile(hStore22.getStoreContext().getFamilyStoreDirectoryPath(),
+ currentTime - 3 * day));
+ hStore22.refreshStoreFiles();
+
+ region2.stores.put(Bytes.toBytes("cf1"), hStore21);
+ region2.stores.put(Bytes.toBytes("cf2"), hStore22);
+
+ for (HStoreFile file : hStoreFiles) {
+ file.initReader();
+ }
+
+ testOnlineRegions.put(region1.getRegionInfo().getEncodedName(), region1);
+ testOnlineRegions.put(region2.getRegionInfo().getEncodedName(), region2);
+ }
+
+ private static HRegion createHRegion(String table) throws IOException {
+ return createHRegion(table, defaultConf);
+ }
+
+ private static HRegion createHRegion(String table, Configuration conf) throws IOException {
+ TableName tableName = TableName.valueOf(table);
+
+ TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName)
+ .setValue(DataTieringManager.DATATIERING_KEY, conf.get(DataTieringManager.DATATIERING_KEY))
+ .setValue(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY,
+ conf.get(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY))
+ .build();
+ RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).build();
+
+ Configuration testConf = new Configuration(conf);
+ CommonFSUtils.setRootDir(testConf, testDir);
+ HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs,
+ CommonFSUtils.getTableDir(testDir, hri.getTable()), hri);
+
+ return new HRegion(regionFs, null, conf, htd, null);
+ }
+
+ private static HStore createHStore(HRegion region, String columnFamily) throws IOException {
+ return createHStore(region, columnFamily, defaultConf);
+ }
+
+ private static HStore createHStore(HRegion region, String columnFamily, Configuration conf)
+ throws IOException {
+ ColumnFamilyDescriptor columnFamilyDescriptor =
+ ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(columnFamily))
+ .setValue(DataTieringManager.DATATIERING_KEY, conf.get(DataTieringManager.DATATIERING_KEY))
+ .setValue(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY,
+ conf.get(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY))
+ .build();
+
+ return new HStore(region, columnFamilyDescriptor, conf, false);
+ }
+
+ private static Configuration getConfWithTimeRangeDataTieringEnabled(long hotDataAge) {
+ Configuration conf = new Configuration(defaultConf);
+ conf.set(DataTieringManager.DATATIERING_KEY, DataTieringType.TIME_RANGE.name());
+ conf.set(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY, String.valueOf(hotDataAge));
+ return conf;
+ }
+
+ private static HStoreFile createHStoreFile(Path storeDir, long timestamp) throws IOException {
+ String columnFamily = storeDir.getName();
+
+ StoreFileWriter storeFileWriter = new StoreFileWriter.Builder(defaultConf, cacheConf, fs)
+ .withOutputDir(storeDir).withFileContext(new HFileContextBuilder().build()).build();
+
+ writeStoreFileRandomData(storeFileWriter, Bytes.toBytes(columnFamily), Bytes.toBytes("random"),
+ timestamp);
+
+ return new HStoreFile(fs, storeFileWriter.getPath(), defaultConf, cacheConf, BloomType.NONE,
+ true);
+ }
+
+ private static void writeStoreFileRandomData(final StoreFileWriter writer, byte[] columnFamily,
+ byte[] qualifier, long timestamp) throws IOException {
+ try {
+ for (char d = 'a'; d <= 'z'; d++) {
+ for (char e = 'a'; e <= 'z'; e++) {
+ byte[] b = new byte[] { (byte) d, (byte) e };
+ writer.append(new KeyValue(b, columnFamily, qualifier, timestamp, b));
+ }
+ }
+ } finally {
+ writer.appendTrackedTimestampsToMetadata();
+ writer.close();
+ }
+ }
+}
(hbase) 03/03: HBASE-28466 Integration of time-based priority logic of bucket cache in prefetch functionality of HBase (#5808)
Posted by wc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
wchevreuil pushed a commit to branch HBASE-28463
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit a2321ce9d824e76ad053cfe915880f83300936a0
Author: vinayak hegde <vi...@gmail.com>
AuthorDate: Mon Apr 22 15:23:30 2024 +0530
HBASE-28466 Integration of time-based priority logic of bucket cache in prefetch functionality of HBase (#5808)
Signed-off-by: Wellington Chevreuil <wc...@apache.org>
---
.../apache/hadoop/hbase/io/hfile/BlockCache.java | 6 +-
.../hadoop/hbase/io/hfile/CombinedBlockCache.java | 7 +-
.../apache/hadoop/hbase/io/hfile/HFileInfo.java | 6 ++
.../hadoop/hbase/io/hfile/HFilePreadReader.java | 6 +-
.../hadoop/hbase/io/hfile/bucket/BucketCache.java | 15 +++-
.../hbase/regionserver/DataTieringManager.java | 91 +++++++++++++++++-----
.../hbase/regionserver/TestDataTieringManager.java | 58 ++++++++++----
7 files changed, 145 insertions(+), 44 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
index bed0194b1fa..ac83af1053a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.io.hfile;
import java.util.Iterator;
import java.util.Map;
import java.util.Optional;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.yetus.audience.InterfaceAudience;
@@ -197,11 +198,12 @@ public interface BlockCache extends Iterable<CachedBlock> {
* overridden by all implementing classes. In such cases, the returned Optional will be empty. For
* subclasses implementing this logic, the returned Optional would contain the boolean value
* reflecting if the passed file should indeed be cached.
- * @param fileName to check if it should be cached.
+ * @param hFileInfo Information about the file to check if it should be cached.
+ * @param conf The configuration object to use for determining caching behavior.
* @return empty optional if this method is not supported, otherwise the returned optional
* contains the boolean value informing if the file should be cached.
*/
- default Optional<Boolean> shouldCacheFile(String fileName) {
+ default Optional<Boolean> shouldCacheFile(HFileInfo hFileInfo, Configuration conf) {
return Optional.empty();
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
index d6692d2e2bf..b12510cdccd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
@@ -22,6 +22,7 @@ import java.util.Map;
import java.util.Optional;
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableBoolean;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.io.HeapSize;
import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
@@ -482,9 +483,9 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
}
@Override
- public Optional<Boolean> shouldCacheFile(String fileName) {
- Optional<Boolean> l1Result = l1Cache.shouldCacheFile(fileName);
- Optional<Boolean> l2Result = l2Cache.shouldCacheFile(fileName);
+ public Optional<Boolean> shouldCacheFile(HFileInfo hFileInfo, Configuration conf) {
+ Optional<Boolean> l1Result = l1Cache.shouldCacheFile(hFileInfo, conf);
+ Optional<Boolean> l2Result = l2Cache.shouldCacheFile(hFileInfo, conf);
final Mutable<Boolean> combinedResult = new MutableBoolean(true);
l1Result.ifPresent(b -> combinedResult.setValue(b && combinedResult.getValue()));
l2Result.ifPresent(b -> combinedResult.setValue(b && combinedResult.getValue()));
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java
index 31e637a0099..e89f86e7c4e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java
@@ -122,6 +122,7 @@ public class HFileInfo implements SortedMap<byte[], byte[]> {
private FixedFileTrailer trailer;
private HFileContext hfileContext;
+ private boolean initialized = false;
public HFileInfo() {
super();
@@ -363,6 +364,10 @@ public class HFileInfo implements SortedMap<byte[], byte[]> {
* should be called after initTrailerAndContext
*/
public void initMetaAndIndex(HFile.Reader reader) throws IOException {
+ if (initialized) {
+ return;
+ }
+
ReaderContext context = reader.getContext();
try {
HFileBlock.FSReader blockReader = reader.getUncachedBlockReader();
@@ -400,6 +405,7 @@ public class HFileInfo implements SortedMap<byte[], byte[]> {
throw new CorruptHFileException(
"Problem reading data index and meta index from file " + context.getFilePath(), t);
}
+ initialized = true;
}
private HFileContext createHFileContext(Path path, FixedFileTrailer trailer, Configuration conf)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java
index 6063ffe6889..7631dc78c3a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java
@@ -37,10 +37,14 @@ public class HFilePreadReader extends HFileReaderImpl {
public HFilePreadReader(ReaderContext context, HFileInfo fileInfo, CacheConfig cacheConf,
Configuration conf) throws IOException {
super(context, fileInfo, cacheConf, conf);
+
final MutableBoolean shouldCache = new MutableBoolean(true);
+ // Initialize HFileInfo object with metadata for caching decisions
+ fileInfo.initMetaAndIndex(this);
+
cacheConf.getBlockCache().ifPresent(cache -> {
- Optional<Boolean> result = cache.shouldCacheFile(path.getName());
+ Optional<Boolean> result = cache.shouldCacheFile(fileInfo, conf);
shouldCache.setValue(result.isPresent() ? result.get().booleanValue() : true);
});
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
index 71bfc757e51..622a57f91c2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
@@ -73,9 +73,11 @@ import org.apache.hadoop.hbase.io.hfile.CachedBlock;
import org.apache.hadoop.hbase.io.hfile.CombinedBlockCache;
import org.apache.hadoop.hbase.io.hfile.HFileBlock;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
+import org.apache.hadoop.hbase.io.hfile.HFileInfo;
import org.apache.hadoop.hbase.nio.ByteBuff;
import org.apache.hadoop.hbase.nio.RefCnt;
import org.apache.hadoop.hbase.protobuf.ProtobufMagic;
+import org.apache.hadoop.hbase.regionserver.DataTieringManager;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.IdReadWriteLock;
@@ -2150,7 +2152,18 @@ public class BucketCache implements BlockCache, HeapSize {
}
@Override
- public Optional<Boolean> shouldCacheFile(String fileName) {
+ public Optional<Boolean> shouldCacheFile(HFileInfo hFileInfo, Configuration conf) {
+ String fileName = hFileInfo.getHFileContext().getHFileName();
+ try {
+ DataTieringManager dataTieringManager = DataTieringManager.getInstance();
+ if (!dataTieringManager.isHotData(hFileInfo, conf)) {
+ LOG.debug("Data tiering is enabled for file: '{}' and it is not hot data", fileName);
+ return Optional.of(false);
+ }
+ } catch (IllegalStateException e) {
+ LOG.error("Error while getting DataTieringManager instance: {}", e.getMessage());
+ }
+
// if we don't have the file in fullyCachedFiles, we should cache it
return Optional.of(!fullyCachedFiles.containsKey(fileName));
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java
index 0bc04ddc428..dec96604774 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.hbase.regionserver;
+import static org.apache.hadoop.hbase.regionserver.HStoreFile.TIMERANGE_KEY;
+
+import java.io.IOException;
import java.util.HashSet;
import java.util.Map;
import java.util.OptionalLong;
@@ -24,6 +27,7 @@ import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
+import org.apache.hadoop.hbase.io.hfile.HFileInfo;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.yetus.audience.InterfaceAudience;
@@ -106,11 +110,13 @@ public class DataTieringManager {
}
/**
- * Determines whether the data associated with the given block cache key is considered hot.
+ * Determines whether the data associated with the given block cache key is considered hot. If the
+ * data tiering type is set to {@link DataTieringType#TIME_RANGE} and maximum timestamp is not
+ * present, it considers {@code Long.MAX_VALUE} as the maximum timestamp, making the data hot by
+ * default.
* @param key the block cache key
* @return {@code true} if the data is hot, {@code false} otherwise
- * @throws DataTieringException if there is an error retrieving data tiering information or the
- * HFile maximum timestamp
+ * @throws DataTieringException if there is an error retrieving data tiering information
*/
public boolean isHotData(BlockCacheKey key) throws DataTieringException {
Path hFilePath = key.getFilePath();
@@ -122,37 +128,82 @@ public class DataTieringManager {
/**
* Determines whether the data in the HFile at the given path is considered hot based on the
- * configured data tiering type and hot data age.
+ * configured data tiering type and hot data age. If the data tiering type is set to
+ * {@link DataTieringType#TIME_RANGE} and maximum timestamp is not present, it considers
+ * {@code Long.MAX_VALUE} as the maximum timestamp, making the data hot by default.
* @param hFilePath the path to the HFile
* @return {@code true} if the data is hot, {@code false} otherwise
- * @throws DataTieringException if there is an error retrieving data tiering information or the
- * HFile maximum timestamp
+ * @throws DataTieringException if there is an error retrieving data tiering information
*/
public boolean isHotData(Path hFilePath) throws DataTieringException {
Configuration configuration = getConfiguration(hFilePath);
DataTieringType dataTieringType = getDataTieringType(configuration);
if (dataTieringType.equals(DataTieringType.TIME_RANGE)) {
- long hotDataAge = getDataTieringHotDataAge(configuration);
-
- HStoreFile hStoreFile = getHStoreFile(hFilePath);
- if (hStoreFile == null) {
- LOG.error("HStoreFile corresponding to " + hFilePath + " doesn't exist");
- return false;
- }
- OptionalLong maxTimestamp = hStoreFile.getMaximumTimestamp();
- if (!maxTimestamp.isPresent()) {
- throw new DataTieringException("Maximum timestamp not present for " + hFilePath);
- }
+ return hotDataValidator(getMaxTimestamp(hFilePath), getDataTieringHotDataAge(configuration));
+ }
+ // DataTieringType.NONE or other types are considered hot by default
+ return true;
+ }
- long currentTimestamp = EnvironmentEdgeManager.getDelegate().currentTime();
- long diff = currentTimestamp - maxTimestamp.getAsLong();
- return diff <= hotDataAge;
+ /**
+ * Determines whether the data in the HFile being read is considered hot based on the configured
+ * data tiering type and hot data age. If the data tiering type is set to
+ * {@link DataTieringType#TIME_RANGE} and maximum timestamp is not present, it considers
+ * {@code Long.MAX_VALUE} as the maximum timestamp, making the data hot by default.
+ * @param hFileInfo Information about the HFile to determine if its data is hot.
+ * @param configuration The configuration object to use for determining hot data criteria.
+ * @return {@code true} if the data is hot, {@code false} otherwise
+ */
+ public boolean isHotData(HFileInfo hFileInfo, Configuration configuration) {
+ DataTieringType dataTieringType = getDataTieringType(configuration);
+ if (dataTieringType.equals(DataTieringType.TIME_RANGE)) {
+ return hotDataValidator(getMaxTimestamp(hFileInfo), getDataTieringHotDataAge(configuration));
}
// DataTieringType.NONE or other types are considered hot by default
return true;
}
+ private boolean hotDataValidator(long maxTimestamp, long hotDataAge) {
+ long currentTimestamp = getCurrentTimestamp();
+ long diff = currentTimestamp - maxTimestamp;
+ return diff <= hotDataAge;
+ }
+
+ private long getMaxTimestamp(Path hFilePath) throws DataTieringException {
+ HStoreFile hStoreFile = getHStoreFile(hFilePath);
+ if (hStoreFile == null) {
+ LOG.error("HStoreFile corresponding to " + hFilePath + " doesn't exist");
+ return Long.MAX_VALUE;
+ }
+ OptionalLong maxTimestamp = hStoreFile.getMaximumTimestamp();
+ if (!maxTimestamp.isPresent()) {
+ LOG.error("Maximum timestamp not present for " + hFilePath);
+ return Long.MAX_VALUE;
+ }
+ return maxTimestamp.getAsLong();
+ }
+
+ private long getMaxTimestamp(HFileInfo hFileInfo) {
+ try {
+ byte[] hFileTimeRange = hFileInfo.get(TIMERANGE_KEY);
+ if (hFileTimeRange == null) {
+ LOG.error("Timestamp information not found for file: {}",
+ hFileInfo.getHFileContext().getHFileName());
+ return Long.MAX_VALUE;
+ }
+ return TimeRangeTracker.parseFrom(hFileTimeRange).getMax();
+ } catch (IOException e) {
+ LOG.error("Error occurred while reading the timestamp metadata of file: {}",
+ hFileInfo.getHFileContext().getHFileName(), e);
+ return Long.MAX_VALUE;
+ }
+ }
+
+ private long getCurrentTimestamp() {
+ return EnvironmentEdgeManager.getDelegate().currentTime();
+ }
+
/**
* Returns a set of cold data filenames from the given set of cached blocks. Cold data is
* determined by the configured data tiering type and hot data age.
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java
index afb5862a8a4..6beed9943b3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java
@@ -17,7 +17,9 @@
*/
package org.apache.hadoop.hbase.regionserver;
+import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_SIZE_KEY;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
@@ -26,14 +28,17 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtil;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.RegionInfo;
@@ -51,6 +56,7 @@ import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.apache.hadoop.hbase.util.Pair;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
@@ -85,22 +91,26 @@ public class TestDataTieringManager {
private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
private static Configuration defaultConf;
private static FileSystem fs;
+ private static BlockCache blockCache;
private static CacheConfig cacheConf;
private static Path testDir;
- private static Map<String, HRegion> testOnlineRegions;
+ private static final Map<String, HRegion> testOnlineRegions = new HashMap<>();
private static DataTieringManager dataTieringManager;
- private static List<HStoreFile> hStoreFiles;
+ private static final List<HStoreFile> hStoreFiles = new ArrayList<>();
@BeforeClass
public static void setupBeforeClass() throws Exception {
testDir = TEST_UTIL.getDataTestDir(TestDataTieringManager.class.getSimpleName());
defaultConf = TEST_UTIL.getConfiguration();
+ defaultConf.setBoolean(CacheConfig.PREFETCH_BLOCKS_ON_OPEN_KEY, true);
+ defaultConf.setStrings(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap");
+ defaultConf.setLong(BUCKET_CACHE_SIZE_KEY, 32);
fs = HFileSystem.get(defaultConf);
- BlockCache blockCache = BlockCacheFactory.createBlockCache(defaultConf);
+ blockCache = BlockCacheFactory.createBlockCache(defaultConf);
cacheConf = new CacheConfig(defaultConf, blockCache);
- setupOnlineRegions();
DataTieringManager.instantiate(testOnlineRegions);
+ setupOnlineRegions();
dataTieringManager = DataTieringManager.getInstance();
}
@@ -189,7 +199,24 @@ public class TestDataTieringManager {
// Test with a filename where corresponding HStoreFile in not present
hFilePath = new Path(hStoreFiles.get(0).getPath().getParent(), "incorrectFileName");
- testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, false);
+ testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, true);
+ }
+
+ @Test
+ public void testPrefetchWhenDataTieringEnabled() throws IOException {
+ // Evict blocks from cache by closing the files and passing evict on close.
+ // Then initialize the reader again. Since Prefetch on open is set to true, it should prefetch
+ // those blocks.
+ for (HStoreFile file : hStoreFiles) {
+ file.closeStoreFile(true);
+ file.initReader();
+ }
+
+ // Since we have one cold file among four files, only three should get prefetched.
+ Optional<Map<String, Pair<String, Long>>> fullyCachedFiles = blockCache.getFullyCachedFiles();
+ assertTrue("We should get the fully cached files from the cache", fullyCachedFiles.isPresent());
+ Waiter.waitFor(defaultConf, 60000, () -> fullyCachedFiles.get().size() == 3);
+ assertEquals("Number of fully cached files are incorrect", 3, fullyCachedFiles.get().size());
}
@Test
@@ -271,21 +298,18 @@ public class TestDataTieringManager {
}
private static void setupOnlineRegions() throws IOException {
- testOnlineRegions = new HashMap<>();
- hStoreFiles = new ArrayList<>();
-
long day = 24 * 60 * 60 * 1000;
long currentTime = System.currentTimeMillis();
HRegion region1 = createHRegion("table1");
HStore hStore11 = createHStore(region1, "cf1", getConfWithTimeRangeDataTieringEnabled(day));
- hStoreFiles
- .add(createHStoreFile(hStore11.getStoreContext().getFamilyStoreDirectoryPath(), currentTime));
+ hStoreFiles.add(createHStoreFile(hStore11.getStoreContext().getFamilyStoreDirectoryPath(),
+ hStore11.getReadOnlyConfiguration(), currentTime));
hStore11.refreshStoreFiles();
HStore hStore12 = createHStore(region1, "cf2");
hStoreFiles.add(createHStoreFile(hStore12.getStoreContext().getFamilyStoreDirectoryPath(),
- currentTime - day));
+ hStore12.getReadOnlyConfiguration(), currentTime - day));
hStore12.refreshStoreFiles();
region1.stores.put(Bytes.toBytes("cf1"), hStore11);
@@ -296,11 +320,11 @@ public class TestDataTieringManager {
HStore hStore21 = createHStore(region2, "cf1");
hStoreFiles.add(createHStoreFile(hStore21.getStoreContext().getFamilyStoreDirectoryPath(),
- currentTime - 2 * day));
+ hStore21.getReadOnlyConfiguration(), currentTime - 2 * day));
hStore21.refreshStoreFiles();
HStore hStore22 = createHStore(region2, "cf2");
hStoreFiles.add(createHStoreFile(hStore22.getStoreContext().getFamilyStoreDirectoryPath(),
- currentTime - 3 * day));
+ hStore22.getReadOnlyConfiguration(), currentTime - 3 * day));
hStore22.refreshStoreFiles();
region2.stores.put(Bytes.toBytes("cf1"), hStore21);
@@ -359,17 +383,17 @@ public class TestDataTieringManager {
return conf;
}
- private static HStoreFile createHStoreFile(Path storeDir, long timestamp) throws IOException {
+ private static HStoreFile createHStoreFile(Path storeDir, Configuration conf, long timestamp)
+ throws IOException {
String columnFamily = storeDir.getName();
- StoreFileWriter storeFileWriter = new StoreFileWriter.Builder(defaultConf, cacheConf, fs)
+ StoreFileWriter storeFileWriter = new StoreFileWriter.Builder(conf, cacheConf, fs)
.withOutputDir(storeDir).withFileContext(new HFileContextBuilder().build()).build();
writeStoreFileRandomData(storeFileWriter, Bytes.toBytes(columnFamily), Bytes.toBytes("random"),
timestamp);
- return new HStoreFile(fs, storeFileWriter.getPath(), defaultConf, cacheConf, BloomType.NONE,
- true);
+ return new HStoreFile(fs, storeFileWriter.getPath(), conf, cacheConf, BloomType.NONE, true);
}
private static void writeStoreFileRandomData(final StoreFileWriter writer, byte[] columnFamily,
(hbase) 02/03: HBASE-28505 Implement enforcement to require Date Tiered Compaction for Time Range Data Tiering (#5809)
Posted by wc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
wchevreuil pushed a commit to branch HBASE-28463
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit c1ea6520f3cc63ab78b25031998d53f5eed3d676
Author: vinayak hegde <vi...@gmail.com>
AuthorDate: Fri Apr 12 14:54:37 2024 +0530
HBASE-28505 Implement enforcement to require Date Tiered Compaction for Time Range Data Tiering (#5809)
Signed-off-by: Wellington Chevreuil <wc...@apache.org>
---
.../hbase/regionserver/DateTieredStoreEngine.java | 3 ++
.../hadoop/hbase/util/TableDescriptorChecker.java | 36 +++++++++++++++++
.../hbase/client/TestIllegalTableDescriptor.java | 45 ++++++++++++++++++++++
3 files changed, 84 insertions(+)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java
index d15a6c92ef0..8fdbb6035ae 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java
@@ -41,6 +41,9 @@ import org.apache.yetus.audience.InterfaceAudience;
@InterfaceAudience.Private
public class DateTieredStoreEngine extends StoreEngine<DefaultStoreFlusher,
DateTieredCompactionPolicy, DateTieredCompactor, DefaultStoreFileManager> {
+
+ public static final String DATE_TIERED_STORE_ENGINE = DateTieredStoreEngine.class.getName();
+
@Override
public boolean needsCompaction(List<HStoreFile> filesCompacting) {
return compactionPolicy.needsCompaction(storeFileManager.getStorefiles(), filesCompacting);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/TableDescriptorChecker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/TableDescriptorChecker.java
index 94e2e4bbfa0..471583b32b7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/TableDescriptorChecker.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/TableDescriptorChecker.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hbase.util;
+import static org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine.DATE_TIERED_STORE_ENGINE;
+
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CompoundConfiguration;
@@ -28,10 +30,13 @@ import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.fs.ErasureCodingUtils;
+import org.apache.hadoop.hbase.regionserver.DataTieringManager;
+import org.apache.hadoop.hbase.regionserver.DataTieringType;
import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
import org.apache.hadoop.hbase.regionserver.HStore;
import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
import org.apache.hadoop.hbase.regionserver.RegionSplitPolicy;
+import org.apache.hadoop.hbase.regionserver.StoreEngine;
import org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy;
import org.apache.hadoop.hbase.regionserver.compactions.FIFOCompactionPolicy;
import org.apache.yetus.audience.InterfaceAudience;
@@ -191,6 +196,8 @@ public final class TableDescriptorChecker {
// check in-memory compaction
warnOrThrowExceptionForFailure(logWarn, hcd::getInMemoryCompaction);
+
+ checkDateTieredCompactionForTimeRangeDataTiering(conf, td);
}
}
@@ -210,6 +217,35 @@ public final class TableDescriptorChecker {
});
}
+ private static void checkDateTieredCompactionForTimeRangeDataTiering(final Configuration conf,
+ final TableDescriptor td) throws IOException {
+ // Table level configurations
+ checkDateTieredCompactionForTimeRangeDataTiering(conf);
+ for (ColumnFamilyDescriptor cfd : td.getColumnFamilies()) {
+ // Column family level configurations
+ Configuration cfdConf =
+ new CompoundConfiguration().add(conf).addStringMap(cfd.getConfiguration());
+ checkDateTieredCompactionForTimeRangeDataTiering(cfdConf);
+ }
+ }
+
+ private static void checkDateTieredCompactionForTimeRangeDataTiering(final Configuration conf)
+ throws IOException {
+ final String errorMessage =
+ "Time Range Data Tiering should be enabled with Date Tiered Compaction.";
+
+ warnOrThrowExceptionForFailure(false, () -> {
+
+ // Determine whether Date Tiered Compaction will be enabled when Time Range Data Tiering is
+ // enabled after the configuration change.
+ if (DataTieringType.TIME_RANGE.name().equals(conf.get(DataTieringManager.DATATIERING_KEY))) {
+ if (!DATE_TIERED_STORE_ENGINE.equals(conf.get(StoreEngine.STORE_ENGINE_CLASS_KEY))) {
+ throw new IllegalArgumentException(errorMessage);
+ }
+ }
+ });
+ }
+
private static void checkCompactionPolicy(final Configuration conf, final TableDescriptor td)
throws IOException {
warnOrThrowExceptionForFailure(false, () -> {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIllegalTableDescriptor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIllegalTableDescriptor.java
index a67c43d31fa..666cfbdf54a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIllegalTableDescriptor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIllegalTableDescriptor.java
@@ -31,6 +31,9 @@ import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.regionserver.DataTieringManager;
+import org.apache.hadoop.hbase.regionserver.DataTieringType;
+import org.apache.hadoop.hbase.regionserver.StoreEngine;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -187,6 +190,48 @@ public class TestIllegalTableDescriptor {
+ "cause very frequent flushing."));
}
+ @Test
+ public void testIllegalTableDescriptorWithDataTiering() throws IOException {
+ // table level configuration changes
+ TableDescriptorBuilder builder =
+ TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()));
+ ColumnFamilyDescriptorBuilder cfBuilder = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY);
+ builder.setColumnFamily(cfBuilder.build());
+
+ // First scenario: DataTieringType set to TIME_RANGE without DateTieredStoreEngine
+ builder.setValue(DataTieringManager.DATATIERING_KEY, DataTieringType.TIME_RANGE.name());
+ checkTableIsIllegal(builder.build());
+
+ // Second scenario: DataTieringType set to TIME_RANGE with DateTieredStoreEngine
+ builder.setValue(StoreEngine.STORE_ENGINE_CLASS_KEY,
+ "org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine");
+ checkTableIsLegal(builder.build());
+
+ // Third scenario: Disabling DateTieredStoreEngine while Time Range DataTiering is active
+ builder.setValue(StoreEngine.STORE_ENGINE_CLASS_KEY,
+ "org.apache.hadoop.hbase.regionserver.DefaultStoreEngine");
+ checkTableIsIllegal(builder.build());
+
+ // column family level configuration changes
+ builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()));
+ cfBuilder = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY);
+
+ // First scenario: DataTieringType set to TIME_RANGE without DateTieredStoreEngine
+ cfBuilder.setConfiguration(DataTieringManager.DATATIERING_KEY,
+ DataTieringType.TIME_RANGE.name());
+ checkTableIsIllegal(builder.setColumnFamily(cfBuilder.build()).build());
+
+ // Second scenario: DataTieringType set to TIME_RANGE with DateTieredStoreEngine
+ cfBuilder.setConfiguration(StoreEngine.STORE_ENGINE_CLASS_KEY,
+ "org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine");
+ checkTableIsLegal(builder.modifyColumnFamily(cfBuilder.build()).build());
+
+ // Third scenario: Disabling DateTieredStoreEngine while Time Range DataTiering is active
+ cfBuilder.setConfiguration(StoreEngine.STORE_ENGINE_CLASS_KEY,
+ "org.apache.hadoop.hbase.regionserver.DefaultStoreEngine");
+ checkTableIsIllegal(builder.modifyColumnFamily(cfBuilder.build()).build());
+ }
+
private void checkTableIsLegal(TableDescriptor tableDescriptor) throws IOException {
Admin admin = TEST_UTIL.getAdmin();
admin.createTable(tableDescriptor);