You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by wc...@apache.org on 2024/04/12 09:01:46 UTC
(hbase) branch HBASE-28463 updated (a9d31709219 -> fe285bcc2f6)
This is an automated email from the ASF dual-hosted git repository.
wchevreuil pushed a change to branch HBASE-28463
in repository https://gitbox.apache.org/repos/asf/hbase.git
omit a9d31709219 HBASE-28465 Implementation of framework for time-based priority bucket-cache (#5793)
add 6101bad5a3a HBASE-28366 Mis-order of SCP and regionServerReport results into region inconsistencies (#5774)
add aea7e7c85cd [ADDENDUM] HBASE-28458 BucketCache.notifyFileCachingCompleted may incorrectly consider a file fully cached (#5777) (#5791)
add 558b151c5b2 HBASE-28483 backup merge fails on bulkloaded hfiles (#5795)
add c1012a9ebec HBASE-28457 Introduce a version field in file based tracker record (#5784)
add e5d59cadc5d HBASE-28481 Prompting table already exists after failing to create table with many region replications (#5789)
add eeebbdfa723 HBASE-28478 Remove the hbase1 compatible code in FixedFileTrailer (#5788)
add 3340d8dd07e HBASE-28183 It's impossible to re-enable the quota table if it gets disabled (#5691)
add bf836a98073 HBASE-26192 Master UI hbck should provide a JSON formatted output option (#5772)
add 1a089cd3935 HBASE-28485 Re-use ZstdDecompressCtx/ZstdCompressCtx for performance (#5797)
add adc79a0a9c2 HBASE-28448 CompressionTest hangs when run over a Ozone ofs path (#5771)
add 5d694dae5e4 HBASE-28506 Remove hbase-compression-xz (#5811)
new fe285bcc2f6 HBASE-28465 Implementation of framework for time-based priority bucket-cache (#5793)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (a9d31709219)
\
N -- N -- N refs/heads/HBASE-28463 (fe285bcc2f6)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
hbase-assembly/pom.xml | 4 -
...=> TestIncrementalBackupMergeWithBulkLoad.java} | 46 +--
...xistException.java => HbckEmptyRegionInfo.java} | 16 +-
...Exception.java => HbckInconsistentRegions.java} | 40 +-
...tsException.java => HbckOrphanRegionsOnFS.java} | 27 +-
...tsException.java => HbckOrphanRegionsOnRS.java} | 27 +-
...ationException.java => HbckOverlapRegions.java} | 32 +-
.../org/apache/hadoop/hbase/HbckRegionDetails.java | 32 +-
...perationException.java => HbckRegionHoles.java} | 32 +-
.../org/apache/hadoop/hbase/HbckServerName.java | 28 +-
...ationException.java => HbckUnknownServers.java} | 32 +-
.../hadoop/hbase/client/MutableRegionInfo.java | 2 +-
hbase-compression/hbase-compression-xz/pom.xml | 166 --------
.../hadoop/hbase/io/compress/xz/LzmaCodec.java | 126 ------
.../hbase/io/compress/xz/LzmaCompressor.java | 223 -----------
.../hbase/io/compress/xz/LzmaDecompressor.java | 151 --------
.../io/compress/xz/TestHFileCompressionLzma.java | 76 ----
.../hadoop/hbase/io/compress/xz/TestLzmaCodec.java | 58 ---
.../io/compress/xz/TestWALCompressionLzma.java | 56 ---
.../hbase/io/compress/zstd/ZstdCompressor.java | 20 +-
.../hbase/io/compress/zstd/ZstdDecompressor.java | 19 +-
hbase-compression/pom.xml | 1 -
.../hadoop/hbase/mapreduce/HFileInputFormat.java | 25 +-
.../protobuf/server/region/StoreFileTracker.proto | 1 +
hbase-replication/pom.xml | 5 -
.../hadoop/hbase/io/hfile/FixedFileTrailer.java | 42 +-
.../hadoop/hbase/io/hfile/bucket/BucketCache.java | 46 ++-
.../org/apache/hadoop/hbase/master/HMaster.java | 48 ++-
.../apache/hadoop/hbase/master/MasterServices.java | 4 +
.../apache/hadoop/hbase/master/ServerManager.java | 20 +-
.../HbckConfigFactory.java} | 15 +-
.../hbase/master/http/hbck/model/HbckMetrics.java | 98 +++++
.../http/hbck/resource/HbckMetricsResource.java | 140 +++++++
.../master/procedure/CreateTableProcedure.java | 14 +-
.../storefiletracker/StoreFileListFile.java | 62 ++-
.../apache/hadoop/hbase/util/CompressionTest.java | 21 +-
.../hbase/io/hfile/TestFixedFileTrailer.java | 76 ++--
.../io/hfile/bucket/TestPrefetchPersistence.java | 35 +-
.../hbase/master/MockNoopMasterServices.java | 6 +
.../hbase/master/http/TestHbckMetricsResource.java | 422 +++++++++++++++++++++
.../master/procedure/TestCreateTableProcedure.java | 29 ++
.../apache/hadoop/hbase/quotas/TestQuotaAdmin.java | 11 +
.../storefiletracker/TestStoreFileListFile.java | 17 +
pom.xml | 11 -
44 files changed, 1109 insertions(+), 1253 deletions(-)
copy hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/{TestBackupRestoreWithModifications.java => TestIncrementalBackupMergeWithBulkLoad.java} (88%)
copy hbase-client/src/main/java/org/apache/hadoop/hbase/{NamespaceExistException.java => HbckEmptyRegionInfo.java} (68%)
copy hbase-client/src/main/java/org/apache/hadoop/hbase/{TableNotEnabledException.java => HbckInconsistentRegions.java} (52%)
copy hbase-client/src/main/java/org/apache/hadoop/hbase/{TableExistsException.java => HbckOrphanRegionsOnFS.java} (62%)
copy hbase-client/src/main/java/org/apache/hadoop/hbase/{TableExistsException.java => HbckOrphanRegionsOnRS.java} (62%)
copy hbase-client/src/main/java/org/apache/hadoop/hbase/{InvalidFamilyOperationException.java => HbckOverlapRegions.java} (59%)
copy hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseIOException.java => hbase-client/src/main/java/org/apache/hadoop/hbase/HbckRegionDetails.java (59%)
copy hbase-client/src/main/java/org/apache/hadoop/hbase/{InvalidFamilyOperationException.java => HbckRegionHoles.java} (59%)
copy hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseIOException.java => hbase-client/src/main/java/org/apache/hadoop/hbase/HbckServerName.java (65%)
copy hbase-client/src/main/java/org/apache/hadoop/hbase/{InvalidFamilyOperationException.java => HbckUnknownServers.java} (59%)
delete mode 100644 hbase-compression/hbase-compression-xz/pom.xml
delete mode 100644 hbase-compression/hbase-compression-xz/src/main/java/org/apache/hadoop/hbase/io/compress/xz/LzmaCodec.java
delete mode 100644 hbase-compression/hbase-compression-xz/src/main/java/org/apache/hadoop/hbase/io/compress/xz/LzmaCompressor.java
delete mode 100644 hbase-compression/hbase-compression-xz/src/main/java/org/apache/hadoop/hbase/io/compress/xz/LzmaDecompressor.java
delete mode 100644 hbase-compression/hbase-compression-xz/src/test/java/org/apache/hadoop/hbase/io/compress/xz/TestHFileCompressionLzma.java
delete mode 100644 hbase-compression/hbase-compression-xz/src/test/java/org/apache/hadoop/hbase/io/compress/xz/TestLzmaCodec.java
delete mode 100644 hbase-compression/hbase-compression-xz/src/test/java/org/apache/hadoop/hbase/io/compress/xz/TestWALCompressionLzma.java
copy hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/{api_v1/ResourceConfigFactory.java => hbck/HbckConfigFactory.java} (85%)
create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/hbck/model/HbckMetrics.java
create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/hbck/resource/HbckMetricsResource.java
create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/http/TestHbckMetricsResource.java
(hbase) 01/01: HBASE-28465 Implementation of framework for time-based priority bucket-cache (#5793)
Posted by wc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
wchevreuil pushed a commit to branch HBASE-28463
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit fe285bcc2f649e1a51d62a3044dc2116ec637377
Author: vinayak hegde <vi...@gmail.com>
AuthorDate: Mon Apr 8 20:54:19 2024 +0530
HBASE-28465 Implementation of framework for time-based priority bucket-cache (#5793)
Signed-off-by: Wellington Chevreuil <wc...@apache.org>
---
.../hbase/regionserver/DataTieringException.java | 27 ++
.../hbase/regionserver/DataTieringManager.java | 222 ++++++++++++
.../hadoop/hbase/regionserver/DataTieringType.java | 26 ++
.../hadoop/hbase/regionserver/HRegionServer.java | 1 +
.../hbase/regionserver/TestDataTieringManager.java | 389 +++++++++++++++++++++
5 files changed, 665 insertions(+)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringException.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringException.java
new file mode 100644
index 00000000000..8d356422f6e
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringException.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.yetus.audience.InterfaceAudience;
+
+@InterfaceAudience.Private
+public class DataTieringException extends Exception {
+ DataTieringException(String reason) {
+ super(reason);
+ }
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java
new file mode 100644
index 00000000000..0bc04ddc428
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.OptionalLong;
+import java.util.Set;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The DataTieringManager class categorizes data into hot data and cold data based on the specified
+ * {@link DataTieringType} when DataTiering is enabled. DataTiering is disabled by default with
+ * {@link DataTieringType} set to {@link DataTieringType#NONE}. The {@link DataTieringType}
+ * determines the logic for distinguishing data into hot or cold. By default, all data is considered
+ * as hot.
+ */
+@InterfaceAudience.Private
+public class DataTieringManager {
+ private static final Logger LOG = LoggerFactory.getLogger(DataTieringManager.class);
+ public static final String DATATIERING_KEY = "hbase.hstore.datatiering.type";
+ public static final String DATATIERING_HOT_DATA_AGE_KEY =
+ "hbase.hstore.datatiering.hot.age.millis";
+ public static final DataTieringType DEFAULT_DATATIERING = DataTieringType.NONE;
+ public static final long DEFAULT_DATATIERING_HOT_DATA_AGE = 7 * 24 * 60 * 60 * 1000; // 7 Days
+ private static DataTieringManager instance;
+ private final Map<String, HRegion> onlineRegions;
+
+ private DataTieringManager(Map<String, HRegion> onlineRegions) {
+ this.onlineRegions = onlineRegions;
+ }
+
+ /**
+ * Initializes the DataTieringManager instance with the provided map of online regions.
+ * @param onlineRegions A map containing online regions.
+ */
+ public static synchronized void instantiate(Map<String, HRegion> onlineRegions) {
+ if (instance == null) {
+ instance = new DataTieringManager(onlineRegions);
+ LOG.info("DataTieringManager instantiated successfully.");
+ } else {
+ LOG.warn("DataTieringManager is already instantiated.");
+ }
+ }
+
+ /**
+ * Retrieves the instance of DataTieringManager.
+ * @return The instance of DataTieringManager.
+ * @throws IllegalStateException if DataTieringManager has not been instantiated.
+ */
+ public static synchronized DataTieringManager getInstance() {
+ if (instance == null) {
+ throw new IllegalStateException(
+ "DataTieringManager has not been instantiated. Call instantiate() first.");
+ }
+ return instance;
+ }
+
+ /**
+ * Determines whether data tiering is enabled for the given block cache key.
+ * @param key the block cache key
+ * @return {@code true} if data tiering is enabled for the HFile associated with the key,
+ * {@code false} otherwise
+ * @throws DataTieringException if there is an error retrieving the HFile path or configuration
+ */
+ public boolean isDataTieringEnabled(BlockCacheKey key) throws DataTieringException {
+ Path hFilePath = key.getFilePath();
+ if (hFilePath == null) {
+ throw new DataTieringException("BlockCacheKey Doesn't Contain HFile Path");
+ }
+ return isDataTieringEnabled(hFilePath);
+ }
+
+ /**
+ * Determines whether data tiering is enabled for the given HFile path.
+ * @param hFilePath the path to the HFile
+ * @return {@code true} if data tiering is enabled, {@code false} otherwise
+ * @throws DataTieringException if there is an error retrieving the configuration
+ */
+ public boolean isDataTieringEnabled(Path hFilePath) throws DataTieringException {
+ Configuration configuration = getConfiguration(hFilePath);
+ DataTieringType dataTieringType = getDataTieringType(configuration);
+ return !dataTieringType.equals(DataTieringType.NONE);
+ }
+
+ /**
+ * Determines whether the data associated with the given block cache key is considered hot.
+ * @param key the block cache key
+ * @return {@code true} if the data is hot, {@code false} otherwise
+ * @throws DataTieringException if there is an error retrieving data tiering information or the
+ * HFile maximum timestamp
+ */
+ public boolean isHotData(BlockCacheKey key) throws DataTieringException {
+ Path hFilePath = key.getFilePath();
+ if (hFilePath == null) {
+ throw new DataTieringException("BlockCacheKey Doesn't Contain HFile Path");
+ }
+ return isHotData(hFilePath);
+ }
+
+ /**
+ * Determines whether the data in the HFile at the given path is considered hot based on the
+ * configured data tiering type and hot data age.
+ * @param hFilePath the path to the HFile
+ * @return {@code true} if the data is hot, {@code false} otherwise
+ * @throws DataTieringException if there is an error retrieving data tiering information or the
+ * HFile maximum timestamp
+ */
+ public boolean isHotData(Path hFilePath) throws DataTieringException {
+ Configuration configuration = getConfiguration(hFilePath);
+ DataTieringType dataTieringType = getDataTieringType(configuration);
+
+ if (dataTieringType.equals(DataTieringType.TIME_RANGE)) {
+ long hotDataAge = getDataTieringHotDataAge(configuration);
+
+ HStoreFile hStoreFile = getHStoreFile(hFilePath);
+ if (hStoreFile == null) {
+ LOG.error("HStoreFile corresponding to " + hFilePath + " doesn't exist");
+ return false;
+ }
+ OptionalLong maxTimestamp = hStoreFile.getMaximumTimestamp();
+ if (!maxTimestamp.isPresent()) {
+ throw new DataTieringException("Maximum timestamp not present for " + hFilePath);
+ }
+
+ long currentTimestamp = EnvironmentEdgeManager.getDelegate().currentTime();
+ long diff = currentTimestamp - maxTimestamp.getAsLong();
+ return diff <= hotDataAge;
+ }
+ // DataTieringType.NONE or other types are considered hot by default
+ return true;
+ }
+
+ /**
+ * Returns a set of cold data filenames from the given set of cached blocks. Cold data is
+ * determined by the configured data tiering type and hot data age.
+ * @param allCachedBlocks a set of all cached block cache keys
+ * @return a set of cold data filenames
+ * @throws DataTieringException if there is an error determining whether a block is hot
+ */
+ public Set<String> getColdDataFiles(Set<BlockCacheKey> allCachedBlocks)
+ throws DataTieringException {
+ Set<String> coldHFiles = new HashSet<>();
+ for (BlockCacheKey key : allCachedBlocks) {
+ if (coldHFiles.contains(key.getHfileName())) {
+ continue;
+ }
+ if (!isHotData(key)) {
+ coldHFiles.add(key.getHfileName());
+ }
+ }
+ return coldHFiles;
+ }
+
+ private HRegion getHRegion(Path hFilePath) throws DataTieringException {
+ if (hFilePath.getParent() == null || hFilePath.getParent().getParent() == null) {
+ throw new DataTieringException("Incorrect HFile Path: " + hFilePath);
+ }
+ String regionId = hFilePath.getParent().getParent().getName();
+ HRegion hRegion = this.onlineRegions.get(regionId);
+ if (hRegion == null) {
+ throw new DataTieringException("HRegion corresponding to " + hFilePath + " doesn't exist");
+ }
+ return hRegion;
+ }
+
+ private HStore getHStore(Path hFilePath) throws DataTieringException {
+ HRegion hRegion = getHRegion(hFilePath);
+ String columnFamily = hFilePath.getParent().getName();
+ HStore hStore = hRegion.getStore(Bytes.toBytes(columnFamily));
+ if (hStore == null) {
+ throw new DataTieringException("HStore corresponding to " + hFilePath + " doesn't exist");
+ }
+ return hStore;
+ }
+
+ private HStoreFile getHStoreFile(Path hFilePath) throws DataTieringException {
+ HStore hStore = getHStore(hFilePath);
+ for (HStoreFile file : hStore.getStorefiles()) {
+ if (file.getPath().equals(hFilePath)) {
+ return file;
+ }
+ }
+ return null;
+ }
+
+ private Configuration getConfiguration(Path hFilePath) throws DataTieringException {
+ HStore hStore = getHStore(hFilePath);
+ return hStore.getReadOnlyConfiguration();
+ }
+
+ private DataTieringType getDataTieringType(Configuration conf) {
+ return DataTieringType.valueOf(conf.get(DATATIERING_KEY, DEFAULT_DATATIERING.name()));
+ }
+
+ private long getDataTieringHotDataAge(Configuration conf) {
+ return Long.parseLong(
+ conf.get(DATATIERING_HOT_DATA_AGE_KEY, String.valueOf(DEFAULT_DATATIERING_HOT_DATA_AGE)));
+ }
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringType.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringType.java
new file mode 100644
index 00000000000..ee54576a648
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringType.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.yetus.audience.InterfaceAudience;
+
+@InterfaceAudience.Public
+public enum DataTieringType {
+ NONE,
+ TIME_RANGE
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 88863c06e4b..e4da74f78cf 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -531,6 +531,7 @@ public class HRegionServer extends HBaseServerBase<RSRpcServices>
regionServerAccounting = new RegionServerAccounting(conf);
blockCache = BlockCacheFactory.createBlockCache(conf);
+ DataTieringManager.instantiate(onlineRegions);
mobFileCache = new MobFileCache(conf);
rsSnapshotVerifier = new RSSnapshotVerifier(conf);
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java
new file mode 100644
index 00000000000..afb5862a8a4
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java
@@ -0,0 +1,389 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtil;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.io.hfile.BlockCache;
+import org.apache.hadoop.hbase.io.hfile.BlockCacheFactory;
+import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
+import org.apache.hadoop.hbase.io.hfile.BlockType;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
+import org.apache.hadoop.hbase.testclassification.RegionServerTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * This class is used to test the functionality of the DataTieringManager.
+ *
+ * The mock online regions are stored in {@link TestDataTieringManager#testOnlineRegions}.
+ * For all tests, the setup of {@link TestDataTieringManager#testOnlineRegions} occurs only once.
+ * Please refer to {@link TestDataTieringManager#setupOnlineRegions()} for the structure.
+ * Additionally, a list of all store files is maintained in {@link TestDataTieringManager#hStoreFiles}.
+ * The characteristics of these store files are listed below:
+ * @formatter:off ## HStoreFile Information
+ *
+ * | HStoreFile | Region | Store | DataTiering | isHot |
+ * |------------------|--------------------|---------------------|-----------------------|-------|
+ * | hStoreFile0 | region1 | hStore11 | TIME_RANGE | true |
+ * | hStoreFile1 | region1 | hStore12 | NONE | true |
+ * | hStoreFile2 | region2 | hStore21 | TIME_RANGE | true |
+ * | hStoreFile3 | region2 | hStore22 | TIME_RANGE | false |
+ * @formatter:on
+ */
+
+@Category({ RegionServerTests.class, SmallTests.class })
+public class TestDataTieringManager {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestDataTieringManager.class);
+
+ private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
+ private static Configuration defaultConf;
+ private static FileSystem fs;
+ private static CacheConfig cacheConf;
+ private static Path testDir;
+ private static Map<String, HRegion> testOnlineRegions;
+
+ private static DataTieringManager dataTieringManager;
+ private static List<HStoreFile> hStoreFiles;
+
+ @BeforeClass
+ public static void setupBeforeClass() throws Exception {
+ testDir = TEST_UTIL.getDataTestDir(TestDataTieringManager.class.getSimpleName());
+ defaultConf = TEST_UTIL.getConfiguration();
+ fs = HFileSystem.get(defaultConf);
+ BlockCache blockCache = BlockCacheFactory.createBlockCache(defaultConf);
+ cacheConf = new CacheConfig(defaultConf, blockCache);
+ setupOnlineRegions();
+ DataTieringManager.instantiate(testOnlineRegions);
+ dataTieringManager = DataTieringManager.getInstance();
+ }
+
+ @FunctionalInterface
+ interface DataTieringMethodCallerWithPath {
+ boolean call(DataTieringManager manager, Path path) throws DataTieringException;
+ }
+
+ @FunctionalInterface
+ interface DataTieringMethodCallerWithKey {
+ boolean call(DataTieringManager manager, BlockCacheKey key) throws DataTieringException;
+ }
+
+ @Test
+ public void testDataTieringEnabledWithKey() {
+ DataTieringMethodCallerWithKey methodCallerWithKey = DataTieringManager::isDataTieringEnabled;
+
+ // Test with valid key
+ BlockCacheKey key = new BlockCacheKey(hStoreFiles.get(0).getPath(), 0, true, BlockType.DATA);
+ testDataTieringMethodWithKeyNoException(methodCallerWithKey, key, true);
+
+ // Test with another valid key
+ key = new BlockCacheKey(hStoreFiles.get(1).getPath(), 0, true, BlockType.DATA);
+ testDataTieringMethodWithKeyNoException(methodCallerWithKey, key, false);
+
+ // Test with valid key with no HFile Path
+ key = new BlockCacheKey(hStoreFiles.get(0).getPath().getName(), 0);
+ testDataTieringMethodWithKeyExpectingException(methodCallerWithKey, key,
+ new DataTieringException("BlockCacheKey Doesn't Contain HFile Path"));
+ }
+
+ @Test
+ public void testDataTieringEnabledWithPath() {
+ DataTieringMethodCallerWithPath methodCallerWithPath = DataTieringManager::isDataTieringEnabled;
+
+ // Test with valid path
+ Path hFilePath = hStoreFiles.get(1).getPath();
+ testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, false);
+
+ // Test with another valid path
+ hFilePath = hStoreFiles.get(3).getPath();
+ testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, true);
+
+ // Test with an incorrect path
+ hFilePath = new Path("incorrectPath");
+ testDataTieringMethodWithPathExpectingException(methodCallerWithPath, hFilePath,
+ new DataTieringException("Incorrect HFile Path: " + hFilePath));
+
+ // Test with a non-existing HRegion path
+ Path basePath = hStoreFiles.get(0).getPath().getParent().getParent().getParent();
+ hFilePath = new Path(basePath, "incorrectRegion/cf1/filename");
+ testDataTieringMethodWithPathExpectingException(methodCallerWithPath, hFilePath,
+ new DataTieringException("HRegion corresponding to " + hFilePath + " doesn't exist"));
+
+ // Test with a non-existing HStore path
+ basePath = hStoreFiles.get(0).getPath().getParent().getParent();
+ hFilePath = new Path(basePath, "incorrectCf/filename");
+ testDataTieringMethodWithPathExpectingException(methodCallerWithPath, hFilePath,
+ new DataTieringException("HStore corresponding to " + hFilePath + " doesn't exist"));
+ }
+
+ @Test
+ public void testHotDataWithKey() {
+ DataTieringMethodCallerWithKey methodCallerWithKey = DataTieringManager::isHotData;
+
+ // Test with valid key
+ BlockCacheKey key = new BlockCacheKey(hStoreFiles.get(0).getPath(), 0, true, BlockType.DATA);
+ testDataTieringMethodWithKeyNoException(methodCallerWithKey, key, true);
+
+ // Test with another valid key
+ key = new BlockCacheKey(hStoreFiles.get(3).getPath(), 0, true, BlockType.DATA);
+ testDataTieringMethodWithKeyNoException(methodCallerWithKey, key, false);
+ }
+
+ @Test
+ public void testHotDataWithPath() {
+ DataTieringMethodCallerWithPath methodCallerWithPath = DataTieringManager::isHotData;
+
+ // Test with valid path
+ Path hFilePath = hStoreFiles.get(2).getPath();
+ testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, true);
+
+ // Test with another valid path
+ hFilePath = hStoreFiles.get(3).getPath();
+ testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, false);
+
+ // Test with a filename where corresponding HStoreFile in not present
+ hFilePath = new Path(hStoreFiles.get(0).getPath().getParent(), "incorrectFileName");
+ testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, false);
+ }
+
+ @Test
+ public void testColdDataFiles() {
+ Set<BlockCacheKey> allCachedBlocks = new HashSet<>();
+ for (HStoreFile file : hStoreFiles) {
+ allCachedBlocks.add(new BlockCacheKey(file.getPath(), 0, true, BlockType.DATA));
+ }
+
+ // Verify hStoreFile3 is identified as cold data
+ DataTieringMethodCallerWithPath methodCallerWithPath = DataTieringManager::isHotData;
+ Path hFilePath = hStoreFiles.get(3).getPath();
+ testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, false);
+
+ // Verify all the other files in hStoreFiles are hot data
+ for (int i = 0; i < hStoreFiles.size() - 1; i++) {
+ hFilePath = hStoreFiles.get(i).getPath();
+ testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, true);
+ }
+
+ try {
+ Set<String> coldFilePaths = dataTieringManager.getColdDataFiles(allCachedBlocks);
+ assertEquals(1, coldFilePaths.size());
+ } catch (DataTieringException e) {
+ fail("Unexpected DataTieringException: " + e.getMessage());
+ }
+ }
+
+ private void testDataTieringMethodWithPath(DataTieringMethodCallerWithPath caller, Path path,
+ boolean expectedResult, DataTieringException exception) {
+ try {
+ boolean value = caller.call(dataTieringManager, path);
+ if (exception != null) {
+ fail("Expected DataTieringException to be thrown");
+ }
+ assertEquals(expectedResult, value);
+ } catch (DataTieringException e) {
+ if (exception == null) {
+ fail("Unexpected DataTieringException: " + e.getMessage());
+ }
+ assertEquals(exception.getMessage(), e.getMessage());
+ }
+ }
+
+ private void testDataTieringMethodWithKey(DataTieringMethodCallerWithKey caller,
+ BlockCacheKey key, boolean expectedResult, DataTieringException exception) {
+ try {
+ boolean value = caller.call(dataTieringManager, key);
+ if (exception != null) {
+ fail("Expected DataTieringException to be thrown");
+ }
+ assertEquals(expectedResult, value);
+ } catch (DataTieringException e) {
+ if (exception == null) {
+ fail("Unexpected DataTieringException: " + e.getMessage());
+ }
+ assertEquals(exception.getMessage(), e.getMessage());
+ }
+ }
+
+ private void testDataTieringMethodWithPathExpectingException(
+ DataTieringMethodCallerWithPath caller, Path path, DataTieringException exception) {
+ testDataTieringMethodWithPath(caller, path, false, exception);
+ }
+
+ private void testDataTieringMethodWithPathNoException(DataTieringMethodCallerWithPath caller,
+ Path path, boolean expectedResult) {
+ testDataTieringMethodWithPath(caller, path, expectedResult, null);
+ }
+
+ private void testDataTieringMethodWithKeyExpectingException(DataTieringMethodCallerWithKey caller,
+ BlockCacheKey key, DataTieringException exception) {
+ testDataTieringMethodWithKey(caller, key, false, exception);
+ }
+
+ private void testDataTieringMethodWithKeyNoException(DataTieringMethodCallerWithKey caller,
+ BlockCacheKey key, boolean expectedResult) {
+ testDataTieringMethodWithKey(caller, key, expectedResult, null);
+ }
+
+ private static void setupOnlineRegions() throws IOException {
+ testOnlineRegions = new HashMap<>();
+ hStoreFiles = new ArrayList<>();
+
+ long day = 24 * 60 * 60 * 1000;
+ long currentTime = System.currentTimeMillis();
+
+ HRegion region1 = createHRegion("table1");
+
+ HStore hStore11 = createHStore(region1, "cf1", getConfWithTimeRangeDataTieringEnabled(day));
+ hStoreFiles
+ .add(createHStoreFile(hStore11.getStoreContext().getFamilyStoreDirectoryPath(), currentTime));
+ hStore11.refreshStoreFiles();
+ HStore hStore12 = createHStore(region1, "cf2");
+ hStoreFiles.add(createHStoreFile(hStore12.getStoreContext().getFamilyStoreDirectoryPath(),
+ currentTime - day));
+ hStore12.refreshStoreFiles();
+
+ region1.stores.put(Bytes.toBytes("cf1"), hStore11);
+ region1.stores.put(Bytes.toBytes("cf2"), hStore12);
+
+ HRegion region2 =
+ createHRegion("table2", getConfWithTimeRangeDataTieringEnabled((long) (2.5 * day)));
+
+ HStore hStore21 = createHStore(region2, "cf1");
+ hStoreFiles.add(createHStoreFile(hStore21.getStoreContext().getFamilyStoreDirectoryPath(),
+ currentTime - 2 * day));
+ hStore21.refreshStoreFiles();
+ HStore hStore22 = createHStore(region2, "cf2");
+ hStoreFiles.add(createHStoreFile(hStore22.getStoreContext().getFamilyStoreDirectoryPath(),
+ currentTime - 3 * day));
+ hStore22.refreshStoreFiles();
+
+ region2.stores.put(Bytes.toBytes("cf1"), hStore21);
+ region2.stores.put(Bytes.toBytes("cf2"), hStore22);
+
+ for (HStoreFile file : hStoreFiles) {
+ file.initReader();
+ }
+
+ testOnlineRegions.put(region1.getRegionInfo().getEncodedName(), region1);
+ testOnlineRegions.put(region2.getRegionInfo().getEncodedName(), region2);
+ }
+
+ private static HRegion createHRegion(String table) throws IOException {
+ return createHRegion(table, defaultConf);
+ }
+
+ private static HRegion createHRegion(String table, Configuration conf) throws IOException {
+ TableName tableName = TableName.valueOf(table);
+
+ TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName)
+ .setValue(DataTieringManager.DATATIERING_KEY, conf.get(DataTieringManager.DATATIERING_KEY))
+ .setValue(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY,
+ conf.get(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY))
+ .build();
+ RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).build();
+
+ Configuration testConf = new Configuration(conf);
+ CommonFSUtils.setRootDir(testConf, testDir);
+ HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs,
+ CommonFSUtils.getTableDir(testDir, hri.getTable()), hri);
+
+ return new HRegion(regionFs, null, conf, htd, null);
+ }
+
+ private static HStore createHStore(HRegion region, String columnFamily) throws IOException {
+ return createHStore(region, columnFamily, defaultConf);
+ }
+
+ private static HStore createHStore(HRegion region, String columnFamily, Configuration conf)
+ throws IOException {
+ ColumnFamilyDescriptor columnFamilyDescriptor =
+ ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(columnFamily))
+ .setValue(DataTieringManager.DATATIERING_KEY, conf.get(DataTieringManager.DATATIERING_KEY))
+ .setValue(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY,
+ conf.get(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY))
+ .build();
+
+ return new HStore(region, columnFamilyDescriptor, conf, false);
+ }
+
+ private static Configuration getConfWithTimeRangeDataTieringEnabled(long hotDataAge) {
+ Configuration conf = new Configuration(defaultConf);
+ conf.set(DataTieringManager.DATATIERING_KEY, DataTieringType.TIME_RANGE.name());
+ conf.set(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY, String.valueOf(hotDataAge));
+ return conf;
+ }
+
+ private static HStoreFile createHStoreFile(Path storeDir, long timestamp) throws IOException {
+ String columnFamily = storeDir.getName();
+
+ StoreFileWriter storeFileWriter = new StoreFileWriter.Builder(defaultConf, cacheConf, fs)
+ .withOutputDir(storeDir).withFileContext(new HFileContextBuilder().build()).build();
+
+ writeStoreFileRandomData(storeFileWriter, Bytes.toBytes(columnFamily), Bytes.toBytes("random"),
+ timestamp);
+
+ return new HStoreFile(fs, storeFileWriter.getPath(), defaultConf, cacheConf, BloomType.NONE,
+ true);
+ }
+
+ private static void writeStoreFileRandomData(final StoreFileWriter writer, byte[] columnFamily,
+ byte[] qualifier, long timestamp) throws IOException {
+ try {
+ for (char d = 'a'; d <= 'z'; d++) {
+ for (char e = 'a'; e <= 'z'; e++) {
+ byte[] b = new byte[] { (byte) d, (byte) e };
+ writer.append(new KeyValue(b, columnFamily, qualifier, timestamp, b));
+ }
+ }
+ } finally {
+ writer.appendTrackedTimestampsToMetadata();
+ writer.close();
+ }
+ }
+}