You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by wc...@apache.org on 2023/11/28 13:47:46 UTC
(hbase) branch branch-2.5 updated: HBASE-28217 PrefetchExecutor should not run for files from CFs that have disabled BLOCKCACHE (#5535)
This is an automated email from the ASF dual-hosted git repository.
wchevreuil pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.5 by this push:
new 42851f6ed87 HBASE-28217 PrefetchExecutor should not run for files from CFs that have disabled BLOCKCACHE (#5535)
42851f6ed87 is described below
commit 42851f6ed8717d8545b49088cb197634d8faf028
Author: Wellington Ramos Chevreuil <wc...@apache.org>
AuthorDate: Tue Nov 28 13:20:15 2023 +0000
HBASE-28217 PrefetchExecutor should not run for files from CFs that have disabled BLOCKCACHE (#5535)
Signed-off-by: Peter Somogyi <ps...@apache.org>
---
.../apache/hadoop/hbase/io/hfile/CacheConfig.java | 2 +-
.../hadoop/hbase/io/hfile/PrefetchExecutor.java | 5 +++
.../apache/hadoop/hbase/io/hfile/TestPrefetch.java | 52 +++++++++++++++++++++-
3 files changed, 56 insertions(+), 3 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
index 32c6dfd6c2a..9f64baa48c4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
@@ -337,7 +337,7 @@ public class CacheConfig {
/** Returns true if blocks should be prefetched into the cache on open, false if not */
public boolean shouldPrefetchOnOpen() {
- return this.prefetchOnOpen;
+ return this.prefetchOnOpen && this.cacheDataOnRead;
}
/** Returns true if blocks should be cached while writing during compaction, false if not */
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/PrefetchExecutor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/PrefetchExecutor.java
index 8bf63909fcb..912241fbb95 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/PrefetchExecutor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/PrefetchExecutor.java
@@ -127,4 +127,9 @@ public final class PrefetchExecutor {
private PrefetchExecutor() {
}
+
+ /* Visible for testing only */
+ static ScheduledExecutorService getExecutorPool() {
+ return prefetchExecutorPool;
+ }
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java
index d2b94177671..dee8dbb9076 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java
@@ -25,6 +25,7 @@ import static org.hamcrest.Matchers.allOf;
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.hasItems;
import static org.hamcrest.Matchers.not;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -34,6 +35,7 @@ import io.opentelemetry.sdk.trace.data.SpanData;
import java.io.IOException;
import java.util.List;
import java.util.Random;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.function.BiConsumer;
@@ -117,6 +119,40 @@ public class TestPrefetch {
assertTrue(cc.shouldPrefetchOnOpen());
}
+ @Test
+ public void testPrefetchBlockCacheDisabled() throws Exception {
+ ScheduledThreadPoolExecutor poolExecutor =
+ (ScheduledThreadPoolExecutor) PrefetchExecutor.getExecutorPool();
+ long totalCompletedBefore = poolExecutor.getCompletedTaskCount();
+ long queueBefore = poolExecutor.getQueue().size();
+ ColumnFamilyDescriptor columnFamilyDescriptor =
+ ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("f")).setPrefetchBlocksOnOpen(true)
+ .setBlockCacheEnabled(false).build();
+ HFileContext meta = new HFileContextBuilder().withBlockSize(DATA_BLOCK_SIZE).build();
+ CacheConfig cacheConfig =
+ new CacheConfig(conf, columnFamilyDescriptor, blockCache, ByteBuffAllocator.HEAP);
+ Path storeFile = writeStoreFile("testPrefetchBlockCacheDisabled", meta, cacheConfig);
+ readStoreFile(storeFile, (r, o) -> {
+ HFileBlock block = null;
+ try {
+ block = r.readBlock(o, -1, false, true, false, true, null, null);
+ } catch (IOException e) {
+ fail(e.getMessage());
+ }
+ return block;
+ }, (key, block) -> {
+ boolean isCached = blockCache.getBlock(key, true, false, true) != null;
+ if (
+ block.getBlockType() == BlockType.DATA || block.getBlockType() == BlockType.ROOT_INDEX
+ || block.getBlockType() == BlockType.INTERMEDIATE_INDEX
+ ) {
+ assertFalse(isCached);
+ }
+ }, cacheConfig);
+ assertEquals(totalCompletedBefore + queueBefore,
+ poolExecutor.getCompletedTaskCount() + poolExecutor.getQueue().size());
+ }
+
@Test
public void testPrefetch() throws Exception {
TraceUtil.trace(() -> {
@@ -209,8 +245,15 @@ public class TestPrefetch {
private void readStoreFile(Path storeFilePath,
BiFunction<HFile.Reader, Long, HFileBlock> readFunction,
BiConsumer<BlockCacheKey, HFileBlock> validationFunction) throws Exception {
+ readStoreFile(storeFilePath, readFunction, validationFunction, cacheConf);
+ }
+
+ private void readStoreFile(Path storeFilePath,
+ BiFunction<HFile.Reader, Long, HFileBlock> readFunction,
+ BiConsumer<BlockCacheKey, HFileBlock> validationFunction, CacheConfig cacheConfig)
+ throws Exception {
// Open the file
- HFile.Reader reader = HFile.createReader(fs, storeFilePath, cacheConf, true, conf);
+ HFile.Reader reader = HFile.createReader(fs, storeFilePath, cacheConfig, true, conf);
while (!reader.prefetchComplete()) {
// Sleep for a bit
@@ -297,8 +340,13 @@ public class TestPrefetch {
}
private Path writeStoreFile(String fname, HFileContext context) throws IOException {
+ return writeStoreFile(fname, context, cacheConf);
+ }
+
+ private Path writeStoreFile(String fname, HFileContext context, CacheConfig cacheConfig)
+ throws IOException {
Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(), fname);
- StoreFileWriter sfw = new StoreFileWriter.Builder(conf, cacheConf, fs)
+ StoreFileWriter sfw = new StoreFileWriter.Builder(conf, cacheConfig, fs)
.withOutputDir(storeFileParentDir).withFileContext(context).build();
Random rand = ThreadLocalRandom.current();
final int rowLen = 32;