You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by wc...@apache.org on 2023/11/28 13:20:23 UTC
(hbase) branch master updated: HBASE-28217 PrefetchExecutor should not run for files from CFs that have disabled BLOCKCACHE (#5535)
This is an automated email from the ASF dual-hosted git repository.
wchevreuil pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new dbfb516a557 HBASE-28217 PrefetchExecutor should not run for files from CFs that have disabled BLOCKCACHE (#5535)
dbfb516a557 is described below
commit dbfb516a557fc9358a68db714fe0933b33ba04f5
Author: Wellington Ramos Chevreuil <wc...@apache.org>
AuthorDate: Tue Nov 28 13:20:15 2023 +0000
HBASE-28217 PrefetchExecutor should not run for files from CFs that have disabled BLOCKCACHE (#5535)
Signed-off-by: Peter Somogyi <ps...@apache.org>
---
.../apache/hadoop/hbase/io/hfile/CacheConfig.java | 2 +-
.../hadoop/hbase/io/hfile/PrefetchExecutor.java | 5 +++
.../apache/hadoop/hbase/io/hfile/TestPrefetch.java | 52 +++++++++++++++++++++-
3 files changed, 56 insertions(+), 3 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
index 57f91fa19f4..4587eced616 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
@@ -343,7 +343,7 @@ public class CacheConfig {
/** Returns true if blocks should be prefetched into the cache on open, false if not */
public boolean shouldPrefetchOnOpen() {
- return this.prefetchOnOpen;
+ return this.prefetchOnOpen && this.cacheDataOnRead;
}
/** Returns true if blocks should be cached while writing during compaction, false if not */
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/PrefetchExecutor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/PrefetchExecutor.java
index 02fbc12e85c..4ae19193c8a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/PrefetchExecutor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/PrefetchExecutor.java
@@ -129,4 +129,9 @@ public final class PrefetchExecutor {
private PrefetchExecutor() {
}
+
+ /* Visible for testing only */
+ static ScheduledExecutorService getExecutorPool() {
+ return prefetchExecutorPool;
+ }
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java
index b58319179c5..0b45a930dce 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java
@@ -26,6 +26,7 @@ import static org.hamcrest.Matchers.allOf;
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.hasItems;
import static org.hamcrest.Matchers.not;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -35,6 +36,7 @@ import io.opentelemetry.sdk.trace.data.SpanData;
import java.io.IOException;
import java.util.List;
import java.util.Random;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.function.BiConsumer;
@@ -120,6 +122,40 @@ public class TestPrefetch {
assertTrue(cc.shouldPrefetchOnOpen());
}
+ @Test
+ public void testPrefetchBlockCacheDisabled() throws Exception {
+ ScheduledThreadPoolExecutor poolExecutor =
+ (ScheduledThreadPoolExecutor) PrefetchExecutor.getExecutorPool();
+ long totalCompletedBefore = poolExecutor.getCompletedTaskCount();
+ long queueBefore = poolExecutor.getQueue().size();
+ ColumnFamilyDescriptor columnFamilyDescriptor =
+ ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("f")).setPrefetchBlocksOnOpen(true)
+ .setBlockCacheEnabled(false).build();
+ HFileContext meta = new HFileContextBuilder().withBlockSize(DATA_BLOCK_SIZE).build();
+ CacheConfig cacheConfig =
+ new CacheConfig(conf, columnFamilyDescriptor, blockCache, ByteBuffAllocator.HEAP);
+ Path storeFile = writeStoreFile("testPrefetchBlockCacheDisabled", meta, cacheConfig);
+ readStoreFile(storeFile, (r, o) -> {
+ HFileBlock block = null;
+ try {
+ block = r.readBlock(o, -1, false, true, false, true, null, null);
+ } catch (IOException e) {
+ fail(e.getMessage());
+ }
+ return block;
+ }, (key, block) -> {
+ boolean isCached = blockCache.getBlock(key, true, false, true) != null;
+ if (
+ block.getBlockType() == BlockType.DATA || block.getBlockType() == BlockType.ROOT_INDEX
+ || block.getBlockType() == BlockType.INTERMEDIATE_INDEX
+ ) {
+ assertFalse(isCached);
+ }
+ }, cacheConfig);
+ assertEquals(totalCompletedBefore + queueBefore,
+ poolExecutor.getCompletedTaskCount() + poolExecutor.getQueue().size());
+ }
+
@Test
public void testPrefetch() throws Exception {
TraceUtil.trace(() -> {
@@ -212,8 +248,15 @@ public class TestPrefetch {
private void readStoreFile(Path storeFilePath,
BiFunction<HFile.Reader, Long, HFileBlock> readFunction,
BiConsumer<BlockCacheKey, HFileBlock> validationFunction) throws Exception {
+ readStoreFile(storeFilePath, readFunction, validationFunction, cacheConf);
+ }
+
+ private void readStoreFile(Path storeFilePath,
+ BiFunction<HFile.Reader, Long, HFileBlock> readFunction,
+ BiConsumer<BlockCacheKey, HFileBlock> validationFunction, CacheConfig cacheConfig)
+ throws Exception {
// Open the file
- HFile.Reader reader = HFile.createReader(fs, storeFilePath, cacheConf, true, conf);
+ HFile.Reader reader = HFile.createReader(fs, storeFilePath, cacheConfig, true, conf);
while (!reader.prefetchComplete()) {
// Sleep for a bit
@@ -350,8 +393,13 @@ public class TestPrefetch {
}
private Path writeStoreFile(String fname, HFileContext context) throws IOException {
+ return writeStoreFile(fname, context, cacheConf);
+ }
+
+ private Path writeStoreFile(String fname, HFileContext context, CacheConfig cacheConfig)
+ throws IOException {
Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(), fname);
- StoreFileWriter sfw = new StoreFileWriter.Builder(conf, cacheConf, fs)
+ StoreFileWriter sfw = new StoreFileWriter.Builder(conf, cacheConfig, fs)
.withOutputDir(storeFileParentDir).withFileContext(context).build();
Random rand = ThreadLocalRandom.current();
final int rowLen = 32;