You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ec...@apache.org on 2016/04/25 23:12:55 UTC
[11/50] [abbrv] hbase git commit: HBASE-15385 PREFETCH_BLOCKS_ON_OPEN
in HColumnDescriptor is ignored
HBASE-15385 PREFETCH_BLOCKS_ON_OPEN in HColumnDescriptor is ignored
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fa215a67
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fa215a67
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fa215a67
Branch: refs/heads/HBASE-14850
Commit: fa215a67e20da8c1a450b16db27c73ee3f9d02c0
Parents: 3e1bdcc
Author: stack <st...@apache.org>
Authored: Wed Apr 20 09:38:30 2016 -0700
Committer: stack <st...@apache.org>
Committed: Wed Apr 20 09:38:30 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/io/hfile/CacheConfig.java | 7 +++++--
.../apache/hadoop/hbase/io/hfile/HFileReaderImpl.java | 6 +++---
.../org/apache/hadoop/hbase/io/hfile/TestPrefetch.java | 13 +++++++++++++
src/main/asciidoc/_chapters/performance.adoc | 5 +++++
4 files changed, 26 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/fa215a67/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
index 2680c3d..ffb9424 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
@@ -246,11 +246,14 @@ public class CacheConfig {
HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1) || family.isCacheDataInL1(),
conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY, DROP_BEHIND_CACHE_COMPACTION_DEFAULT)
);
+ LOG.info("Created cacheConfig for " + family.getNameAsString() + ": " + this);
}
/**
* Create a cache configuration using the specified configuration object and
- * defaults for family level settings.
+ * defaults for family level settings. Only use if no column family context. Prefer
+ * {@link CacheConfig#CacheConfig(Configuration, HColumnDescriptor)}
+ * @see #CacheConfig(Configuration, HColumnDescriptor)
* @param conf hbase configuration
*/
public CacheConfig(Configuration conf) {
@@ -268,6 +271,7 @@ public class CacheConfig {
HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1),
conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY, DROP_BEHIND_CACHE_COMPACTION_DEFAULT)
);
+ LOG.info("Created cacheConfig: " + this);
}
/**
@@ -303,7 +307,6 @@ public class CacheConfig {
this.prefetchOnOpen = prefetchOnOpen;
this.cacheDataInL1 = cacheDataInL1;
this.dropBehindCompaction = dropBehindCompaction;
- LOG.info(this);
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/fa215a67/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
index 0b1ac83..9a63d2d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
@@ -253,7 +253,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable {
try {
end = getTrailer().getLoadOnOpenDataOffset();
if (LOG.isTraceEnabled()) {
- LOG.trace("File=" + path.toString() + ", offset=" + offset + ", end=" + end);
+ LOG.trace("Prefetch=" + path.toString() + ", offset=" + offset + ", end=" + end);
}
// TODO: Could we use block iterator in here? Would that get stuff into the cache?
HFileBlock prevBlock = null;
@@ -279,11 +279,11 @@ public class HFileReaderImpl implements HFile.Reader, Configurable {
} catch (IOException e) {
// IOExceptions are probably due to region closes (relocation, etc.)
if (LOG.isTraceEnabled()) {
- LOG.trace("File=" + path.toString() + ", offset=" + offset + ", end=" + end, e);
+ LOG.trace("Prefetch=" + path.toString() + ", offset=" + offset + ", end=" + end, e);
}
} catch (Exception e) {
// Other exceptions are interesting
- LOG.warn("File=" + path.toString() + ", offset=" + offset + ", end=" + end, e);
+ LOG.warn("Prefetch=" + path.toString() + ", offset=" + offset + ", end=" + end, e);
} finally {
PrefetchExecutor.complete(path);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/fa215a67/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java
index e31ac52..4c3db03 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java
@@ -27,6 +27,9 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.fs.HFileSystem;
@@ -60,6 +63,16 @@ public class TestPrefetch {
cacheConf = new CacheConfig(conf);
}
+ @Test
+ public void testPrefetchSetInHCDWorks() {
+ HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes("f"));
+ hcd.setPrefetchBlocksOnOpen(true);
+ Configuration c = HBaseConfiguration.create();
+ assertFalse(c.getBoolean(CacheConfig.PREFETCH_BLOCKS_ON_OPEN_KEY, false));
+ CacheConfig cc = new CacheConfig(c, hcd);
+ assertTrue(cc.shouldPrefetchOnOpen());
+ }
+
@Test(timeout=60000)
public void testPrefetch() throws Exception {
Path storeFile = writeStoreFile();
http://git-wip-us.apache.org/repos/asf/hbase/blob/fa215a67/src/main/asciidoc/_chapters/performance.adoc
----------------------------------------------------------------------
diff --git a/src/main/asciidoc/_chapters/performance.adoc b/src/main/asciidoc/_chapters/performance.adoc
index 01956d5..efb6ace 100644
--- a/src/main/asciidoc/_chapters/performance.adoc
+++ b/src/main/asciidoc/_chapters/performance.adoc
@@ -207,6 +207,11 @@ tableDesc.addFamily(cfDesc);
See the API documentation for
link:https://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/io/hfile/CacheConfig.html[CacheConfig].
+To see prefetch in operation, enable TRACE level logging on
+`org.apache.hadoop.hbase.io.hfile.HFileReaderImpl` in hbase-2.0+
+or on `org.apache.hadoop.hbase.io.hfile.HFileReaderV2` in earlier versions, hbase-1.x, of HBase.
+
+
[[perf.rs.memstore.size]]
=== `hbase.regionserver.global.memstore.size`