You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2016/04/01 16:26:58 UTC
lucene-solr:branch_6x: SOLR-8856: Do not cache merge or 'read once'
contexts in the hdfs block cache.
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x d9d88b894 -> 461c9b4fe
SOLR-8856: Do not cache merge or 'read once' contexts in the hdfs block cache.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/461c9b4f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/461c9b4f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/461c9b4f
Branch: refs/heads/branch_6x
Commit: 461c9b4fef5c173c96ac2aa68daa87ef2bed2c16
Parents: d9d88b8
Author: markrmiller <ma...@apache.org>
Authored: Fri Apr 1 10:26:54 2016 -0400
Committer: markrmiller <ma...@apache.org>
Committed: Fri Apr 1 10:26:54 2016 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
.../apache/solr/core/HdfsDirectoryFactory.java | 11 +++++++++-
.../solr/store/blockcache/BlockDirectory.java | 22 ++++++++++++++++++++
3 files changed, 34 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/461c9b4f/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 5224f1b..2aba27d 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -77,6 +77,8 @@ Optimizations
* SOLR-8745: Deprecate costly ZkStateReader.updateClusterState(), replace with a narrow
forceUpdateCollection(collection) (Scott Blum via shalin)
+
+* SOLR-8856: Do not cache merge or 'read once' contexts in the hdfs block cache. (Mark Miller, Mike Drob)
Other Changes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/461c9b4f/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
index 0d53667..d22cbe3 100644
--- a/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
+++ b/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
@@ -90,12 +90,19 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
public static final String CONFIG_DIRECTORY = "solr.hdfs.confdir";
+ public static final String CACHE_MERGES = "solr.hdfs.blockcache.cachemerges";
+ public static final String CACHE_READONCE = "solr.hdfs.blockcache.cachereadonce";
+
private SolrParams params;
private String hdfsDataDir;
private String confDir;
+ private boolean cacheReadOnce;
+
+ private boolean cacheMerges;
+
private static BlockCache globalBlockCache;
public static Metrics metrics;
@@ -143,6 +150,8 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
} else {
LOG.info(HDFS_HOME + "=" + this.hdfsDataDir);
}
+ cacheMerges = getConfig(CACHE_MERGES, false);
+ cacheReadOnce = getConfig(CACHE_READONCE, false);
boolean kerberosEnabled = getConfig(KERBEROS_ENABLED, false);
LOG.info("Solr Kerberos Authentication "
+ (kerberosEnabled ? "enabled" : "disabled"));
@@ -214,7 +223,7 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
Cache cache = new BlockDirectoryCache(blockCache, path, metrics, blockCacheGlobal);
hdfsDir = new HdfsDirectory(new Path(path), lockFactory, conf);
- dir = new BlockDirectory(path, hdfsDir, cache, null, blockCacheReadEnabled, false);
+ dir = new BlockDirectory(path, hdfsDir, cache, null, blockCacheReadEnabled, false, cacheMerges, cacheReadOnce);
} else {
hdfsDir = new HdfsDirectory(new Path(path), lockFactory, conf);
dir = hdfsDir;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/461c9b4f/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java b/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java
index d890ee8..d26bc44 100644
--- a/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java
+++ b/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java
@@ -91,10 +91,21 @@ public class BlockDirectory extends FilterDirectory {
private final boolean blockCacheReadEnabled;
private final boolean blockCacheWriteEnabled;
+ private boolean cacheMerges;
+ private boolean cacheReadOnce;
+
public BlockDirectory(String dirName, Directory directory, Cache cache,
Set<String> blockCacheFileTypes, boolean blockCacheReadEnabled,
boolean blockCacheWriteEnabled) throws IOException {
+ this(dirName, directory, cache, blockCacheFileTypes, blockCacheReadEnabled, blockCacheWriteEnabled, true, true);
+ }
+
+ public BlockDirectory(String dirName, Directory directory, Cache cache,
+ Set<String> blockCacheFileTypes, boolean blockCacheReadEnabled,
+ boolean blockCacheWriteEnabled, boolean cacheMerges, boolean cacheReadOnce) throws IOException {
super(directory);
+ this.cacheMerges = cacheMerges;
+ this.cacheReadOnce = cacheReadOnce;
this.dirName = dirName;
blockSize = BLOCK_SIZE;
this.cache = cache;
@@ -284,6 +295,17 @@ public class BlockDirectory extends FilterDirectory {
return false;
}
switch (context.context) {
+ // depending on params, we don't cache on merges or when only reading once
+ case MERGE: {
+ return cacheMerges;
+ }
+ case READ: {
+ if (context.readOnce) {
+ return cacheReadOnce;
+ } else {
+ return true;
+ }
+ }
default: {
return true;
}