You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2018/01/29 17:02:02 UTC
lucene-solr:branch_7x: SOLR-11873: Use time based expiration cache in
all necessary places in HdfsDirectoryFactory.
Repository: lucene-solr
Updated Branches:
refs/heads/branch_7x 603bb7fb1 -> e16d50b75
SOLR-11873: Use time based expiration cache in all necessary places in HdfsDirectoryFactory.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e16d50b7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e16d50b7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e16d50b7
Branch: refs/heads/branch_7x
Commit: e16d50b75b02f616d998bc3e0121a38c62e7daf0
Parents: 603bb7f
Author: markrmiller <ma...@apache.org>
Authored: Mon Jan 29 09:49:44 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Mon Jan 29 11:01:51 2018 -0600
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../apache/solr/core/HdfsDirectoryFactory.java | 46 +++++++-------------
2 files changed, 18 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e16d50b7/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index fa560b6..8ec11bb 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -158,6 +158,8 @@ Bug Fixes
* SOLR-10525: Stacked recovery requests do no cancel an in progress recovery first. (Mike Drob via Cao Manh Dat)
+* SOLR-11873: Use time based expiration cache in all necessary places in HdfsDirectoryFactory. (Mihaly Toth via Mark Miller)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e16d50b7/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
index f248152..761745b 100644
--- a/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
+++ b/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
@@ -322,14 +322,7 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
@Override
public boolean exists(String path) {
final Path hdfsDirPath = new Path(path);
- final Configuration conf = getConf();
- FileSystem fileSystem = null;
- try {
- // no need to close the fs, the cache will do it
- fileSystem = tmpFsCache.get(path, () -> FileSystem.get(hdfsDirPath.toUri(), conf));
- } catch (ExecutionException e) {
- throw new RuntimeException(e);
- }
+ FileSystem fileSystem = getCachedFileSystem(path);
try {
return fileSystem.exists(hdfsDirPath);
@@ -349,16 +342,8 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
protected synchronized void removeDirectory(final CacheValue cacheValue)
throws IOException {
- final Configuration conf = getConf();
- FileSystem fileSystem = null;
-
- try {
- // no need to close the fs, the cache will do it
- fileSystem = tmpFsCache.get(cacheValue.path, () -> FileSystem.get(new Path(cacheValue.path).toUri(), conf));
- } catch (ExecutionException e) {
- throw new RuntimeException(e);
- }
-
+ FileSystem fileSystem = getCachedFileSystem(cacheValue.path);
+
try {
boolean success = fileSystem.delete(new Path(cacheValue.path), true);
if (!success) {
@@ -438,11 +423,9 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
@Override
public long size(String path) throws IOException {
Path hdfsDirPath = new Path(path);
- FileSystem fileSystem = null;
+ FileSystem fileSystem = getCachedFileSystem(path);
try {
- fileSystem = FileSystem.newInstance(hdfsDirPath.toUri(), getConf());
- long size = fileSystem.getContentSummary(hdfsDirPath).getLength();
- return size;
+ return fileSystem.getContentSummary(hdfsDirPath).getLength();
} catch (IOException e) {
LOG.error("Error checking if hdfs path exists", e);
throw new SolrException(ErrorCode.SERVER_ERROR, "Error checking if hdfs path exists", e);
@@ -450,7 +433,16 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
IOUtils.closeQuietly(fileSystem);
}
}
-
+
+ private FileSystem getCachedFileSystem(String path) {
+ try {
+ // no need to close the fs, the cache will do it
+ return tmpFsCache.get(path, () -> FileSystem.get(new Path(path).toUri(), getConf()));
+ } catch (ExecutionException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
public String getConfDir() {
return confDir;
}
@@ -515,13 +507,7 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
// Get the FileSystem object
final Path dataDirPath = new Path(dataDir);
- final Configuration conf = getConf();
- FileSystem fileSystem = null;
- try {
- fileSystem = tmpFsCache.get(dataDir, () -> FileSystem.get(dataDirPath.toUri(), conf));
- } catch (ExecutionException e) {
- throw new RuntimeException(e);
- }
+ FileSystem fileSystem = getCachedFileSystem(dataDir);
boolean pathExists = false;
try {