You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2014/08/25 01:27:22 UTC
svn commit: r1620237 - in /lucene/dev/branches/branch_4x: ./ solr/
solr/core/ solr/core/src/java/org/apache/solr/store/blockcache/
Author: markrmiller
Date: Sun Aug 24 23:27:22 2014
New Revision: 1620237
URL: http://svn.apache.org/r1620237
Log:
SOLR-6425: If you using the new global hdfs block cache option, you can end up reading corrupt files on file name reuse.
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockCache.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectoryCache.java
Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1620237&r1=1620236&r2=1620237&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Sun Aug 24 23:27:22 2014
@@ -52,6 +52,9 @@ Bug Fixes
* SOLR-6424: The hdfs block cache BLOCKCACHE_WRITE_ENABLED is not defaulting to false like it
should. (Mark Miller)
+* SOLR-6425: If you using the new global hdfs block cache option, you can end up reading corrupt
+ files on file name reuse. (Mark Miller, Gregory Chanan)
+
Other Changes
---------------------
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockCache.java?rev=1620237&r1=1620236&r2=1620237&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockCache.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockCache.java Sun Aug 24 23:27:22 2014
@@ -81,7 +81,7 @@ public class BlockCache {
}
public void release(BlockCacheKey key) {
- releaseLocation(cache.get(key));
+ releaseLocation(cache.remove(key));
}
private void releaseLocation(BlockCacheLocation location) {
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java?rev=1620237&r1=1620236&r2=1620237&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java Sun Aug 24 23:27:22 2014
@@ -96,11 +96,6 @@ public class BlockDirectory extends Dire
public BlockDirectory(String dirName, Directory directory, Cache cache,
Set<String> blockCacheFileTypes, boolean blockCacheReadEnabled,
boolean blockCacheWriteEnabled) throws IOException {
- this(dirName, directory, cache, blockCacheFileTypes, blockCacheReadEnabled, blockCacheWriteEnabled, false);
- }
- public BlockDirectory(String dirName, Directory directory, Cache cache,
- Set<String> blockCacheFileTypes, boolean blockCacheReadEnabled,
- boolean blockCacheWriteEnabled, boolean releaseBlocksOnClose) throws IOException {
this.dirName = dirName;
this.directory = directory;
blockSize = BLOCK_SIZE;
@@ -244,11 +239,11 @@ public class BlockDirectory extends Dire
// segments.gen won't be removed above
cache.delete(dirName + "/" + "segments.gen");
- cache.releaseResources();
} catch (FileNotFoundException e) {
// the local file system folder may be gone
} finally {
directory.close();
+ cache.releaseResources();
}
}
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectoryCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectoryCache.java?rev=1620237&r1=1620236&r2=1620237&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectoryCache.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectoryCache.java Sun Aug 24 23:27:22 2014
@@ -31,7 +31,7 @@ public class BlockDirectoryCache impleme
private final BlockCache blockCache;
private final AtomicInteger counter = new AtomicInteger();
private final Map<String,Integer> names = new ConcurrentHashMap<>();
- private Set<BlockCacheKey> keys;
+ private Set<BlockCacheKey> keysToRelease;
private final String path;
private final Metrics metrics;
@@ -44,7 +44,7 @@ public class BlockDirectoryCache impleme
this.path = path;
this.metrics = metrics;
if (releaseBlocks) {
- keys = Collections.synchronizedSet(new HashSet<BlockCacheKey>());
+ keysToRelease = Collections.synchronizedSet(new HashSet<BlockCacheKey>());
}
}
@@ -74,9 +74,8 @@ public class BlockDirectoryCache impleme
blockCacheKey.setPath(path);
blockCacheKey.setBlock(blockId);
blockCacheKey.setFile(file);
- blockCache.store(blockCacheKey, blockOffset, buffer, offset, length);
- if (keys != null) {
- keys.add(blockCacheKey);
+ if (blockCache.store(blockCacheKey, blockOffset, buffer, offset, length) && keysToRelease != null) {
+ keysToRelease.add(blockCacheKey);
}
}
@@ -117,8 +116,8 @@ public class BlockDirectoryCache impleme
@Override
public void releaseResources() {
- if (keys != null) {
- for (BlockCacheKey key : keys) {
+ if (keysToRelease != null) {
+ for (BlockCacheKey key : keysToRelease) {
blockCache.release(key);
}
}