You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2014/03/04 05:16:52 UTC
svn commit: r1573849 - in /lucene/dev/branches/branch_4x: ./ solr/
solr/core/ solr/core/src/java/org/apache/solr/core/
solr/core/src/java/org/apache/solr/store/blockcache/
solr/core/src/java/org/apache/solr/store/hdfs/
solr/core/src/test-files/solr/col...
Author: markrmiller
Date: Tue Mar 4 04:16:51 2014
New Revision: 1573849
URL: http://svn.apache.org/r1573849
Log:
SOLR-5714: You can now use one pool of memory for for the HDFS block cache that all collections share.
Added:
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java
- copied, changed from r1573847, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java
lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/StopableIndexingThread.java
- copied, changed from r1573847, lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/cloud/StopableIndexingThread.java
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockCache.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockCacheKey.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockCacheLocation.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectoryCache.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockLocks.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BufferStore.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/Cache.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/CachedIndexOutput.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/ReusedBufferedIndexOutput.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/Store.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsFileReader.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsFileWriter.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/NullIndexOutput.java
lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/store/blockcache/BlockCacheTest.java
lucene/dev/branches/branch_4x/solr/example/ (props changed)
lucene/dev/branches/branch_4x/solr/example/solr/collection1/conf/solrconfig.xml
lucene/dev/branches/branch_4x/solr/test-framework/ (props changed)
lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1573849&r1=1573848&r2=1573849&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Tue Mar 4 04:16:51 2014
@@ -42,6 +42,9 @@ New Features
* SOLR-5183: JSON updates now support nested child documents using a
"_childDocument_" object key. (Varun Thacker, hossman)
+* SOLR-5714: You can now use one pool of memory for for the HDFS block cache
+ that all collections share. (Mark Miller, Gregory Chanan)
+
Bug Fixes
----------------------
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java?rev=1573849&r1=1573848&r2=1573849&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java Tue Mar 4 04:16:51 2014
@@ -51,6 +51,7 @@ public class HdfsDirectoryFactory extend
public static final String BLOCKCACHE_SLAB_COUNT = "solr.hdfs.blockcache.slab.count";
public static final String BLOCKCACHE_DIRECT_MEMORY_ALLOCATION = "solr.hdfs.blockcache.direct.memory.allocation";
public static final String BLOCKCACHE_ENABLED = "solr.hdfs.blockcache.enabled";
+ public static final String BLOCKCACHE_GLOBAL = "solr.hdfs.blockcache.global";
public static final String BLOCKCACHE_READ_ENABLED = "solr.hdfs.blockcache.read.enabled";
public static final String BLOCKCACHE_WRITE_ENABLED = "solr.hdfs.blockcache.write.enabled";
@@ -72,6 +73,8 @@ public class HdfsDirectoryFactory extend
private String hdfsDataDir;
private String confDir;
+
+ private static BlockCache globalBlockCache;
public static Metrics metrics;
private static Boolean kerberosInit;
@@ -102,6 +105,7 @@ public class HdfsDirectoryFactory extend
}
boolean blockCacheEnabled = params.getBool(BLOCKCACHE_ENABLED, true);
+ boolean blockCacheGlobal = params.getBool(BLOCKCACHE_GLOBAL, false); // default to false for back compat
boolean blockCacheReadEnabled = params.getBool(BLOCKCACHE_READ_ENABLED,
true);
boolean blockCacheWriteEnabled = params.getBool(BLOCKCACHE_WRITE_ENABLED, true);
@@ -117,8 +121,6 @@ public class HdfsDirectoryFactory extend
boolean directAllocation = params.getBool(
BLOCKCACHE_DIRECT_MEMORY_ALLOCATION, true);
- BlockCache blockCache;
-
int slabSize = numberOfBlocksPerBank * blockSize;
LOG.info(
"Number of slabs of block cache [{}] with direct memory allocation set to [{}]",
@@ -131,22 +133,13 @@ public class HdfsDirectoryFactory extend
int bufferSize = params.getInt("solr.hdfs.blockcache.bufferstore.buffersize", 128);
int bufferCount = params.getInt("solr.hdfs.blockcache.bufferstore.buffercount", 128 * 128);
- BufferStore.initNewBuffer(bufferSize, bufferCount);
- long totalMemory = (long) bankCount * (long) numberOfBlocksPerBank
- * (long) blockSize;
- try {
- blockCache = new BlockCache(metrics, directAllocation, totalMemory,
- slabSize, blockSize);
- } catch (OutOfMemoryError e) {
- throw new RuntimeException(
- "The max direct memory is likely too low. Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args)"
- + " or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap,"
- + " your java heap size might not be large enough."
- + " Failed allocating ~" + totalMemory / 1000000.0 + " MB.", e);
- }
- Cache cache = new BlockDirectoryCache(blockCache, metrics);
+ BlockCache blockCache = getBlockDirectoryCache(path, numberOfBlocksPerBank,
+ blockSize, bankCount, directAllocation, slabSize,
+ bufferSize, bufferCount, blockCacheGlobal);
+
+ Cache cache = new BlockDirectoryCache(blockCache, path, metrics);
HdfsDirectory hdfsDirectory = new HdfsDirectory(new Path(path), conf);
- dir = new BlockDirectory("solrcore", hdfsDirectory, cache, null,
+ dir = new BlockDirectory(path, hdfsDirectory, cache, null,
blockCacheReadEnabled, blockCacheWriteEnabled);
} else {
dir = new HdfsDirectory(new Path(path), conf);
@@ -164,6 +157,45 @@ public class HdfsDirectoryFactory extend
}
return dir;
}
+
+ private BlockCache getBlockDirectoryCache(String path,
+ int numberOfBlocksPerBank, int blockSize, int bankCount,
+ boolean directAllocation, int slabSize, int bufferSize, int bufferCount, boolean staticBlockCache) {
+ if (!staticBlockCache) {
+ LOG.info("Creating new single instance HDFS BlockCache");
+ return createBlockCache(numberOfBlocksPerBank, blockSize, bankCount, directAllocation, slabSize, bufferSize, bufferCount);
+ }
+ LOG.info("Creating new global HDFS BlockCache");
+ synchronized (HdfsDirectoryFactory.class) {
+
+ if (globalBlockCache == null) {
+ globalBlockCache = createBlockCache(numberOfBlocksPerBank, blockSize, bankCount,
+ directAllocation, slabSize, bufferSize, bufferCount);
+ }
+ }
+ return globalBlockCache;
+ }
+
+ private BlockCache createBlockCache(int numberOfBlocksPerBank, int blockSize,
+ int bankCount, boolean directAllocation, int slabSize, int bufferSize,
+ int bufferCount) {
+ BufferStore.initNewBuffer(bufferSize, bufferCount);
+ long totalMemory = (long) bankCount * (long) numberOfBlocksPerBank
+ * (long) blockSize;
+
+ BlockCache blockCache;
+ try {
+ blockCache = new BlockCache(metrics, directAllocation, totalMemory, slabSize, blockSize);
+ } catch (OutOfMemoryError e) {
+ throw new RuntimeException(
+ "The max direct memory is likely too low. Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args)"
+ + " or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap,"
+ + " your java heap size might not be large enough."
+ + " Failed allocating ~" + totalMemory / 1000000.0 + " MB.",
+ e);
+ }
+ return blockCache;
+ }
@Override
public boolean exists(String path) {
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockCache.java?rev=1573849&r1=1573848&r2=1573849&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockCache.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockCache.java Tue Mar 4 04:16:51 2014
@@ -24,6 +24,9 @@ import java.util.concurrent.atomic.Atomi
import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
import com.googlecode.concurrentlinkedhashmap.EvictionListener;
+/**
+ * @lucene.experimental
+ */
public class BlockCache {
public static final int _128M = 134217728;
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockCacheKey.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockCacheKey.java?rev=1573849&r1=1573848&r2=1573849&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockCacheKey.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockCacheKey.java Tue Mar 4 04:16:51 2014
@@ -16,12 +16,23 @@ package org.apache.solr.store.blockcache
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
+/**
+ * @lucene.experimental
+ */
public class BlockCacheKey implements Cloneable {
private long block;
private int file;
+ private String path;
+ public String getPath() {
+ return path;
+ }
+
+ public void setPath(String path) {
+ this.path = path;
+ }
+
public long getBlock() {
return block;
}
@@ -44,9 +55,10 @@ public class BlockCacheKey implements Cl
int result = 1;
result = prime * result + (int) (block ^ (block >>> 32));
result = prime * result + file;
+ result = prime * result + ((path == null) ? 0 : path.hashCode());
return result;
}
-
+
@Override
public boolean equals(Object obj) {
if (this == obj) return true;
@@ -55,9 +67,12 @@ public class BlockCacheKey implements Cl
BlockCacheKey other = (BlockCacheKey) obj;
if (block != other.block) return false;
if (file != other.file) return false;
+ if (path == null) {
+ if (other.path != null) return false;
+ } else if (!path.equals(other.path)) return false;
return true;
}
-
+
@Override
public BlockCacheKey clone() {
try {
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockCacheLocation.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockCacheLocation.java?rev=1573849&r1=1573848&r2=1573849&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockCacheLocation.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockCacheLocation.java Tue Mar 4 04:16:51 2014
@@ -19,6 +19,9 @@ package org.apache.solr.store.blockcache
import java.util.concurrent.atomic.AtomicBoolean;
+/**
+ * @lucene.experimental
+ */
public class BlockCacheLocation {
private int block;
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java?rev=1573849&r1=1573848&r2=1573849&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java Tue Mar 4 04:16:51 2014
@@ -34,6 +34,9 @@ import org.apache.solr.store.hdfs.HdfsDi
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+/**
+ * @lucene.experimental
+ */
public class BlockDirectory extends Directory {
public static Logger LOG = LoggerFactory.getLogger(BlockDirectory.class);
@@ -82,11 +85,11 @@ public class BlockDirectory extends Dire
private Directory directory;
private int blockSize;
private String dirName;
- private Cache cache;
+ private final Cache cache;
private Set<String> blockCacheFileTypes;
private final boolean blockCacheReadEnabled;
private final boolean blockCacheWriteEnabled;
-
+
public BlockDirectory(String dirName, Directory directory, Cache cache,
Set<String> blockCacheFileTypes, boolean blockCacheReadEnabled,
boolean blockCacheWriteEnabled) throws IOException {
@@ -265,6 +268,15 @@ public class BlockDirectory extends Dire
return dirName + "/" + name;
}
+ /**
+ * Expert: mostly for tests
+ *
+ * @lucene.experimental
+ */
+ public Cache getCache() {
+ return cache;
+ }
+
@Override
public void copy(Directory to, String src, String dest, IOContext context)
throws IOException {
@@ -383,4 +395,13 @@ public class BlockDirectory extends Dire
return directory;
}
+
+ public boolean isBlockCacheReadEnabled() {
+ return blockCacheReadEnabled;
+ }
+
+ public boolean isBlockCacheWriteEnabled() {
+ return blockCacheWriteEnabled;
+ }
+
}
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectoryCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectoryCache.java?rev=1573849&r1=1573848&r2=1573849&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectoryCache.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectoryCache.java Tue Mar 4 04:16:51 2014
@@ -21,17 +21,31 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
+/**
+ * @lucene.experimental
+ */
public class BlockDirectoryCache implements Cache {
- private BlockCache blockCache;
+ private final BlockCache blockCache;
private AtomicInteger counter = new AtomicInteger();
private Map<String,Integer> names = new ConcurrentHashMap<String,Integer>();
+ private String path;
private Metrics metrics;
- public BlockDirectoryCache(BlockCache blockCache, Metrics metrics) {
+ public BlockDirectoryCache(BlockCache blockCache, String path, Metrics metrics) {
this.blockCache = blockCache;
+ this.path = path;
this.metrics = metrics;
}
+ /**
+ * Expert: mostly for tests
+ *
+ * @lucene.experimental
+ */
+ public BlockCache getBlockCache() {
+ return blockCache;
+ }
+
@Override
public void delete(String name) {
names.remove(name);
@@ -46,6 +60,7 @@ public class BlockDirectoryCache impleme
names.put(name, file);
}
BlockCacheKey blockCacheKey = new BlockCacheKey();
+ blockCacheKey.setPath(path);
blockCacheKey.setBlock(blockId);
blockCacheKey.setFile(file);
blockCache.store(blockCacheKey, blockOffset, buffer, offset, length);
@@ -59,6 +74,7 @@ public class BlockDirectoryCache impleme
return false;
}
BlockCacheKey blockCacheKey = new BlockCacheKey();
+ blockCacheKey.setPath(path);
blockCacheKey.setBlock(blockId);
blockCacheKey.setFile(file);
boolean fetch = blockCache.fetch(blockCacheKey, b, blockOffset, off,
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockLocks.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockLocks.java?rev=1573849&r1=1573848&r2=1573849&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockLocks.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BlockLocks.java Tue Mar 4 04:16:51 2014
@@ -21,6 +21,9 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.util.LongBitSet;
+/**
+ * @lucene.experimental
+ */
public class BlockLocks {
private AtomicLongArray bits;
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BufferStore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BufferStore.java?rev=1573849&r1=1573848&r2=1573849&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BufferStore.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/BufferStore.java Tue Mar 4 04:16:51 2014
@@ -22,7 +22,9 @@ import java.util.concurrent.BlockingQueu
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
-
+/**
+ * @lucene.experimental
+ */
public class BufferStore implements Store {
private static final Store EMPTY = new Store() {
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/Cache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/Cache.java?rev=1573849&r1=1573848&r2=1573849&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/Cache.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/Cache.java Tue Mar 4 04:16:51 2014
@@ -17,6 +17,9 @@ package org.apache.solr.store.blockcache
* limitations under the License.
*/
+/**
+ * @lucene.experimental
+ */
public interface Cache {
/**
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/CachedIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/CachedIndexOutput.java?rev=1573849&r1=1573848&r2=1573849&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/CachedIndexOutput.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/CachedIndexOutput.java Tue Mar 4 04:16:51 2014
@@ -21,10 +21,11 @@ import java.io.IOException;
import org.apache.lucene.store.IndexOutput;
-/*
+/**
* Cache the blocks as they are written. The cache file name is the name of
* the file until the file is closed, at which point the cache is updated
* to include the last modified date (which is unknown until that point).
+ * @lucene.experimental
*/
public class CachedIndexOutput extends ReusedBufferedIndexOutput {
private final BlockDirectory directory;
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.java?rev=1573849&r1=1573848&r2=1573849&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.java Tue Mar 4 04:16:51 2014
@@ -23,6 +23,9 @@ import java.io.IOException;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
+/**
+ * @lucene.experimental
+ */
public abstract class CustomBufferedIndexInput extends IndexInput {
public static final int BUFFER_SIZE = 32768;
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java?rev=1573849&r1=1573848&r2=1573849&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java Tue Mar 4 04:16:51 2014
@@ -29,6 +29,9 @@ import org.apache.hadoop.metrics.Metrics
import org.apache.hadoop.metrics.Updater;
import org.apache.hadoop.metrics.jvm.JvmMetrics;
+/**
+ * @lucene.experimental
+ */
public class Metrics implements Updater {
public static class MethodCall {
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/ReusedBufferedIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/ReusedBufferedIndexOutput.java?rev=1573849&r1=1573848&r2=1573849&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/ReusedBufferedIndexOutput.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/ReusedBufferedIndexOutput.java Tue Mar 4 04:16:51 2014
@@ -21,6 +21,9 @@ import java.io.IOException;
import org.apache.lucene.store.IndexOutput;
+/**
+ * @lucene.experimental
+ */
public abstract class ReusedBufferedIndexOutput extends IndexOutput {
public static final int BUFFER_SIZE = 1024;
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/Store.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/Store.java?rev=1573849&r1=1573848&r2=1573849&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/Store.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/blockcache/Store.java Tue Mar 4 04:16:51 2014
@@ -17,6 +17,9 @@ package org.apache.solr.store.blockcache
* limitations under the License.
*/
+/**
+ * @lucene.experimental
+ */
public interface Store {
byte[] takeBuffer(int bufferSize);
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsFileReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsFileReader.java?rev=1573849&r1=1573848&r2=1573849&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsFileReader.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsFileReader.java Tue Mar 4 04:16:51 2014
@@ -28,6 +28,9 @@ import org.apache.lucene.store.DataInput
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+/**
+ * @lucene.experimental
+ */
public class HdfsFileReader extends DataInput {
public static Logger LOG = LoggerFactory.getLogger(HdfsFileReader.class);
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsFileWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsFileWriter.java?rev=1573849&r1=1573848&r2=1573849&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsFileWriter.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsFileWriter.java Tue Mar 4 04:16:51 2014
@@ -32,6 +32,9 @@ import org.apache.lucene.store.DataOutpu
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+/**
+ * @lucene.experimental
+ */
public class HdfsFileWriter extends DataOutput implements Closeable {
public static Logger LOG = LoggerFactory.getLogger(HdfsFileWriter.class);
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/NullIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/NullIndexOutput.java?rev=1573849&r1=1573848&r2=1573849&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/NullIndexOutput.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/store/hdfs/NullIndexOutput.java Tue Mar 4 04:16:51 2014
@@ -21,6 +21,9 @@ import java.io.IOException;
import org.apache.lucene.store.IndexOutput;
+/**
+ * @lucene.experimental
+ */
public class NullIndexOutput extends IndexOutput {
private long pos;
Modified: lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml?rev=1573849&r1=1573848&r2=1573849&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml Tue Mar 4 04:16:51 2014
@@ -28,6 +28,7 @@
<int name="solr.hdfs.blockcache.blocksperbank">${solr.hdfs.blockcache.blocksperbank:1024}</int>
<str name="solr.hdfs.home">${solr.hdfs.home:}</str>
<str name="solr.hdfs.confdir">${solr.hdfs.confdir:}</str>
+ <str name="solr.hdfs.blockcache.global">${solr.hdfs.blockcache.global:false}</str>
</directoryFactory>
<dataDir>${solr.data.dir:}</dataDir>
Modified: lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml?rev=1573849&r1=1573848&r2=1573849&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml Tue Mar 4 04:16:51 2014
@@ -46,7 +46,8 @@
<double name="maxWriteMBPerSecMerge">3000000</double>
<double name="maxWriteMBPerSecRead">4000000</double>
<str name="solr.hdfs.home">${solr.hdfs.home:}</str>
- <bool name="solr.hdfs.blockcache.enabled">${solr.hdfs.blockcache.enabled:true}</bool>
+ <bool name="solr.hdfs.blockcache.enabled">${solr.hdfs.blockcache.enabled:true}</bool>
+ <str name="solr.hdfs.blockcache.global">${solr.hdfs.blockcache.global:false}</str>
</directoryFactory>
<luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java?rev=1573849&r1=1573848&r2=1573849&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java Tue Mar 4 04:16:51 2014
@@ -131,7 +131,7 @@ public class ChaosMonkeyNothingIsSafeTes
int threadCount = 1;
int i = 0;
for (i = 0; i < threadCount; i++) {
- StopableIndexingThread indexThread = new StopableIndexingThread(Integer.toString(i), true);
+ StopableIndexingThread indexThread = new StopableIndexingThread(controlClient, cloudClient, Integer.toString(i), true);
threads.add(indexThread);
indexThread.start();
}
@@ -265,7 +265,7 @@ public class ChaosMonkeyNothingIsSafeTes
public FullThrottleStopableIndexingThread(List<SolrServer> clients,
String id, boolean doDeletes) {
- super(id, doDeletes);
+ super(controlClient, cloudClient, id, doDeletes);
setName("FullThrottleStopableIndexingThread");
setDaemon(true);
this.clients = clients;
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java?rev=1573849&r1=1573848&r2=1573849&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java Tue Mar 4 04:16:51 2014
@@ -108,7 +108,7 @@ public class ChaosMonkeySafeLeaderTest e
List<StopableIndexingThread> threads = new ArrayList<StopableIndexingThread>();
int threadCount = 2;
for (int i = 0; i < threadCount; i++) {
- StopableIndexingThread indexThread = new StopableIndexingThread(Integer.toString(i), true);
+ StopableIndexingThread indexThread = new StopableIndexingThread(controlClient, cloudClient, Integer.toString(i), true);
threads.add(indexThread);
indexThread.start();
}
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java?rev=1573849&r1=1573848&r2=1573849&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java Tue Mar 4 04:16:51 2014
@@ -66,10 +66,10 @@ public class RecoveryZkTest extends Abst
int maxDoc = maxDocList[random().nextInt(maxDocList.length - 1)];
- indexThread = new StopableIndexingThread("1", true, maxDoc);
+ indexThread = new StopableIndexingThread(controlClient, cloudClient, "1", true, maxDoc);
indexThread.start();
- indexThread2 = new StopableIndexingThread("2", true, maxDoc);
+ indexThread2 = new StopableIndexingThread(controlClient, cloudClient, "2", true, maxDoc);
indexThread2.start();
@@ -100,7 +100,7 @@ public class RecoveryZkTest extends Abst
Thread.sleep(1000);
- waitForThingsToLevelOut(45);
+ waitForThingsToLevelOut(90);
Thread.sleep(2000);
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java?rev=1573849&r1=1573848&r2=1573849&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java Tue Mar 4 04:16:51 2014
@@ -64,6 +64,8 @@ public class HdfsTestUtil {
System.setProperty("solr.hdfs.home", "/solr_hdfs_home");
+ System.setProperty("solr.hdfs.blockcache.global", Boolean.toString(LuceneTestCase.random().nextBoolean()));
+
final MiniDFSCluster dfsCluster = new MiniDFSCluster(conf, dataNodes, true, null);
dfsCluster.waitActive();
@@ -92,6 +94,7 @@ public class HdfsTestUtil {
System.clearProperty("test.build.data");
System.clearProperty("test.cache.data");
System.clearProperty("solr.hdfs.home");
+ System.clearProperty("solr.hdfs.blockcache.global");
if (dfsCluster != null) {
timers.remove(dfsCluster);
dfsCluster.shutdown();
Copied: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java (from r1573847, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java?p2=lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java&p1=lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java&r1=1573847&r2=1573849&rev=1573849&view=diff
==============================================================================
(empty)
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/store/blockcache/BlockCacheTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/store/blockcache/BlockCacheTest.java?rev=1573849&r1=1573848&r2=1573849&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/store/blockcache/BlockCacheTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/store/blockcache/BlockCacheTest.java Tue Mar 4 04:16:51 2014
@@ -51,6 +51,7 @@ public class BlockCacheTest extends Luce
int file = 0;
blockCacheKey.setBlock(block);
blockCacheKey.setFile(file);
+ blockCacheKey.setPath("/");
if (blockCache.fetch(blockCacheKey, buffer)) {
hitsInCache.incrementAndGet();
@@ -91,6 +92,7 @@ public class BlockCacheTest extends Luce
BlockCacheKey blockCacheKey = new BlockCacheKey();
blockCacheKey.setBlock(0);
blockCacheKey.setFile(0);
+ blockCacheKey.setPath("/");
byte[] newData = new byte[blockSize*3];
byte[] testData = testData(random, blockSize, newData);
Modified: lucene/dev/branches/branch_4x/solr/example/solr/collection1/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/example/solr/collection1/conf/solrconfig.xml?rev=1573849&r1=1573848&r2=1573849&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/example/solr/collection1/conf/solrconfig.xml (original)
+++ lucene/dev/branches/branch_4x/solr/example/solr/collection1/conf/solrconfig.xml Tue Mar 4 04:16:51 2014
@@ -129,6 +129,9 @@
<str name="solr.hdfs.confdir">${solr.hdfs.confdir:}</str>
<!-- Enable/Disable the hdfs cache. -->
<str name="solr.hdfs.blockcache.enabled">${solr.hdfs.blockcache.enabled:true}</str>
+ <!-- Enable/Disable using one global cache for all SolrCores.
+ The settings used will be from the first HdfsDirectoryFactory created. -->
+ <str name="solr.hdfs.blockcache.global">${solr.hdfs.blockcache.global:true}</str>
</directoryFactory>
Modified: lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java?rev=1573849&r1=1573848&r2=1573849&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java (original)
+++ lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java Tue Mar 4 04:16:51 2014
@@ -1428,122 +1428,13 @@ public abstract class AbstractFullDistri
return rsp;
}
- abstract class StopableThread extends Thread {
+ static abstract class StopableThread extends Thread {
public StopableThread(String name) {
super(name);
}
public abstract void safeStop();
}
- class StopableIndexingThread extends StopableThread {
- private volatile boolean stop = false;
- protected final String id;
- protected final List<String> deletes = new ArrayList<String>();
- protected Set<String> addFails = new HashSet<String>();
- protected Set<String> deleteFails = new HashSet<String>();
- protected boolean doDeletes;
- private int numCycles;
-
- public StopableIndexingThread(String id, boolean doDeletes) {
- this(id, doDeletes, -1);
- }
-
- public StopableIndexingThread(String id, boolean doDeletes, int numCycles) {
- super("StopableIndexingThread");
- this.id = id;
- this.doDeletes = doDeletes;
- this.numCycles = numCycles;
- setDaemon(true);
- }
-
- @Override
- public void run() {
- int i = 0;
- int numDone = 0;
- int numDeletes = 0;
- int numAdds = 0;
-
- while (true && !stop) {
- if (numCycles != -1) {
- if (numDone > numCycles) {
- break;
- }
- }
- ++numDone;
- String id = this.id + "-" + i;
- ++i;
- boolean addFailed = false;
-
- if (doDeletes && random().nextBoolean() && deletes.size() > 0) {
- String delete = deletes.remove(0);
- try {
- numDeletes++;
- UpdateRequest req = new UpdateRequest();
- req.deleteById(delete);
- req.setParam("CONTROL", "TRUE");
- req.process(controlClient);
-
- cloudClient.deleteById(delete);
- } catch (Exception e) {
- System.err.println("REQUEST FAILED:");
- e.printStackTrace();
- if (e instanceof SolrServerException) {
- System.err.println("ROOT CAUSE:");
- ((SolrServerException) e).getRootCause().printStackTrace();
- }
- deleteFails.add(id);
- }
- }
-
- try {
- numAdds++;
- indexr("id", id, i1, 50, t1,
- "to come to the aid of their country.");
- } catch (Exception e) {
- addFailed = true;
- System.err.println("REQUEST FAILED:");
- e.printStackTrace();
- if (e instanceof SolrServerException) {
- System.err.println("ROOT CAUSE:");
- ((SolrServerException) e).getRootCause().printStackTrace();
- }
- addFails.add(id);
- }
-
- if (!addFailed && doDeletes && random().nextBoolean()) {
- deletes.add(id);
- }
-
- try {
- Thread.currentThread().sleep(random().nextInt(100));
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- }
-
- System.err.println("added docs:" + numAdds + " with " + (addFails.size() + deleteFails.size()) + " fails"
- + " deletes:" + numDeletes);
- }
-
- @Override
- public void safeStop() {
- stop = true;
- }
-
- public Set<String> getAddFails() {
- return addFails;
- }
-
- public Set<String> getDeleteFails() {
- return deleteFails;
- }
-
- public int getFailCount() {
- return addFails.size() + deleteFails.size();
- }
-
- };
-
class StopableSearchThread extends StopableThread {
private volatile boolean stop = false;
protected final AtomicInteger queryFails = new AtomicInteger();
Copied: lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/StopableIndexingThread.java (from r1573847, lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/cloud/StopableIndexingThread.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/StopableIndexingThread.java?p2=lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/StopableIndexingThread.java&p1=lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/cloud/StopableIndexingThread.java&r1=1573847&r2=1573849&rev=1573849&view=diff
==============================================================================
(empty)