You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/01/13 22:54:10 UTC
svn commit: r1557868 - in /lucene/dev/branches/lucene5376/lucene/server/src:
java/org/apache/lucene/server/ java/org/apache/lucene/server/handlers/
java/org/apache/lucene/server/params/ test/org/apache/lucene/server/
Author: mikemccand
Date: Mon Jan 13 21:54:10 2014
New Revision: 1557868
URL: http://svn.apache.org/r1557868
Log:
LUCENE-5376: add merge max MB/sec rate limiting
Modified:
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/IndexState.java
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/SettingsHandler.java
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/Request.java
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestIndexing.java
Modified: lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/IndexState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/IndexState.java?rev=1557868&r1=1557867&r2=1557868&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/IndexState.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/IndexState.java Mon Jan 13 21:54:10 2014
@@ -95,10 +95,12 @@ import org.apache.lucene.server.params.S
import org.apache.lucene.server.params.StructType;
import org.apache.lucene.server.params.Type;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.NRTCachingDirectory;
import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.store.RateLimitedDirectoryWrapper;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.PrintStreamInfoStream;
import org.apache.lucene.util.Version;
@@ -1068,6 +1070,13 @@ public class IndexState implements Close
indexDir = origIndexDir;
}
+ // Install rate limiting for merges:
+ if (settingsSaveState.containsKey("mergeMaxMBPerSec")) {
+ RateLimitedDirectoryWrapper rateLimitDir = new RateLimitedDirectoryWrapper(indexDir);
+ rateLimitDir.setMaxWriteMBPerSec(getDoubleSetting("mergeMaxMBPerSec"), IOContext.Context.MERGE);
+ indexDir = rateLimitDir;
+ }
+
// Rather than rely on IndexWriter/TaxonomyWriter to
// figure out if an index is new or not by passing
// CREATE_OR_APPEND (which can be dangerous), we
Modified: lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/SettingsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/SettingsHandler.java?rev=1557868&r1=1557867&r2=1557868&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/SettingsHandler.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/SettingsHandler.java Mon Jan 13 21:54:10 2014
@@ -46,6 +46,7 @@ public class SettingsHandler extends Han
new StructType(
new Param("indexName", "Index name", new StringType()),
RegisterFieldHandler.MATCH_VERSION_PARAM,
+ new Param("mergeMaxMBPerSec", "Rate limit merges to at most this many MB/sec", new FloatType()),
new Param("nrtCachingDirectory.maxMergeSizeMB", "Largest merged segment size to cache in RAMDirectory", new FloatType(), 5.0),
new Param("nrtCachingDirectory.maxSizeMB", "Largest overall size for all files cached in NRTCachingDirectory; set to 0 to disable NRTCachingDirectory", new FloatType(), 60.0),
new Param("concurrentMergeScheduler.maxThreadCount", "How many merge threads to allow at once", new IntType(), 1),
Modified: lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/Request.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/Request.java?rev=1557868&r1=1557867&r2=1557868&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/Request.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/Request.java Mon Jan 13 21:54:10 2014
@@ -28,6 +28,8 @@ import net.minidev.json.JSONObject;
// nocommit instead of removing as we getXXX, we could do a
// Set<String> seen?
+// - would close the loophole of foobar={} not being detected
+// - would allow us to lookup same param more than once
/** Pairs up the actual parameters with its type. For
* complex requests, e.g. {@code search}, this is used
Modified: lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestIndexing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestIndexing.java?rev=1557868&r1=1557867&r2=1557868&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestIndexing.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestIndexing.java Mon Jan 13 21:54:10 2014
@@ -234,6 +234,8 @@ public class TestIndexing extends Server
curIndexName = "boost";
send("createIndex");
send("settings", "{directory: RAMDirectory, matchVersion: LUCENE_40}");
+ // Just to test merge rate limiting:
+ send("settings", "{mergeMaxMBPerSec: 10.0}");
// Just to test index.ramBufferSizeMB:
send("liveSettings", "{index.ramBufferSizeMB: 20.0}");
send("registerFields", "{fields: {id: {type: atom, store: true}, body: {type: text, analyzer: StandardAnalyzer}}}");
@@ -343,7 +345,6 @@ public class TestIndexing extends Server
System.out.println("\nTEST: settings3");
}
result = send("settings");
- System.out.println("GOT: " + result);
assertEquals(dirImpl, getString(result, "directory"));
}
send("deleteIndex");