You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2017/05/09 07:07:18 UTC
[1/2] lucene-solr:branch_6x: SOLR-10524: Explore in-memory
partitioning for processing Overseer queue messages
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x 219442df9 -> 5c626dc9e
SOLR-10524: Explore in-memory partitioning for processing Overseer queue messages
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9cab9c0c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9cab9c0c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9cab9c0c
Branch: refs/heads/branch_6x
Commit: 9cab9c0cf2777a21a81386be2262e84da2bca751
Parents: 219442d
Author: Cao Manh Dat <da...@apache.org>
Authored: Mon May 8 16:21:19 2017 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Tue May 9 14:06:29 2017 +0700
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
.../core/src/java/org/apache/solr/cloud/Overseer.java | 3 ++-
.../org/apache/solr/cloud/overseer/ZkStateWriter.java | 14 ++------------
3 files changed, 6 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9cab9c0c/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index ec08c7f..e89ee4b 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -154,6 +154,8 @@ Optimizations
* SOLR-10548: JSON Facet API now uses hyper-log-log++ for determining the number of buckets
when merging requests from a multi-shard distributed request. (yonik)
+* SOLR-10524: Explore in-memory partitioning for processing Overseer queue messages (Cao Manh Dat, Noble Paul, shalin, Scott Blum)
+
Bug Fixes
----------------------
* SOLR-10281: ADMIN_PATHS is duplicated in two places and inconsistent. This can cause automatic
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9cab9c0c/solr/core/src/java/org/apache/solr/cloud/Overseer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
index e27251e..1f28932 100644
--- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java
+++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
@@ -64,7 +64,8 @@ import static org.apache.solr.common.params.CommonParams.ID;
public class Overseer implements Closeable {
public static final String QUEUE_OPERATION = "operation";
- public static final int STATE_UPDATE_DELAY = 1500; // delay between cloud state updates
+ public static final int STATE_UPDATE_DELAY = 2000; // delay between cloud state updates
+ public static final int STATE_UPDATE_BATCH_SIZE = 10000;
public static final int NUM_RESPONSES_TO_STORE = 10000;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9cab9c0c/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
index 23fb56c..48d2ecb 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
@@ -166,17 +166,7 @@ public class ZkStateWriter {
* @return true if a flush is required, false otherwise
*/
protected boolean maybeFlushBefore(ZkWriteCommand cmd) {
- if (lastUpdatedTime == 0) {
- // first update, make sure we go through
- return false;
- }
- if (cmd.collection == null) {
- return false;
- }
- if (cmd.collection.getStateFormat() != lastStateFormat) {
- return true;
- }
- return cmd.collection.getStateFormat() > 1 && !cmd.name.equals(lastCollectionName);
+ return cmd.collection.getStateFormat() != lastStateFormat;
}
/**
@@ -190,7 +180,7 @@ public class ZkStateWriter {
return false;
lastCollectionName = cmd.name;
lastStateFormat = cmd.collection.getStateFormat();
- return System.nanoTime() - lastUpdatedTime > MAX_FLUSH_INTERVAL;
+ return System.nanoTime() - lastUpdatedTime > MAX_FLUSH_INTERVAL || updates.size() > Overseer.STATE_UPDATE_BATCH_SIZE;
}
public boolean hasPendingUpdates() {
[2/2] lucene-solr:branch_6x: SOLR-10524: Build fix for NPE
Posted by da...@apache.org.
SOLR-10524: Build fix for NPE
Introduced by ZkStateWriter batching optimizations.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5c626dc9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5c626dc9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5c626dc9
Branch: refs/heads/branch_6x
Commit: 5c626dc9e0a488d43e9a2f41947fd2ec3b0b046f
Parents: 9cab9c0
Author: Scott Blum <dr...@apache.org>
Authored: Mon May 8 20:32:38 2017 -0400
Committer: Cao Manh Dat <da...@apache.org>
Committed: Tue May 9 14:06:37 2017 +0700
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +-
.../org/apache/solr/cloud/overseer/ZkStateWriter.java | 3 +++
.../org/apache/solr/cloud/overseer/ZkStateWriterTest.java | 10 +++++-----
3 files changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5c626dc9/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index e89ee4b..f079dbf 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -154,7 +154,7 @@ Optimizations
* SOLR-10548: JSON Facet API now uses hyper-log-log++ for determining the number of buckets
when merging requests from a multi-shard distributed request. (yonik)
-* SOLR-10524: Explore in-memory partitioning for processing Overseer queue messages (Cao Manh Dat, Noble Paul, shalin, Scott Blum)
+* SOLR-10524: Better ZkStateWriter batching (Cao Manh Dat, Noble Paul, shalin, Scott Blum)
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5c626dc9/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
index 48d2ecb..da0f57c 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
@@ -166,6 +166,9 @@ public class ZkStateWriter {
* @return true if a flush is required, false otherwise
*/
protected boolean maybeFlushBefore(ZkWriteCommand cmd) {
+ if (cmd.collection == null || lastStateFormat <= 0) {
+ return false;
+ }
return cmd.collection.getStateFormat() != lastStateFormat;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5c626dc9/solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateWriterTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateWriterTest.java b/solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateWriterTest.java
index 85dbf4a..f96b5e2 100644
--- a/solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateWriterTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateWriterTest.java
@@ -77,7 +77,7 @@ public class ZkStateWriterTest extends SolrTestCaseJ4 {
ZkWriteCommand c2 = new ZkWriteCommand("c2",
new DocCollection("c2", new HashMap<>(), new HashMap<>(), DocRouter.DEFAULT, 0, ZkStateReader.COLLECTIONS_ZKNODE + "/c2"));
- assertTrue("Different (new) collection create cannot be batched together with another create", writer.maybeFlushBefore(c2));
+ assertFalse("Different (new) collection create can be batched together with another create", writer.maybeFlushBefore(c2));
// simulate three state changes on same collection, all should be batched together before
assertFalse(writer.maybeFlushBefore(c1));
@@ -88,13 +88,13 @@ public class ZkStateWriterTest extends SolrTestCaseJ4 {
assertFalse(writer.maybeFlushAfter(c1));
assertFalse(writer.maybeFlushAfter(c1));
- // simulate three state changes on two different collections with stateFormat=2, none should be batched
+ // simulate three state changes on two different collections with stateFormat=2, all should be batched
assertFalse(writer.maybeFlushBefore(c1));
// flushAfter has to be called as it updates the internal batching related info
assertFalse(writer.maybeFlushAfter(c1));
- assertTrue(writer.maybeFlushBefore(c2));
+ assertFalse(writer.maybeFlushBefore(c2));
assertFalse(writer.maybeFlushAfter(c2));
- assertTrue(writer.maybeFlushBefore(c1));
+ assertFalse(writer.maybeFlushBefore(c1));
assertFalse(writer.maybeFlushAfter(c1));
// create a collection in stateFormat = 1 i.e. inside the main cluster state
@@ -168,7 +168,7 @@ public class ZkStateWriterTest extends SolrTestCaseJ4 {
}
}
- public void testSingleExternalCollection() throws Exception{
+ public void testSingleExternalCollection() throws Exception {
String zkDir = createTempDir("testSingleExternalCollection").toFile().getAbsolutePath();
ZkTestServer server = new ZkTestServer(zkDir);