You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2018/04/02 07:37:48 UTC
[01/34] lucene-solr:jira/solr-12095: LUCENE-8227: Under pressure,
had to @Ignore tests that caused intermittent failures. This means no
further work on Geo3D until these tests can be re-enabled.
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-12095 059f495e3 -> d83fcbd1f
LUCENE-8227: Under pressure, had to @Ignore tests that caused intermittent failures. This means no further work on Geo3D until these tests can be re-enabled.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5b429df5
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5b429df5
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5b429df5
Branch: refs/heads/jira/solr-12095
Commit: 5b429df56f36031c8423803d5157029c59bcbb3d
Parents: b151b2c
Author: Karl Wright <Da...@gmail.com>
Authored: Thu Mar 29 10:37:28 2018 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Thu Mar 29 10:37:28 2018 -0400
----------------------------------------------------------------------
.../apache/lucene/spatial3d/TestGeo3DPoint.java | 10 ++++
.../lucene/spatial3d/geom/GeoPolygonTest.java | 60 ++++++++++++++++++++
2 files changed, 70 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5b429df5/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java
index 861e26e..38b4114 100644
--- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java
+++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java
@@ -81,6 +81,8 @@ import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.TestUtil;
+import org.junit.Ignore;
+
import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
public class TestGeo3DPoint extends LuceneTestCase {
@@ -188,6 +190,8 @@ public class TestGeo3DPoint extends LuceneTestCase {
}
/** Tests consistency of GeoArea.getRelationship vs GeoShape.isWithin */
+ //@AwaitsFix("https://issues.apache.org/jira/browse/LUCENE-8227")
+ @Ignore
public void testGeo3DRelations() throws Exception {
int numDocs = atLeast(1000);
@@ -467,16 +471,22 @@ public class TestGeo3DPoint extends LuceneTestCase {
}
}
+ //@AwaitsFix("https://issues.apache.org/jira/browse/LUCENE-8227")
+ @Ignore
public void testRandomTiny() throws Exception {
// Make sure single-leaf-node case is OK:
doTestRandom(10);
}
+ //@AwaitsFix("https://issues.apache.org/jira/browse/LUCENE-8227")
+ @Ignore
public void testRandomMedium() throws Exception {
doTestRandom(10000);
}
@Nightly
+ //@AwaitsFix("https://issues.apache.org/jira/browse/LUCENE-8227")
+ @Ignore
public void testRandomBig() throws Exception {
doTestRandom(50000);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5b429df5/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
index 2625ba7..ebfb0f4 100755
--- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
+++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
@@ -22,6 +22,7 @@ import java.util.BitSet;
import java.util.Collections;
import org.junit.Test;
+import org.junit.Ignore;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -1204,4 +1205,63 @@ shape:
Collections.reverse(points);
polygon = GeoPolygonFactory.makeGeoPolygon(PlanetModel.SPHERE, points);
}
+
+ /*
+ [lat=-0.63542308910253, lon=0.9853722928232957([X=0.4446759777403525, Y=0.6707549854468698, Z=-0.5934780737681111])],
+ [lat=0.0, lon=0.0([X=1.0011188539924791, Y=0.0, Z=0.0])],
+ [lat=0.45435018176633574, lon=3.141592653589793([X=-0.8989684544372841, Y=1.1009188402610632E-16, Z=0.4390846549572752])],
+ [lat=-0.375870856827283, lon=2.9129132647718414([X=-0.9065744420970767, Y=0.21100590938346708, Z=-0.36732668582405886])],
+ [lat=-1.2205765069413237, lon=3.141592653589793([X=-0.3424714964202101, Y=4.194066218902145E-17, Z=-0.9375649457139603])]}}
+
+ [junit4] 1> unquantized=[lat=-3.1780051348770987E-74, lon=-3.032608859187692([X=-0.9951793580358298, Y=-0.1088898762907205, Z=-3.181560858610375E-74])]
+ [junit4] 1> quantized=[X=-0.9951793580415914, Y=-0.10888987641797832, Z=-2.3309121299774915E-10]
+ */
+ @Test
+ @Ignore
+ public void testLUCENE8227() throws Exception {
+ List<GeoPoint> points = new ArrayList<>();
+ points.add(new GeoPoint(PlanetModel.WGS84, -0.63542308910253, 0.9853722928232957));
+ points.add(new GeoPoint(PlanetModel.WGS84, 0.0, 0.0));
+ points.add(new GeoPoint(PlanetModel.WGS84, 0.45435018176633574, 3.141592653589793));
+ points.add(new GeoPoint(PlanetModel.WGS84, -0.375870856827283, 2.9129132647718414));
+ points.add(new GeoPoint(PlanetModel.WGS84, -1.2205765069413237, 3.141592653589793));
+ GeoPolygonFactory.PolygonDescription pd = new GeoPolygonFactory.PolygonDescription(points);
+
+ for (int i = 0; i < points.size(); i++) {
+ System.out.println("Point "+i+": "+points.get(i));
+ }
+
+ final GeoPoint unquantized = new GeoPoint(PlanetModel.WGS84, -3.1780051348770987E-74, -3.032608859187692);
+ final GeoPoint quantized = new GeoPoint(-0.9951793580415914, -0.10888987641797832, -2.3309121299774915E-10);
+
+ final GeoPoint negativeX = new GeoPoint(PlanetModel.WGS84, 0.0, Math.PI);
+ final GeoPoint negativeY = new GeoPoint(PlanetModel.WGS84, 0.0, -Math.PI * 0.5);
+
+ // Construct a standard polygon first to see what that does
+ GeoPolygon standard = GeoPolygonFactory.makeGeoPolygon(PlanetModel.WGS84, pd);
+
+ System.out.println("Standard polygon: "+standard);
+
+ // This shows y < 0 hemisphere is all in-set
+ //assertTrue(standard.isWithin(negativeY));
+ // This should be in-set too, but isn't!!
+ assertTrue(standard.isWithin(negativeX));
+
+/*
+ final XYZBounds standardBounds = new XYZBounds();
+ standard.getBounds(standardBounds);
+ final XYZSolid standardSolid = XYZSolidFactory.makeXYZSolid(PlanetModel.WGS84, standardBounds);
+
+ System.out.println("Standard bounds: "+standardBounds);
+
+ assertFalse(standardSolid.isWithin(quantized));
+ assertFalse(standardSolid.isWithin(unquantized));
+*/
+ // Now, both points should also not be in the poly
+ assertFalse(standard.isWithin(unquantized));
+ assertFalse(standard.isWithin(quantized));
+
+
+ }
+
}
[03/34] lucene-solr:jira/solr-12095: SOLR-12136: Docs: Improve hl.fl,
hl.q, hl.qparser
Posted by sh...@apache.org.
SOLR-12136: Docs: Improve hl.fl, hl.q, hl.qparser
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/701af06f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/701af06f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/701af06f
Branch: refs/heads/jira/solr-12095
Commit: 701af06f627be98ddc8db083dc4dd51dbfe4936a
Parents: 358e595
Author: David Smiley <ds...@apache.org>
Authored: Thu Mar 29 12:22:54 2018 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Thu Mar 29 12:22:54 2018 -0400
----------------------------------------------------------------------
solr/solr-ref-guide/src/highlighting.adoc | 23 ++++++++++++++---------
1 file changed, 14 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/701af06f/solr/solr-ref-guide/src/highlighting.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/highlighting.adoc b/solr/solr-ref-guide/src/highlighting.adoc
index 7d96673..b6c9bf2 100644
--- a/solr/solr-ref-guide/src/highlighting.adoc
+++ b/solr/solr-ref-guide/src/highlighting.adoc
@@ -36,23 +36,28 @@ The highlighting implementation to use. Acceptable values are: `unified`, `origi
See the <<Choosing a Highlighter>> section below for more details on the differences between the available highlighters.
`hl.fl`::
-Specifies a list of fields to highlight. Accepts a comma- or space-delimited list of fields for which Solr should generate highlighted snippets.
+Specifies a list of fields to highlight, either comma- or space-delimited.
+A wildcard of `\*` (asterisk) can be used to match field globs, such as `text_*` or even `\*` to highlight on all fields where highlighting is possible.
+When using `*`, consider adding `hl.requireFieldMatch=true`.
+
-A wildcard of `\*` (asterisk) can be used to match field globs, such as `text_*` or even `\*` to highlight on all fields where highlighting is possible. When using `*`, consider adding `hl.requireFieldMatch=true`.
+Note that the field(s) listed here ought to have compatible text-analysis (defined in the schema) with field(s) referenced in the query to be highlighted.
+It may be necessary to modify `hl.q` and `hl.qparser` and/or modify the text analysis.
+The following example uses the <<local-parameters-in-queries.adoc,local-params>> syntax and <<the-extended-dismax-query-parser.adoc,the edismax parser>> to highlight fields in `hl.fl`:
+`&hl.fl=field1 field2&hl.q={!edismax qf=$hl.fl v=$q}&hl.qparser=lucene&hl.requireFieldMatch=true` (along with other applicable parameters, of course).
+
-When not defined, the defaults defined for the `df` query parameter will be used.
+The default is the value of the `df` parameter which in turn has no default.
`hl.q`::
-A query to use for highlighting. This parameter allows you to highlight different terms than those being used to retrieve documents.
+A query to use for highlighting.
+This parameter allows you to highlight different terms or fields than those being used to search for documents.
+When setting this, you might also need to set `hl.qparser`.
+
-When not defined, the query defined with the `q` parameter will the used.
-+
-When `hl.qparser` is not defined, the query parser defined with the `defType` query parameter will be used and terms will be analyzed using those rules. This behavior can be overridden by specifying a field, for example: `hl.q=field:term`.
+The default is the value of the `q` parameter (already parsed).
`hl.qparser`::
-The query parser to use for the `hl.q` query.
+The query parser to use for the `hl.q` query. It only applies when `hl.q` is set.
+
-When not defined, the query parser defined with the `defType` query parameter will be used.
+The default is the value of the `defType` parameter which in turn defaults to `lucene`.
`hl.requireFieldMatch`::
By default, `false`, all query terms will be highlighted for each field to be highlighted (`hl.fl`) no matter what fields the parsed query refer to. If set to `true`, only query terms aligning with the field being highlighted will in turn be highlighted.
[30/34] lucene-solr:jira/solr-12095: Add a description to all public
targets
Posted by sh...@apache.org.
Add a description to all public targets
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d98138ac
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d98138ac
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d98138ac
Branch: refs/heads/jira/solr-12095
Commit: d98138ac039788969bb816f9c1e0180a6a35135e
Parents: b4a83ff
Author: Uwe Schindler <us...@apache.org>
Authored: Sun Apr 1 14:40:33 2018 +0200
Committer: Uwe Schindler <us...@apache.org>
Committed: Sun Apr 1 14:40:33 2018 +0200
----------------------------------------------------------------------
build.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d98138ac/build.xml
----------------------------------------------------------------------
diff --git a/build.xml b/build.xml
index 49b8eec..9c8deff 100755
--- a/build.xml
+++ b/build.xml
@@ -124,7 +124,7 @@
</subant>
</target>
- <target name="validate-source-patterns" unless="disable.source-patterns" depends="resolve-groovy,rat-sources-typedef">
+ <target name="validate-source-patterns" description="Validate source code (invalid code patterns,...)" unless="disable.source-patterns" depends="resolve-groovy,rat-sources-typedef">
<groovy taskname="source-patterns" classpathref="rat.classpath" src="${common.dir}/tools/src/groovy/check-source-patterns.groovy"/>
</target>
[21/34] lucene-solr:jira/solr-12095: SOLR-12169: Update jira number
in BadApple annotation
Posted by sh...@apache.org.
SOLR-12169: Update jira number in BadApple annotation
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e3c67b19
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e3c67b19
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e3c67b19
Branch: refs/heads/jira/solr-12095
Commit: e3c67b1980da35df78b95b39049a3a7258ebf335
Parents: 35bfe89
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Fri Mar 30 19:42:13 2018 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Fri Mar 30 19:42:13 2018 +0530
----------------------------------------------------------------------
.../org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e3c67b19/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
index 943e8fc..720cc4f 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
@@ -397,7 +397,7 @@ public class ComputePlanActionTest extends SolrCloudTestCase {
}
@Test
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12169")
public void testSelectedCollections() throws Exception {
AssertingTriggerAction.expectedNode = null;
[25/34] lucene-solr:jira/solr-12095: LUCENE-8232: Separate out
PendingDeletes from ReadersAndUpdates
Posted by sh...@apache.org.
LUCENE-8232: Separate out PendingDeletes from ReadersAndUpdates
Today ReadersAndUpdates is tightly coupled with IW and all the
handling of pending deletes. This change decouples IW and pending
deletes from ReadersAndUpdates and makes PendingDeletes unittestable.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/acb3c379
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/acb3c379
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/acb3c379
Branch: refs/heads/jira/solr-12095
Commit: acb3c379427193036f3d56503529400736ac5dff
Parents: ab09294
Author: Simon Willnauer <si...@apache.org>
Authored: Thu Mar 29 17:21:59 2018 +0200
Committer: Simon Willnauer <si...@apache.org>
Committed: Sat Mar 31 10:25:02 2018 +0200
----------------------------------------------------------------------
.../lucene/index/BufferedUpdatesStream.java | 4 +-
.../org/apache/lucene/index/IndexWriter.java | 32 +--
.../org/apache/lucene/index/PendingDeletes.java | 193 +++++++++++++++++++
.../apache/lucene/index/ReadersAndUpdates.java | 185 +++++-------------
.../src/java/org/apache/lucene/util/Bits.java | 2 +-
.../apache/lucene/index/TestPendingDeletes.java | 142 ++++++++++++++
6 files changed, 404 insertions(+), 154 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/acb3c379/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java b/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java
index e887e4d..63001d4 100644
--- a/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java
+++ b/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java
@@ -333,8 +333,8 @@ class BufferedUpdatesStream implements Accountable {
if (success) {
totDelCount += segState.rld.getPendingDeleteCount() - segState.startDelCount;
int fullDelCount = segState.rld.info.getDelCount() + segState.rld.getPendingDeleteCount();
- assert fullDelCount <= segState.rld.info.info.maxDoc();
- if (fullDelCount == segState.rld.info.info.maxDoc()) {
+ assert fullDelCount <= segState.rld.info.info.maxDoc() : fullDelCount + " > " + segState.rld.info.info.maxDoc();
+ if (segState.rld.isFullyDeleted()) {
if (allDeleted == null) {
allDeleted = new ArrayList<>();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/acb3c379/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index 40a53e0..3791e19 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -604,7 +604,10 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
if (!poolReaders && rld.refCount() == 1 && readerMap.containsKey(rld.info)) {
// This is the last ref to this RLD, and we're not
// pooling, so remove it:
- if (rld.writeLiveDocs(directory)) {
+ boolean changed = rld.writeLiveDocs(directory);
+ changed |= rld.writeFieldUpdates(directory, globalFieldNumberMap, bufferedUpdatesStream.getCompletedDelGen(), infoStream);
+
+ if (changed) {
// Make sure we only write del docs for a live segment:
assert assertInfoLive == false || assertInfoIsLive(rld.info);
// Must checkpoint because we just
@@ -616,9 +619,6 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
// did was move the state to disk:
checkpointNoSIS();
}
-
- rld.writeFieldUpdates(directory, bufferedUpdatesStream.getCompletedDelGen(), infoStream);
-
if (rld.getNumDVUpdates() == 0) {
rld.dropReaders();
readerMap.remove(rld.info);
@@ -635,13 +635,15 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
}
void writeAllDocValuesUpdates() throws IOException {
+ assert Thread.holdsLock(IndexWriter.this);
Collection<ReadersAndUpdates> copy;
synchronized (this) {
+ // this needs to be protected by the reader pool lock otherwise we hit ConcurrentModificationException
copy = new HashSet<>(readerMap.values());
}
boolean any = false;
for (ReadersAndUpdates rld : copy) {
- any |= rld.writeFieldUpdates(directory, bufferedUpdatesStream.getCompletedDelGen(), infoStream);
+ any |= rld.writeFieldUpdates(directory, globalFieldNumberMap, bufferedUpdatesStream.getCompletedDelGen(), infoStream);
}
if (any) {
checkpoint();
@@ -649,11 +651,12 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
}
void writeDocValuesUpdatesForMerge(List<SegmentCommitInfo> infos) throws IOException {
+ assert Thread.holdsLock(IndexWriter.this);
boolean any = false;
for (SegmentCommitInfo info : infos) {
ReadersAndUpdates rld = get(info, false);
if (rld != null) {
- any |= rld.writeFieldUpdates(directory, bufferedUpdatesStream.getCompletedDelGen(), infoStream);
+ any |= rld.writeFieldUpdates(directory, globalFieldNumberMap, bufferedUpdatesStream.getCompletedDelGen(), infoStream);
rld.setIsMerging();
}
}
@@ -706,7 +709,9 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
// Only acquire IW lock on each write, since this is a time consuming operation. This way
// other threads get a chance to run in between our writes.
synchronized (IndexWriter.this) {
- rld.writeFieldUpdates(directory, bufferedUpdatesStream.getCompletedDelGen(), infoStream);
+ if (rld.writeFieldUpdates(directory, globalFieldNumberMap, bufferedUpdatesStream.getCompletedDelGen(), infoStream)) {
+ checkpointNoSIS();
+ }
}
long bytesUsedAfter = rld.ramBytesUsed.get();
ramBytesUsed -= bytesUsedBefore - bytesUsedAfter;
@@ -789,8 +794,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
if (rld != null) {
assert rld.info == info;
boolean changed = rld.writeLiveDocs(directory);
-
- changed |= rld.writeFieldUpdates(directory, bufferedUpdatesStream.getCompletedDelGen(), infoStream);
+ changed |= rld.writeFieldUpdates(directory, globalFieldNumberMap, bufferedUpdatesStream.getCompletedDelGen(), infoStream);
if (changed) {
// Make sure we only write del docs for a live segment:
@@ -838,7 +842,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
if (create == false) {
return null;
}
- rld = new ReadersAndUpdates(IndexWriter.this, info);
+ rld = new ReadersAndUpdates(segmentInfos.getIndexCreatedVersionMajor(), info, null, new PendingDeletes(null, info));
// Steal initial reference:
readerMap.put(info, rld);
} else {
@@ -1147,7 +1151,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
LeafReaderContext leaf = leaves.get(i);
SegmentReader segReader = (SegmentReader) leaf.reader();
SegmentReader newReader = new SegmentReader(segmentInfos.info(i), segReader, segReader.getLiveDocs(), segReader.numDocs());
- readerPool.readerMap.put(newReader.getSegmentInfo(), new ReadersAndUpdates(this, newReader));
+ readerPool.readerMap.put(newReader.getSegmentInfo(), new ReadersAndUpdates(segmentInfos.getIndexCreatedVersionMajor(), newReader, new PendingDeletes(newReader, newReader.getSegmentInfo())));
}
// We always assume we are carrying over incoming changes when opening from reader:
@@ -1637,8 +1641,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
if (rld != null) {
synchronized(bufferedUpdatesStream) {
if (rld.delete(docID)) {
- final int fullDelCount = rld.info.getDelCount() + rld.getPendingDeleteCount();
- if (fullDelCount == rld.info.info.maxDoc()) {
+ if (rld.isFullyDeleted()) {
dropDeletedSegment(rld.info);
checkpoint();
}
@@ -4000,8 +4003,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
final boolean allDeleted = merge.segments.size() == 0 ||
merge.info.info.maxDoc() == 0 ||
- (mergedUpdates != null &&
- mergedUpdates.getPendingDeleteCount() == merge.info.info.maxDoc());
+ (mergedUpdates != null && mergedUpdates.isFullyDeleted());
if (infoStream.isEnabled("IW")) {
if (allDeleted) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/acb3c379/lucene/core/src/java/org/apache/lucene/index/PendingDeletes.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/PendingDeletes.java b/lucene/core/src/java/org/apache/lucene/index/PendingDeletes.java
new file mode 100644
index 0000000..74043f3
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/index/PendingDeletes.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.index;
+
+import java.io.IOException;
+
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.LiveDocsFormat;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.TrackingDirectoryWrapper;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.MutableBits;
+
+/**
+ * This class handles accounting and applying pending deletes for live segment readers
+ */
+final class PendingDeletes {
+ private final SegmentCommitInfo info;
+ // True if the current liveDocs is referenced by an
+ // external NRT reader:
+ private boolean liveDocsShared;
+ // Holds the current shared (readable and writable)
+ // liveDocs. This is null when there are no deleted
+ // docs, and it's copy-on-write (cloned whenever we need
+ // to change it but it's been shared to an external NRT
+ // reader).
+ private Bits liveDocs;
+ private int pendingDeleteCount;
+
+ PendingDeletes(SegmentReader reader, SegmentCommitInfo info) {
+ this.info = info;
+ liveDocsShared = true;
+ liveDocs = reader != null ? reader.getLiveDocs() : null;
+ if (reader != null) {
+ pendingDeleteCount = reader.numDeletedDocs() - info.getDelCount();
+ } else {
+ pendingDeleteCount = 0;
+ }
+ }
+
+
+ /**
+ * Marks a document as deleted in this segment and return true if a document got actually deleted or
+ * if the document was already deleted.
+ */
+ boolean delete(int docID) throws IOException {
+ assert info.info.maxDoc() > 0;
+ if (liveDocsShared) {
+ // Copy on write: this means we've cloned a
+ // SegmentReader sharing the current liveDocs
+ // instance; must now make a private clone so we can
+ // change it:
+ LiveDocsFormat liveDocsFormat = info.info.getCodec().liveDocsFormat();
+ if (liveDocs == null) {
+ liveDocs = liveDocsFormat.newLiveDocs(info.info.maxDoc());
+ } else {
+ liveDocs = liveDocsFormat.newLiveDocs(liveDocs);
+ }
+ liveDocsShared = false;
+ }
+
+ assert liveDocs != null;
+ assert docID >= 0 && docID < liveDocs.length() : "out of bounds: docid=" + docID + " liveDocsLength=" + liveDocs.length() + " seg=" + info.info.name + " maxDoc=" + info.info.maxDoc();
+ assert !liveDocsShared;
+ final boolean didDelete = liveDocs.get(docID);
+ if (didDelete) {
+ ((MutableBits) liveDocs).clear(docID);
+ pendingDeleteCount++;
+ }
+ return didDelete;
+ }
+
+ /**
+ * Should be called if the live docs returned from {@link #getLiveDocs()} are shared outside of the
+ * {@link ReadersAndUpdates}
+ */
+ void liveDocsShared() {
+ liveDocsShared = true;
+ }
+
+ /**
+ * Returns the current live docs or null if all docs are live. The returned instance might be mutable or is mutated behind the scenes.
+ * If the returned live docs are shared outside of the ReadersAndUpdates {@link #liveDocsShared()} should be called
+ * first.
+ */
+ Bits getLiveDocs() {
+ return liveDocs;
+ }
+
+ /**
+ * Returns the number of pending deletes that are not written to disk.
+ */
+ int numPendingDeletes() {
+ return pendingDeleteCount;
+ }
+
+ /**
+ * Called once a new reader is opened for this segment ie. when deletes or updates are applied.
+ */
+ void onNewReader(SegmentReader reader, SegmentCommitInfo info) {
+ if (liveDocs == null) {
+ liveDocs = reader.getLiveDocs();
+ }
+ }
+
+ /**
+ * Resets the pending docs
+ */
+ void reset() {
+ pendingDeleteCount = 0;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("PendingDeletes(seg=").append(info);
+ sb.append(" numPendingDeletes=").append(pendingDeleteCount);
+ sb.append(" liveDocsShared=").append(liveDocsShared);
+ return sb.toString();
+ }
+
+ /**
+ * Writes the live docs to disk and returns <code>true</code> if any new docs were written.
+ */
+ boolean writeLiveDocs(Directory dir) throws IOException {
+ if (pendingDeleteCount == 0) {
+ return false;
+ }
+
+ Bits liveDocs = this.liveDocs;
+ assert liveDocs != null;
+ // We have new deletes
+ assert liveDocs.length() == info.info.maxDoc();
+
+ // Do this so we can delete any created files on
+ // exception; this saves all codecs from having to do
+ // it:
+ TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(dir);
+
+ // We can write directly to the actual name (vs to a
+ // .tmp & renaming it) because the file is not live
+ // until segments file is written:
+ boolean success = false;
+ try {
+ Codec codec = info.info.getCodec();
+ codec.liveDocsFormat().writeLiveDocs((MutableBits)liveDocs, trackingDir, info, pendingDeleteCount, IOContext.DEFAULT);
+ success = true;
+ } finally {
+ if (!success) {
+ // Advance only the nextWriteDelGen so that a 2nd
+ // attempt to write will write to a new file
+ info.advanceNextWriteDelGen();
+
+ // Delete any partially created file(s):
+ for (String fileName : trackingDir.getCreatedFiles()) {
+ IOUtils.deleteFilesIgnoringExceptions(dir, fileName);
+ }
+ }
+ }
+
+ // If we hit an exc in the line above (eg disk full)
+ // then info's delGen remains pointing to the previous
+ // (successfully written) del docs:
+ info.advanceDelGen();
+ info.setDelCount(info.getDelCount() + pendingDeleteCount);
+ reset();
+ return true;
+ }
+
+ /**
+ * Returns <code>true</code> iff the segment represented by this {@link PendingDeletes} is fully deleted
+ */
+ boolean isFullyDeleted() {
+ return info.getDelCount() + pendingDeleteCount == info.info.maxDoc();
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/acb3c379/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
index 16ea1e5..8a0e17e 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
@@ -20,6 +20,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -34,7 +35,6 @@ import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.FieldInfosFormat;
-import org.apache.lucene.codecs.LiveDocsFormat;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FlushInfo;
import org.apache.lucene.store.IOContext;
@@ -43,38 +43,27 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.InfoStream;
-import org.apache.lucene.util.MutableBits;
// Used by IndexWriter to hold open SegmentReaders (for
// searching or merging), plus pending deletes and updates,
// for a given segment
-class ReadersAndUpdates {
+final class ReadersAndUpdates {
// Not final because we replace (clone) when we need to
// change it and it's been shared:
- public final SegmentCommitInfo info;
+ final SegmentCommitInfo info;
// Tracks how many consumers are using this instance:
private final AtomicInteger refCount = new AtomicInteger(1);
- private final IndexWriter writer;
-
// Set once (null, and then maybe set, and never set again):
private SegmentReader reader;
- // Holds the current shared (readable and writable)
- // liveDocs. This is null when there are no deleted
- // docs, and it's copy-on-write (cloned whenever we need
- // to change it but it's been shared to an external NRT
- // reader).
- private Bits liveDocs;
-
// How many further deletions we've done against
// liveDocs vs when we loaded it or last wrote it:
- private int pendingDeleteCount;
+ private final PendingDeletes pendingDeletes;
- // True if the current liveDocs is referenced by an
- // external NRT reader:
- private boolean liveDocsShared;
+ // the major version this index was created with
+ private final int indexCreatedVersionMajor;
// Indicates whether this segment is currently being merged. While a segment
// is merging, all field updates are also registered in the
@@ -96,25 +85,23 @@ class ReadersAndUpdates {
// Only set if there are doc values updates against this segment, and the index is sorted:
Sorter.DocMap sortMap;
- public final AtomicLong ramBytesUsed = new AtomicLong();
-
- public ReadersAndUpdates(IndexWriter writer, SegmentCommitInfo info) {
- this.writer = writer;
+ final AtomicLong ramBytesUsed = new AtomicLong();
+
+ ReadersAndUpdates(int indexCreatedVersionMajor, SegmentCommitInfo info, SegmentReader reader,
+ PendingDeletes pendingDeletes) {
this.info = info;
- liveDocsShared = true;
+ this.pendingDeletes = pendingDeletes;
+ this.indexCreatedVersionMajor = indexCreatedVersionMajor;
+ this.reader = reader;
}
/** Init from a previously opened SegmentReader.
*
* <p>NOTE: steals incoming ref from reader. */
- public ReadersAndUpdates(IndexWriter writer, SegmentReader reader) {
- this.writer = writer;
- this.reader = reader;
- info = reader.getSegmentInfo();
- liveDocs = reader.getLiveDocs();
- liveDocsShared = true;
- pendingDeleteCount = reader.numDeletedDocs() - info.getDelCount();
- assert pendingDeleteCount >= 0: "got " + pendingDeleteCount + " reader.numDeletedDocs()=" + reader.numDeletedDocs() + " info.getDelCount()=" + info.getDelCount() + " maxDoc=" + reader.maxDoc() + " numDocs=" + reader.numDocs();
+ ReadersAndUpdates(int indexCreatedVersionMajor, SegmentReader reader, PendingDeletes pendingDeletes) {
+ this(indexCreatedVersionMajor, reader.getSegmentInfo(), reader, pendingDeletes);
+ assert pendingDeletes.numPendingDeletes() >= 0
+ : "got " + pendingDeletes.numPendingDeletes() + " reader.numDeletedDocs()=" + reader.numDeletedDocs() + " info.getDelCount()=" + info.getDelCount() + " maxDoc=" + reader.maxDoc() + " numDocs=" + reader.numDocs();
}
public void incRef() {
@@ -134,7 +121,7 @@ class ReadersAndUpdates {
}
public synchronized int getPendingDeleteCount() {
- return pendingDeleteCount;
+ return pendingDeletes.numPendingDeletes();
}
private synchronized boolean assertNoDupGen(List<DocValuesFieldUpdates> fieldUpdates, DocValuesFieldUpdates update) {
@@ -186,6 +173,7 @@ class ReadersAndUpdates {
// Call only from assert!
public synchronized boolean verifyDocCounts() {
int count;
+ Bits liveDocs = pendingDeletes.getLiveDocs();
if (liveDocs != null) {
count = 0;
for(int docID=0;docID<info.info.maxDoc();docID++) {
@@ -197,7 +185,7 @@ class ReadersAndUpdates {
count = info.info.maxDoc();
}
- assert info.info.maxDoc() - info.getDelCount() - pendingDeleteCount == count: "info.maxDoc=" + info.info.maxDoc() + " info.getDelCount()=" + info.getDelCount() + " pendingDeleteCount=" + pendingDeleteCount + " count=" + count;
+ assert info.info.maxDoc() - info.getDelCount() - pendingDeletes.numPendingDeletes() == count: "info.maxDoc=" + info.info.maxDoc() + " info.getDelCount()=" + info.getDelCount() + " pendingDeletes=" + pendingDeletes.numPendingDeletes() + " count=" + count;
return true;
}
@@ -205,12 +193,9 @@ class ReadersAndUpdates {
public synchronized SegmentReader getReader(IOContext context) throws IOException {
if (reader == null) {
// We steal returned ref:
- reader = new SegmentReader(info, writer.segmentInfos.getIndexCreatedVersionMajor(), context);
- if (liveDocs == null) {
- liveDocs = reader.getLiveDocs();
- }
+ reader = new SegmentReader(info, indexCreatedVersionMajor, context);
+ pendingDeletes.onNewReader(reader, info);
}
-
// Ref for caller
reader.incRef();
return reader;
@@ -222,16 +207,7 @@ class ReadersAndUpdates {
}
public synchronized boolean delete(int docID) throws IOException {
- initWritableLiveDocs();
- assert liveDocs != null;
- assert docID >= 0 && docID < liveDocs.length() : "out of bounds: docid=" + docID + " liveDocsLength=" + liveDocs.length() + " seg=" + info.info.name + " maxDoc=" + info.info.maxDoc();
- assert !liveDocsShared;
- final boolean didDelete = liveDocs.get(docID);
- if (didDelete) {
- ((MutableBits) liveDocs).clear(docID);
- pendingDeleteCount++;
- }
- return didDelete;
+ return pendingDeletes.delete(docID);
}
// NOTE: removes callers ref
@@ -258,10 +234,11 @@ class ReadersAndUpdates {
getReader(context).decRef();
assert reader != null;
}
- // force new liveDocs in initWritableLiveDocs even if it's null
- liveDocsShared = true;
+ // force new liveDocs
+ Bits liveDocs = pendingDeletes.getLiveDocs();
+ pendingDeletes.liveDocsShared();
if (liveDocs != null) {
- return new SegmentReader(reader.getSegmentInfo(), reader, liveDocs, info.info.maxDoc() - info.getDelCount() - pendingDeleteCount);
+ return new SegmentReader(reader.getSegmentInfo(), reader, liveDocs, info.info.maxDoc() - info.getDelCount() - pendingDeletes.numPendingDeletes());
} else {
// liveDocs == null and reader != null. That can only be if there are no deletes
assert reader.getLiveDocs() == null;
@@ -270,29 +247,12 @@ class ReadersAndUpdates {
}
}
- private synchronized void initWritableLiveDocs() throws IOException {
- assert info.info.maxDoc() > 0;
- if (liveDocsShared) {
- // Copy on write: this means we've cloned a
- // SegmentReader sharing the current liveDocs
- // instance; must now make a private clone so we can
- // change it:
- LiveDocsFormat liveDocsFormat = info.info.getCodec().liveDocsFormat();
- if (liveDocs == null) {
- liveDocs = liveDocsFormat.newLiveDocs(info.info.maxDoc());
- } else {
- liveDocs = liveDocsFormat.newLiveDocs(liveDocs);
- }
- liveDocsShared = false;
- }
- }
public synchronized Bits getLiveDocs() {
- return liveDocs;
+ return pendingDeletes.getLiveDocs();
}
public synchronized void dropChanges() {
- assert Thread.holdsLock(writer);
// Discard (don't save) changes when we are dropping
// the reader; this is used only on the sub-readers
// after a successful merge. If deletes had
@@ -300,7 +260,7 @@ class ReadersAndUpdates {
// is running, by now we have carried forward those
// deletes onto the newly merged segment, so we can
// discard them on the sub-readers:
- pendingDeleteCount = 0;
+ pendingDeletes.reset();
dropMergingUpdates();
}
@@ -308,47 +268,7 @@ class ReadersAndUpdates {
// _X_N updates files) to the directory; returns true if it wrote any file
// and false if there were no new deletes or updates to write:
public synchronized boolean writeLiveDocs(Directory dir) throws IOException {
- if (pendingDeleteCount == 0) {
- return false;
- }
-
- // We have new deletes
- assert liveDocs.length() == info.info.maxDoc();
-
- // Do this so we can delete any created files on
- // exception; this saves all codecs from having to do
- // it:
- TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(dir);
-
- // We can write directly to the actual name (vs to a
- // .tmp & renaming it) because the file is not live
- // until segments file is written:
- boolean success = false;
- try {
- Codec codec = info.info.getCodec();
- codec.liveDocsFormat().writeLiveDocs((MutableBits)liveDocs, trackingDir, info, pendingDeleteCount, IOContext.DEFAULT);
- success = true;
- } finally {
- if (!success) {
- // Advance only the nextWriteDelGen so that a 2nd
- // attempt to write will write to a new file
- info.advanceNextWriteDelGen();
-
- // Delete any partially created file(s):
- for (String fileName : trackingDir.getCreatedFiles()) {
- IOUtils.deleteFilesIgnoringExceptions(dir, fileName);
- }
- }
- }
-
- // If we hit an exc in the line above (eg disk full)
- // then info's delGen remains pointing to the previous
- // (successfully written) del docs:
- info.advanceDelGen();
- info.setDelCount(info.getDelCount() + pendingDeleteCount);
- pendingDeleteCount = 0;
-
- return true;
+ return pendingDeletes.writeLiveDocs(dir);
}
@SuppressWarnings("synthetic-access")
@@ -404,7 +324,6 @@ class ReadersAndUpdates {
if (fieldInfoIn != fieldInfo) {
throw new IllegalArgumentException("wrong fieldInfo");
}
- final int maxDoc = reader.maxDoc();
DocValuesFieldUpdates.Iterator[] subs = new DocValuesFieldUpdates.Iterator[updatesToApply.size()];
for(int i=0;i<subs.length;i++) {
subs[i] = updatesToApply.get(i).iterator();
@@ -623,8 +542,8 @@ class ReadersAndUpdates {
}
}
- private synchronized Set<String> writeFieldInfosGen(FieldInfos fieldInfos, Directory dir, DocValuesFormat dvFormat,
- FieldInfosFormat infosFormat) throws IOException {
+ private synchronized Set<String> writeFieldInfosGen(FieldInfos fieldInfos, Directory dir,
+ FieldInfosFormat infosFormat) throws IOException {
final long nextFieldInfosGen = info.getNextFieldInfosGen();
final String segmentSuffix = Long.toString(nextFieldInfosGen, Character.MAX_RADIX);
// we write approximately that many bytes (based on Lucene46DVF):
@@ -639,22 +558,15 @@ class ReadersAndUpdates {
return trackingDir.getCreatedFiles();
}
- public synchronized boolean writeFieldUpdates(Directory dir, long maxDelGen, InfoStream infoStream) throws IOException {
-
+ public synchronized boolean writeFieldUpdates(Directory dir, FieldInfos.FieldNumbers fieldNumbers, long maxDelGen, InfoStream infoStream) throws IOException {
long startTimeNS = System.nanoTime();
-
- assert Thread.holdsLock(writer);
-
final Map<Integer,Set<String>> newDVFiles = new HashMap<>();
Set<String> fieldInfosFiles = null;
FieldInfos fieldInfos = null;
-
boolean any = false;
- int count = 0;
for (List<DocValuesFieldUpdates> updates : pendingDVUpdates.values()) {
// Sort by increasing delGen:
- Collections.sort(updates, (a, b) -> Long.compare(a.delGen, b.delGen));
- count += updates.size();
+ Collections.sort(updates, Comparator.comparingLong(a -> a.delGen));
for (DocValuesFieldUpdates update : updates) {
if (update.delGen <= maxDelGen && update.any()) {
any = true;
@@ -680,7 +592,7 @@ class ReadersAndUpdates {
// IndexWriter.commitMergedDeletes).
final SegmentReader reader;
if (this.reader == null) {
- reader = new SegmentReader(info, writer.segmentInfos.getIndexCreatedVersionMajor(), IOContext.READONCE);
+ reader = new SegmentReader(info, indexCreatedVersionMajor, IOContext.READONCE);
} else {
reader = this.reader;
}
@@ -688,7 +600,7 @@ class ReadersAndUpdates {
try {
// clone FieldInfos so that we can update their dvGen separately from
// the reader's infos and write them to a new fieldInfos_gen file
- FieldInfos.Builder builder = new FieldInfos.Builder(writer.globalFieldNumberMap);
+ FieldInfos.Builder builder = new FieldInfos.Builder(fieldNumbers);
// cannot use builder.add(reader.getFieldInfos()) because it does not
// clone FI.attributes as well FI.dvGen
for (FieldInfo fi : reader.getFieldInfos()) {
@@ -713,7 +625,7 @@ class ReadersAndUpdates {
handleNumericDVUpdates(fieldInfos, trackingDir, docValuesFormat, reader, newDVFiles, maxDelGen, infoStream);
handleBinaryDVUpdates(fieldInfos, trackingDir, docValuesFormat, reader, newDVFiles, maxDelGen, infoStream);
- fieldInfosFiles = writeFieldInfosGen(fieldInfos, trackingDir, docValuesFormat, codec.fieldInfosFormat());
+ fieldInfosFiles = writeFieldInfosGen(fieldInfos, trackingDir, codec.fieldInfosFormat());
} finally {
if (reader != this.reader) {
reader.close();
@@ -763,11 +675,12 @@ class ReadersAndUpdates {
// if there is a reader open, reopen it to reflect the updates
if (reader != null) {
- SegmentReader newReader = new SegmentReader(info, reader, liveDocs, info.info.maxDoc() - info.getDelCount() - pendingDeleteCount);
+ SegmentReader newReader = new SegmentReader(info, reader, pendingDeletes.getLiveDocs(), info.info.maxDoc() - info.getDelCount() - pendingDeletes.numPendingDeletes());
boolean success2 = false;
try {
reader.decRef();
reader = newReader;
+ pendingDeletes.onNewReader(reader, info);
success2 = true;
} finally {
if (success2 == false) {
@@ -792,14 +705,10 @@ class ReadersAndUpdates {
}
info.setDocValuesUpdatesFiles(newDVFiles);
- // wrote new files, should checkpoint()
- writer.checkpointNoSIS();
-
if (infoStream.isEnabled("BD")) {
infoStream.message("BD", String.format(Locale.ROOT, "done write field updates for seg=%s; took %.3fs; new files: %s",
info, (System.nanoTime() - startTimeNS)/1000000000.0, newDVFiles));
}
-
return true;
}
@@ -829,12 +738,11 @@ class ReadersAndUpdates {
}
SegmentReader reader = getReader(context);
- int delCount = pendingDeleteCount + info.getDelCount();
+ int delCount = pendingDeletes.numPendingDeletes() + info.getDelCount();
if (delCount != reader.numDeletedDocs()) {
-
// beware of zombies:
assert delCount > reader.numDeletedDocs(): "delCount=" + delCount + " reader.numDeletedDocs()=" + reader.numDeletedDocs();
-
+ Bits liveDocs = pendingDeletes.getLiveDocs();
assert liveDocs != null;
// Create a new reader with the latest live docs:
@@ -842,6 +750,7 @@ class ReadersAndUpdates {
boolean success = false;
try {
reader.decRef();
+ pendingDeletes.onNewReader(newReader, info);
success = true;
} finally {
if (success == false) {
@@ -851,7 +760,7 @@ class ReadersAndUpdates {
reader = newReader;
}
- liveDocsShared = true;
+ pendingDeletes.liveDocsShared();
assert verifyDocCounts();
@@ -877,8 +786,12 @@ class ReadersAndUpdates {
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("ReadersAndLiveDocs(seg=").append(info);
- sb.append(" pendingDeleteCount=").append(pendingDeleteCount);
- sb.append(" liveDocsShared=").append(liveDocsShared);
+ sb.append(" pendingDeletes=").append(pendingDeletes);
return sb.toString();
}
+
+ public synchronized boolean isFullyDeleted() {
+ return pendingDeletes.isFullyDeleted();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/acb3c379/lucene/core/src/java/org/apache/lucene/util/Bits.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/Bits.java b/lucene/core/src/java/org/apache/lucene/util/Bits.java
index 101122e..29935e7 100644
--- a/lucene/core/src/java/org/apache/lucene/util/Bits.java
+++ b/lucene/core/src/java/org/apache/lucene/util/Bits.java
@@ -22,7 +22,7 @@ package org.apache.lucene.util;
* @lucene.experimental
*/
-public interface Bits {
+public interface Bits {
/**
* Returns the value of the bit with the specified <code>index</code>.
* @param index index, should be non-negative and < {@link #length()}.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/acb3c379/lucene/core/src/test/org/apache/lucene/index/TestPendingDeletes.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPendingDeletes.java b/lucene/core/src/test/org/apache/lucene/index/TestPendingDeletes.java
new file mode 100644
index 0000000..39f5680
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/index/TestPendingDeletes.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.index;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.StringHelper;
+import org.apache.lucene.util.TestUtil;
+import org.apache.lucene.util.Version;
+
+public class TestPendingDeletes extends LuceneTestCase {
+
+ public void testDeleteDoc() throws IOException {
+ RAMDirectory dir = new RAMDirectory();
+ SegmentInfo si = new SegmentInfo(dir, Version.LATEST, Version.LATEST, "test", 10, false, Codec.getDefault(),
+ Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), null);
+ SegmentCommitInfo commitInfo = new SegmentCommitInfo(si, 0, 0, 0, 0);
+ PendingDeletes deletes = new PendingDeletes(null, commitInfo);
+ assertNull(deletes.getLiveDocs());
+ int docToDelete = TestUtil.nextInt(random(), 0, 7);
+ assertTrue(deletes.delete(docToDelete));
+ assertNotNull(deletes.getLiveDocs());
+ assertEquals(1, deletes.numPendingDeletes());
+
+ Bits liveDocs = deletes.getLiveDocs();
+ assertFalse(liveDocs.get(docToDelete));
+ assertFalse(deletes.delete(docToDelete)); // delete again
+
+ // make sure we are live ie. mutable
+ assertTrue(liveDocs.get(8));
+ assertTrue(deletes.delete(8));
+ assertFalse(liveDocs.get(8));
+ assertEquals(2, deletes.numPendingDeletes());
+
+ deletes.liveDocsShared();
+
+ // make sure we are live ie. mutable
+ assertTrue(liveDocs.get(9));
+ assertTrue(deletes.delete(9));
+ assertTrue(liveDocs.get(9));
+ liveDocs = deletes.getLiveDocs();
+ assertFalse(liveDocs.get(9));
+ assertFalse(liveDocs.get(8));
+ assertFalse(liveDocs.get(docToDelete));
+ assertEquals(3, deletes.numPendingDeletes());
+ dir.close();
+ }
+
+ public void testWriteLiveDocs() throws IOException {
+ RAMDirectory dir = new RAMDirectory();
+ SegmentInfo si = new SegmentInfo(dir, Version.LATEST, Version.LATEST, "test", 6, false, Codec.getDefault(),
+ Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), null);
+ SegmentCommitInfo commitInfo = new SegmentCommitInfo(si, 0, 0, 0, 0);
+ PendingDeletes deletes = new PendingDeletes(null, commitInfo);
+ assertFalse(deletes.writeLiveDocs(dir));
+ assertEquals(0, dir.listAll().length);
+ boolean secondDocDeletes = random().nextBoolean();
+ deletes.delete(5);
+ if (secondDocDeletes) {
+ deletes.liveDocsShared();
+ deletes.delete(2);
+ }
+ assertEquals(0, commitInfo.getDelGen());
+ assertEquals(0, commitInfo.getDelCount());
+
+ assertEquals(secondDocDeletes ? 2 : 1, deletes.numPendingDeletes());
+ assertTrue(deletes.writeLiveDocs(dir));
+ assertEquals(1, dir.listAll().length);
+ Bits liveDocs = Codec.getDefault().liveDocsFormat().readLiveDocs(dir, commitInfo, IOContext.DEFAULT);
+ assertFalse(liveDocs.get(5));
+ if (secondDocDeletes) {
+ assertFalse(liveDocs.get(2));
+ } else {
+ assertTrue(liveDocs.get(2));
+ }
+ assertTrue(liveDocs.get(0));
+ assertTrue(liveDocs.get(1));
+ assertTrue(liveDocs.get(3));
+ assertTrue(liveDocs.get(4));
+
+ assertEquals(0, deletes.numPendingDeletes());
+ assertEquals(secondDocDeletes ? 2 : 1, commitInfo.getDelCount());
+ assertEquals(1, commitInfo.getDelGen());
+
+ deletes.delete(0);
+ assertTrue(deletes.writeLiveDocs(dir));
+ assertEquals(2, dir.listAll().length);
+ liveDocs = Codec.getDefault().liveDocsFormat().readLiveDocs(dir, commitInfo, IOContext.DEFAULT);
+ assertFalse(liveDocs.get(5));
+ if (secondDocDeletes) {
+ assertFalse(liveDocs.get(2));
+ } else {
+ assertTrue(liveDocs.get(2));
+ }
+ assertFalse(liveDocs.get(0));
+ assertTrue(liveDocs.get(1));
+ assertTrue(liveDocs.get(3));
+ assertTrue(liveDocs.get(4));
+
+ assertEquals(0, deletes.numPendingDeletes());
+ assertEquals(secondDocDeletes ? 3 : 2, commitInfo.getDelCount());
+ assertEquals(2, commitInfo.getDelGen());
+ dir.close();
+ }
+
+ public void testIsFullyDeleted() throws IOException {
+ RAMDirectory dir = new RAMDirectory();
+ SegmentInfo si = new SegmentInfo(dir, Version.LATEST, Version.LATEST, "test", 3, false, Codec.getDefault(),
+ Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), null);
+ SegmentCommitInfo commitInfo = new SegmentCommitInfo(si, 0, 0, 0, 0);
+ PendingDeletes deletes = new PendingDeletes(null, commitInfo);
+ for (int i = 0; i < 3; i++) {
+ assertTrue(deletes.delete(i));
+ if (random().nextBoolean()) {
+ assertTrue(deletes.writeLiveDocs(dir));
+ }
+ assertEquals(i == 2, deletes.isFullyDeleted());
+ }
+ }
+}
[02/34] lucene-solr:jira/solr-12095: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/lucene-solr
Posted by sh...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/lucene-solr
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/358e5959
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/358e5959
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/358e5959
Branch: refs/heads/jira/solr-12095
Commit: 358e59596d17ba34452ea923e048afee6233d597
Parents: 5b429df 668b817
Author: Karl Wright <Da...@gmail.com>
Authored: Thu Mar 29 10:37:45 2018 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Thu Mar 29 10:37:45 2018 -0400
----------------------------------------------------------------------
dev-tools/scripts/crawl.maven.release.dist.sh | 4 +-
dev-tools/scripts/prep-solr-ref-guide-rc.sh | 20 +++----
dev-tools/scripts/smokeTestRelease.py | 44 +++++++--------
lucene/CHANGES.txt | 9 +++
lucene/common-build.xml | 10 ++--
.../lucene/index/IndexDeletionPolicy.java | 18 +++---
.../apache/lucene/index/NoDeletionPolicy.java | 6 --
solr/CHANGES.txt | 14 ++++-
solr/build.xml | 2 +-
.../handler/dataimport/DataImportHandler.java | 6 +-
.../cloud/LeaderInitiatedRecoveryThread.java | 6 +-
.../org/apache/solr/core/CoreContainer.java | 45 ++++++++-------
.../apache/solr/core/HdfsDirectoryFactory.java | 8 +--
.../solr/core/IndexDeletionPolicyWrapper.java | 6 --
.../src/java/org/apache/solr/core/SolrCore.java | 41 ++++++++------
.../java/org/apache/solr/core/SolrInfoBean.java | 2 +-
.../apache/solr/handler/ReplicationHandler.java | 44 +++++++--------
.../apache/solr/handler/RequestHandlerBase.java | 12 ++--
.../solr/handler/admin/CoreAdminHandler.java | 4 +-
.../component/HttpShardHandlerFactory.java | 9 ++-
.../solr/handler/component/SearchComponent.java | 2 +-
.../handler/component/SuggestComponent.java | 11 +++-
.../solr/highlight/HighlightingPluginBase.java | 6 +-
.../solr/metrics/SolrCoreMetricManager.java | 7 ++-
.../apache/solr/metrics/SolrMetricManager.java | 58 ++++++++++++++++++--
.../apache/solr/metrics/SolrMetricProducer.java | 7 ++-
.../metrics/reporters/solr/SolrReporter.java | 2 +-
.../solr/search/ExtendedDismaxQParser.java | 2 +-
.../org/apache/solr/search/FastLRUCache.java | 9 ++-
.../java/org/apache/solr/search/LFUCache.java | 8 ++-
.../java/org/apache/solr/search/LRUCache.java | 10 +++-
.../apache/solr/search/SolrFieldCacheBean.java | 4 +-
.../apache/solr/search/SolrIndexSearcher.java | 37 +++++++------
.../apache/solr/servlet/SolrDispatchFilter.java | 26 ++++++---
.../apache/solr/store/blockcache/Metrics.java | 8 ++-
.../solr/store/hdfs/HdfsLocalityReporter.java | 8 ++-
.../solr/update/DirectUpdateHandler2.java | 28 ++++++----
.../java/org/apache/solr/update/PeerSync.java | 2 +-
.../org/apache/solr/update/SolrIndexWriter.java | 34 +++++++-----
.../java/org/apache/solr/update/UpdateLog.java | 14 +++--
.../apache/solr/update/UpdateShardHandler.java | 6 +-
.../processor/AtomicUpdateProcessorFactory.java | 26 ++++++---
.../stats/InstrumentedHttpRequestExecutor.java | 7 ++-
...entedPoolingHttpClientConnectionManager.java | 19 +++++--
.../solr/collection1/conf/schema12.xml | 12 ++++
.../test/org/apache/solr/CursorPagingTest.java | 5 +-
.../test/org/apache/solr/SolrInfoBeanTest.java | 2 +-
.../org/apache/solr/cloud/ZkSolrClientTest.java | 30 ++++++++--
.../sim/SimClusterStateProvider.java | 6 +-
.../solr/core/ExitableDirectoryReaderTest.java | 7 ++-
.../solr/core/HdfsDirectoryFactoryTest.java | 4 +-
.../test/org/apache/solr/core/MockInfoBean.java | 8 ++-
.../core/MockQuerySenderListenerReqHandler.java | 6 +-
.../apache/solr/core/TestJmxIntegration.java | 4 +-
.../org/apache/solr/metrics/JvmMetricsTest.java | 2 +-
.../solr/metrics/SolrCoreMetricManagerTest.java | 2 +-
.../solr/metrics/SolrMetricManagerTest.java | 10 ++--
.../solr/metrics/SolrMetricTestUtils.java | 2 +-
.../solr/search/TestExtendedDismaxParser.java | 19 ++++++-
.../apache/solr/search/TestFastLRUCache.java | 14 ++---
.../org/apache/solr/search/TestLFUCache.java | 6 +-
.../org/apache/solr/search/TestLRUCache.java | 4 +-
.../solr/search/TestReRankQParserPlugin.java | 3 +-
.../apache/solr/search/TestSolr4Spatial2.java | 3 +-
.../solr/search/TestSolrFieldCacheBean.java | 8 +--
.../apache/solr/search/TestSolrQueryParser.java | 9 +--
.../apache/solr/search/join/BJQParserTest.java | 9 +--
.../solr/search/join/TestScoreJoinQPScore.java | 3 +-
.../solr/store/blockcache/BufferStoreTest.java | 4 +-
.../AtomicUpdateProcessorFactoryTest.java | 1 -
solr/solr-ref-guide/src/meta-docs/publish.adoc | 8 ++-
71 files changed, 524 insertions(+), 318 deletions(-)
----------------------------------------------------------------------
[11/34] lucene-solr:jira/solr-12095: SOLR-12152: Fix compilation
error due to missing import
Posted by sh...@apache.org.
SOLR-12152: Fix compilation error due to missing import
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/1aafc90c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/1aafc90c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/1aafc90c
Branch: refs/heads/jira/solr-12095
Commit: 1aafc90cbeeb9435fa5a42939f92e060bed0a848
Parents: ac8cbaa
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Fri Mar 30 12:23:06 2018 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Fri Mar 30 12:23:06 2018 +0530
----------------------------------------------------------------------
.../solr/cloud/autoscaling/ScheduledTriggerIntegrationTest.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1aafc90c/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledTriggerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledTriggerIntegrationTest.java
index 24e7420..24a8c6c 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledTriggerIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledTriggerIntegrationTest.java
@@ -27,6 +27,7 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
+import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
@@ -47,7 +48,7 @@ import static org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.createAut
* Integration test for {@link ScheduledTrigger}
*/
@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG")
-@BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 26-Mar-2018
+@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 26-Mar-2018
public class ScheduledTriggerIntegrationTest extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
[20/34] lucene-solr:jira/solr-12095: SOLR-12066: Cleanup deleted core
when node start
Posted by sh...@apache.org.
SOLR-12066: Cleanup deleted core when node start
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/35bfe897
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/35bfe897
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/35bfe897
Branch: refs/heads/jira/solr-12095
Commit: 35bfe897901f1b51bce654b49aecd9560bfa797f
Parents: d483108
Author: Cao Manh Dat <da...@apache.org>
Authored: Fri Mar 30 20:11:39 2018 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Fri Mar 30 20:11:39 2018 +0700
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
.../org/apache/solr/cloud/ZkController.java | 22 ++++++++++---
.../org/apache/solr/core/CoreContainer.java | 7 ++++-
.../apache/solr/cloud/DeleteReplicaTest.java | 33 ++++++++++++++++++++
4 files changed, 59 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/35bfe897/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 5854e0f..12bc25a 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -110,6 +110,8 @@ Optimizations
* SOLR-12146: LIR should skip deleted replicas (Cao Manh Dat)
+* SOLR-12066: Cleanup deleted core when node start (Cao Manh Dat)
+
Other Changes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/35bfe897/solr/core/src/java/org/apache/solr/cloud/ZkController.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index c0ddd26..872a8b9 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -1661,6 +1661,9 @@ public class ZkController {
Thread.currentThread().interrupt();
log.error("", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
+ } catch (NotInClusterStateException e) {
+ // make the stack trace less verbose
+ throw e;
} catch (Exception e) {
log.error("", e);
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "", e);
@@ -1688,7 +1691,7 @@ public class ZkController {
return true;
}
- private void checkStateInZk(CoreDescriptor cd) throws InterruptedException {
+ private void checkStateInZk(CoreDescriptor cd) throws InterruptedException, NotInClusterStateException {
if (!Overseer.isLegacy(zkStateReader)) {
CloudDescriptor cloudDesc = cd.getCloudDescriptor();
String nodeName = cloudDesc.getCoreNodeName();
@@ -1722,7 +1725,8 @@ public class ZkController {
}
Replica replica = slice.getReplica(coreNodeName);
if (replica == null) {
- errorMessage.set("coreNodeName " + coreNodeName + " does not exist in shard " + cloudDesc.getShardId());
+ errorMessage.set("coreNodeName " + coreNodeName + " does not exist in shard " + cloudDesc.getShardId() +
+ ", ignore the exception if the replica was deleted");
return false;
}
return true;
@@ -1730,8 +1734,9 @@ public class ZkController {
} catch (TimeoutException e) {
String error = errorMessage.get();
if (error == null)
- error = "Replica " + coreNodeName + " is not present in cluster state";
- throw new SolrException(ErrorCode.SERVER_ERROR, error + ": " + collectionState.get());
+ error = "coreNodeName " + coreNodeName + " does not exist in shard " + cloudDesc.getShardId() +
+ ", ignore the exception if the replica was deleted";
+ throw new NotInClusterStateException(ErrorCode.SERVER_ERROR, error);
}
}
}
@@ -2711,6 +2716,15 @@ public class ZkController {
}
}
+ /**
+ * Thrown during pre register process if the replica is not present in clusterstate
+ */
+ public static class NotInClusterStateException extends SolrException {
+ public NotInClusterStateException(ErrorCode code, String msg) {
+ super(code, msg);
+ }
+ }
+
public boolean checkIfCoreNodeNameAlreadyExists(CoreDescriptor dcore) {
DocCollection collection = zkStateReader.getClusterState().getCollectionOrNull(dcore.getCollectionName());
if (collection != null) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/35bfe897/solr/core/src/java/org/apache/solr/core/CoreContainer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index b667bc0..74b718c 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -677,7 +677,7 @@ public class CoreContainer {
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
} catch (ExecutionException e) {
- log.error("Error waiting for SolrCore to be created", e);
+ log.error("Error waiting for SolrCore to be loaded on startup", e.getCause());
}
}
} finally {
@@ -1063,6 +1063,11 @@ public class CoreContainer {
return core;
} catch (Exception e) {
coreInitFailures.put(dcore.getName(), new CoreLoadFailure(dcore, e));
+ if (e instanceof ZkController.NotInClusterStateException && !newCollection) {
+ // this mostly happen when the core is deleted when this node is down
+ unload(dcore.getName(), true, true, true);
+ throw e;
+ }
solrCores.removeCoreDescriptor(dcore);
final SolrException solrException = new SolrException(ErrorCode.SERVER_ERROR, "Unable to create core [" + dcore.getName() + "]", e);
if(core != null && !core.isClosed())
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/35bfe897/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java b/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java
index 3208ebd..1a021d7 100644
--- a/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java
@@ -41,7 +41,10 @@ import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.TimeSource;
import org.apache.solr.common.util.Utils;
+import org.apache.solr.core.CoreDescriptor;
+import org.apache.solr.core.SolrCore;
import org.apache.solr.core.ZkContainer;
+import org.apache.solr.util.FileUtils;
import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.KeeperException;
import org.junit.BeforeClass;
@@ -153,6 +156,36 @@ public class DeleteReplicaTest extends SolrCloudTestCase {
}
@Test
+ public void deleteReplicaOnDownNode() throws Exception {
+ final String collectionName = "deleteReplicaOnDownNode";
+ CollectionAdminRequest.createCollection(collectionName, "conf", 1, 2).process(cluster.getSolrClient());
+ waitForState("Expected one shards with two replicas", collectionName, clusterShape(1, 2));
+
+ Slice shard = getCollectionState(collectionName).getSlice("shard1");
+ Replica replica = shard.getReplicas(rep -> !rep.getName().equals(shard.getLeader().getName())).get(0);
+ JettySolrRunner replicaJetty = getJettyForReplica(replica);
+ CoreDescriptor replicaCd;
+ try (SolrCore core = replicaJetty.getCoreContainer().getCore(replica.getCoreName())) {
+ replicaCd = core.getCoreDescriptor();
+ }
+ assertNotNull("Expected core descriptor of "+ replica.getName() + " is not null",replicaCd);
+ String replicaJettyNodeName = replicaJetty.getNodeName();
+
+ // shutdown node of a replica
+ replicaJetty.stop();
+ waitForNodeLeave(replicaJettyNodeName);
+ waitForState("Expected one shards with one replica", collectionName, clusterShape(1, 1));
+ CollectionAdminRequest.deleteReplica(collectionName, shard.getName(), replica.getName()).process(cluster.getSolrClient());
+ waitForState("Expected only one replica left", collectionName, (liveNodes, collectionState) -> collectionState.getReplicas().size() == 1);
+
+ // restart the test and make sure the data get deleted
+ replicaJetty.start();
+ TimeOut timeOut = new TimeOut(60, TimeUnit.SECONDS, TimeSource.NANO_TIME);
+ timeOut.waitFor("Expected data dir and instance dir of " + replica.getName() + " is deleted", ()
+ -> !Files.exists(replicaCd.getInstanceDir()) && !FileUtils.fileExists(replicaCd.getDataDir()));
+ }
+
+ @Test
public void deleteReplicaByCountForAllShards() throws Exception {
final String collectionName = "deleteByCountNew";
[17/34] lucene-solr:jira/solr-12095: LUCENE-8227: Widen plane
envelopes a small amount to allow for some imprecision when intersecting
inner and outer edges.
Posted by sh...@apache.org.
LUCENE-8227: Widen plane envelopes a small amount to allow for some imprecision when intersecting inner and outer edges.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e06554ad
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e06554ad
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e06554ad
Branch: refs/heads/jira/solr-12095
Commit: e06554ad408a60dc37eb77c0aeeff542e506f1a5
Parents: ae6d29f
Author: Karl Wright <Da...@gmail.com>
Authored: Fri Mar 30 08:36:47 2018 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Fri Mar 30 08:36:47 2018 -0400
----------------------------------------------------------------------
.../org/apache/lucene/spatial3d/geom/Plane.java | 4 +-
.../apache/lucene/spatial3d/TestGeo3DPoint.java | 1 -
.../lucene/spatial3d/geom/GeoPolygonTest.java | 73 ++++++++++++++++++++
3 files changed, 76 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e06554ad/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java
index e40fb27..5ecee28 100755
--- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java
@@ -23,6 +23,8 @@ package org.apache.lucene.spatial3d.geom;
* @lucene.experimental
*/
public class Plane extends Vector {
+ /** For plane envelopes, we need a small distance that can't lead to numerical confusion. */
+ public final static double MINIMUM_PLANE_OFFSET = MINIMUM_RESOLUTION * 1.5;
/** An array with no points in it */
public final static GeoPoint[] NO_POINTS = new GeoPoint[0];
/** An array with no bounds in it */
@@ -114,7 +116,7 @@ public class Plane extends Vector {
* or false in the negative direction.
*/
public Plane(final Plane basePlane, final boolean above) {
- this(basePlane.x, basePlane.y, basePlane.z, above?Math.nextUp(basePlane.D + MINIMUM_RESOLUTION):Math.nextDown(basePlane.D - MINIMUM_RESOLUTION));
+ this(basePlane.x, basePlane.y, basePlane.z, above?Math.nextUp(basePlane.D + MINIMUM_PLANE_OFFSET):Math.nextDown(basePlane.D - MINIMUM_PLANE_OFFSET));
}
/** Construct the most accurate normalized plane through an x-y point and including the Z axis.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e06554ad/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java
index 5d58d5e..861e26e 100644
--- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java
+++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java
@@ -188,7 +188,6 @@ public class TestGeo3DPoint extends LuceneTestCase {
}
/** Tests consistency of GeoArea.getRelationship vs GeoShape.isWithin */
- @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/LUCENE-8227")
public void testGeo3DRelations() throws Exception {
int numDocs = atLeast(1000);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e06554ad/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
index 65659b3..11e44af 100755
--- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
+++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
@@ -1257,4 +1257,77 @@ shape:
}
+ /*
+ [junit4] 1> doc=754 is contained by shape but is outside the returned XYZBounds
+ [junit4] 1> unquantized=[lat=2.4043303687704734E-204, lon=3.1342447995980507([X=-1.0010918284309325, Y=0.007356008974104805, Z=2.4070204634028112E-204])]
+ [junit4] 1> quantized=[X=-1.0010918285430614, Y=0.007356008812298254, Z=2.3309121299774915E-10]
+
+ [junit4] 1> doc=3728 is contained by shape but is outside the returned XYZBounds
+ [junit4] 1> unquantized=[lat=2.4457272005608357E-47, lon=-3.1404077424936307([X=-1.001118151199965, Y=-0.0011862365610909341, Z=2.448463612203698E-47])]
+ [junit4] 1> quantized=[X=-1.0011181510675629, Y=-0.001186236379718708, Z=2.3309121299774915E-10]
+
+ [junit4] 1> shape=GeoComplexPolygon: {planetmodel=PlanetModel.WGS84, number of shapes=1, address=7969cab3,
+ testPoint=[X=-0.07416172733314662, Y=0.5686488061136892, Z=0.8178445379402641], testPointInSet=true, shapes={ {
+ [lat=-1.5707963267948966, lon=-1.0755217966112058([X=2.903696886845155E-17, Y=-5.375400029710238E-17, Z=-0.997762292022105])],
+ [lat=-1.327365682666958, lon=-2.9674513704178316([X=-0.23690293696956322, Y=-0.04167672037374933, Z=-0.9685334156912658])],
+ [lat=0.32288591161895097, lon=3.141592653589793([X=-0.9490627533610154, Y=1.1622666630935417E-16, Z=0.3175519551883462])],
+ [lat=0.0, lon=0.0([X=1.0011188539924791, Y=0.0, Z=0.0])],
+ [lat=0.2839194570254642, lon=-1.2434404554202965([X=0.30893121415043073, Y=-0.9097632721627391, Z=0.2803596238536593])]}}
+ */
+ @Test
+ public void testLUCENE8227_case2() {
+ List<GeoPoint> points = new ArrayList<>();
+ points.add(new GeoPoint(PlanetModel.WGS84, -1.5707963267948966, -1.0755217966112058));
+ points.add(new GeoPoint(PlanetModel.WGS84, -1.327365682666958, -2.9674513704178316));
+ points.add(new GeoPoint(PlanetModel.WGS84, 0.32288591161895097, 3.141592653589793));
+ points.add(new GeoPoint(PlanetModel.WGS84, 0.0, 0.0));
+ points.add(new GeoPoint(PlanetModel.WGS84, 0.2839194570254642, -1.2434404554202965));
+ GeoPolygonFactory.PolygonDescription pd = new GeoPolygonFactory.PolygonDescription(points);
+
+ for (int i = 0; i < points.size(); i++) {
+ System.out.println("Point "+i+": "+points.get(i));
+ }
+
+ final GeoPoint unquantized = new GeoPoint(PlanetModel.WGS84, 2.4457272005608357E-47, -3.1404077424936307);
+ final GeoPoint quantized = new GeoPoint(-1.0011181510675629, -0.001186236379718708, 2.3309121299774915E-10);
+
+ // Is the north pole in set, or out of set?
+ final GeoPoint northPole = new GeoPoint(PlanetModel.WGS84, Math.PI * 0.5, 0.0);
+ final GeoPoint negativeX = new GeoPoint(PlanetModel.WGS84, 0.0, Math.PI);
+ final GeoPoint negativeY = new GeoPoint(PlanetModel.WGS84, 0.0, -Math.PI * 0.5);
+ final GeoPoint positiveY = new GeoPoint(PlanetModel.WGS84, 0.0, Math.PI * 0.5);
+ final GeoPoint testPoint = new GeoPoint(-0.07416172733314662, 0.5686488061136892, 0.8178445379402641);
+
+ // Construct a standard polygon first to see what that does. This winds up being a large polygon under the covers.
+ GeoPolygon standard = GeoPolygonFactory.makeGeoPolygon(PlanetModel.WGS84, pd);
+
+ System.out.println("Shape = "+standard);
+
+ // This should be true, by inspection, but is false. That's the cause for the failure.
+ assertTrue(standard.isWithin(negativeX));
+ System.out.println("Negative x pole in set? "+standard.isWithin(negativeX));
+
+ System.out.println("Test point in set? "+standard.isWithin(testPoint));
+ assertTrue(standard.isWithin(testPoint));
+
+ // This is in-set because it's on an edge
+ System.out.println("North pole in set? "+standard.isWithin(northPole));
+ assertTrue(standard.isWithin(northPole));
+
+ // This is in-set
+ System.out.println("Plus-Y pole in set? "+standard.isWithin(positiveY));
+ assertTrue(standard.isWithin(positiveY));
+
+
+ final XYZBounds standardBounds = new XYZBounds();
+ standard.getBounds(standardBounds);
+ System.out.println("Bounds = "+standardBounds);
+ final XYZSolid standardSolid = XYZSolidFactory.makeXYZSolid(PlanetModel.WGS84, standardBounds);
+
+ // If within shape, should be within bounds
+ assertTrue(standard.isWithin(unquantized)?standardSolid.isWithin(unquantized):true);
+ assertTrue(standard.isWithin(quantized)?standardSolid.isWithin(quantized):true);
+
+ }
+
}
[06/34] lucene-solr:jira/solr-12095: SOLR-12162:
CorePropertiesLocator Exception message contains a typo when unable to create
Solr Core
Posted by sh...@apache.org.
SOLR-12162: CorePropertiesLocator Exception message contains a typo when unable to create Solr Core
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e55b7e99
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e55b7e99
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e55b7e99
Branch: refs/heads/jira/solr-12095
Commit: e55b7e9911165fdf99682990c743e9bcd6cbd4f9
Parents: b5a3678
Author: Erick Erickson <er...@apache.org>
Authored: Thu Mar 29 18:01:38 2018 -0700
Committer: Erick Erickson <er...@apache.org>
Committed: Thu Mar 29 18:01:38 2018 -0700
----------------------------------------------------------------------
solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e55b7e99/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java b/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java
index 70acd84..3c8a40d 100644
--- a/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java
+++ b/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java
@@ -66,7 +66,7 @@ public class CorePropertiesLocator implements CoresLocator {
if (Files.exists(propertiesFile))
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Could not create a new core in " + cd.getInstanceDir()
- + "as another core is already defined there");
+ + " as another core is already defined there");
writePropertiesFile(cd, propertiesFile);
}
}
[19/34] lucene-solr:jira/solr-12095: SOLR-12168:
LIROnShardRestartTest failures
Posted by sh...@apache.org.
SOLR-12168: LIROnShardRestartTest failures
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d483108a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d483108a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d483108a
Branch: refs/heads/jira/solr-12095
Commit: d483108a1508e9a5f6324a5fe5547deb4c6a713f
Parents: 0ef68f7
Author: Cao Manh Dat <da...@apache.org>
Authored: Fri Mar 30 19:54:18 2018 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Fri Mar 30 19:54:18 2018 +0700
----------------------------------------------------------------------
.../apache/solr/cloud/LIROnShardRestartTest.java | 17 ++++++++++++++++-
1 file changed, 16 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d483108a/solr/core/src/test/org/apache/solr/cloud/LIROnShardRestartTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/LIROnShardRestartTest.java b/solr/core/src/test/org/apache/solr/cloud/LIROnShardRestartTest.java
index c83739e..31947be 100644
--- a/solr/core/src/test/org/apache/solr/cloud/LIROnShardRestartTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/LIROnShardRestartTest.java
@@ -17,6 +17,7 @@
package org.apache.solr.cloud;
+import java.lang.invoke.MethodHandles;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@@ -45,12 +46,16 @@ import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.KeeperException;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@LuceneTestCase.Nightly
@LuceneTestCase.Slow
@Deprecated
public class LIROnShardRestartTest extends SolrCloudTestCase {
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
@BeforeClass
public static void setupCluster() throws Exception {
System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");
@@ -132,6 +137,9 @@ public class LIROnShardRestartTest extends SolrCloudTestCase {
// now expire each node
for (Replica replica : docCollection.getReplicas()) {
try {
+ // todo remove the condition for skipping leader after SOLR-12166 is fixed
+ if (newLeader.getName().equals(replica.getName())) continue;
+
cluster.getZkClient().makePath("/collections/" + collection + "/leader_initiated_recovery/shard1/" + replica.getName(),
znodeData, true);
} catch (KeeperException.NodeExistsException e) {
@@ -153,7 +161,14 @@ public class LIROnShardRestartTest extends SolrCloudTestCase {
if (electionNodes.isEmpty()) break;
}
assertFalse("Timeout waiting for replicas rejoin election", timeOut.hasTimedOut());
- waitForState("Timeout waiting for active replicas", collection, clusterShape(1, 3));
+ try {
+ waitForState("Timeout waiting for active replicas", collection, clusterShape(1, 3));
+ } catch (Throwable th) {
+ String electionPath = "/collections/allReplicasInLIR/leader_elect/shard1/election/";
+ List<String> children = zkClient().getChildren(electionPath, null, true);
+ LOG.info("Election queue {}", children);
+ throw th;
+ }
assertEquals(103, cluster.getSolrClient().query(collection, new SolrQuery("*:*")).getResults().getNumFound());
[29/34] lucene-solr:jira/solr-12095: LUCENE-7642: Take the slightly
more cautious approach of introducing a plane method for functional
identicality.
Posted by sh...@apache.org.
LUCENE-7642: Take the slightly more cautious approach of introducing a plane method for functional identicality.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b4a83fff
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b4a83fff
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b4a83fff
Branch: refs/heads/jira/solr-12095
Commit: b4a83fffbdc2796b5147bda65085dc03ad1fdeab
Parents: dc9c603
Author: Karl Wright <Da...@gmail.com>
Authored: Sat Mar 31 11:42:32 2018 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Sat Mar 31 11:42:32 2018 -0400
----------------------------------------------------------------------
.../spatial3d/geom/GeoPolygonFactory.java | 2 +-
.../org/apache/lucene/spatial3d/geom/Plane.java | 40 +++++++++++++++++++-
2 files changed, 39 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4a83fff/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java
index 43ae32f..0bbae80 100755
--- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java
@@ -1239,7 +1239,7 @@ public class GeoPolygonFactory {
final Edge newLastEdge = edgeBuffer.getNext(lastEdge);
// Planes that are almost identical cannot be properly handled by the standard polygon logic. Detect this case and, if found,
// give up on the tiling -- we'll need to create a large poly instead.
- if (lastEdge.plane.isNumericallyIdentical(newLastEdge.plane) /*isNearlyIdentical(lastEdge.plane, newLastEdge.plane) */) {
+ if (lastEdge.plane.isFunctionallyIdentical(newLastEdge.plane)) {
throw new TileException("Two adjacent edge planes are effectively parallel despite filtering; give up on tiling");
}
if (Plane.arePointsCoplanar(lastEdge.startPoint, lastEdge.endPoint, newLastEdge.endPoint)) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4a83fff/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java
index f4d6103..fe3418c 100755
--- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java
@@ -2345,6 +2345,43 @@ public class Plane extends Vector {
}
/**
+ * Returns true if this plane and the other plane are functionally identical within the margin of error.
+ * Functionally identical means that the planes are so close to parallel that many aspects of planar math,
+ * like intersections, no longer have answers to within the required precision.
+ * @param p is the plane to compare against.
+ * @return true if the planes are functionally identical.
+ */
+ public boolean isFunctionallyIdentical(final Plane p) {
+ // We can get the correlation by just doing a parallel plane check. That's basically finding
+ // out if the magnitude of the cross-product is "zero".
+ final double cross1 = this.y * p.z - this.z * p.y;
+ final double cross2 = this.z * p.x - this.x * p.z;
+ final double cross3 = this.x * p.y - this.y * p.x;
+ //System.out.println("cross product magnitude = "+(cross1 * cross1 + cross2 * cross2 + cross3 * cross3));
+ // Should be MINIMUM_RESOLUTION_SQUARED, but that gives us planes that are *almost* parallel, and those are problematic too,
+ // so we have a tighter constraint on parallelism in this method.
+ if (cross1 * cross1 + cross2 * cross2 + cross3 * cross3 >= MINIMUM_RESOLUTION) {
+ return false;
+ }
+
+ // Now, see whether the parallel planes are in fact on top of one another.
+ // The math:
+ // We need a single point that fulfills:
+ // Ax + By + Cz + D = 0
+ // Pick:
+ // x0 = -(A * D) / (A^2 + B^2 + C^2)
+ // y0 = -(B * D) / (A^2 + B^2 + C^2)
+ // z0 = -(C * D) / (A^2 + B^2 + C^2)
+ // Check:
+ // A (x0) + B (y0) + C (z0) + D =? 0
+ // A (-(A * D) / (A^2 + B^2 + C^2)) + B (-(B * D) / (A^2 + B^2 + C^2)) + C (-(C * D) / (A^2 + B^2 + C^2)) + D ?= 0
+ // -D [ A^2 / (A^2 + B^2 + C^2) + B^2 / (A^2 + B^2 + C^2) + C^2 / (A^2 + B^2 + C^2)] + D ?= 0
+ // Yes.
+ final double denom = 1.0 / (p.x * p.x + p.y * p.y + p.z * p.z);
+ return evaluateIsZero(-p.x * p.D * denom, -p.y * p.D * denom, -p.z * p.D * denom);
+ }
+
+ /**
* Returns true if this plane and the other plane are identical within the margin of error.
* @param p is the plane to compare against.
* @return true if the planes are numerically identical.
@@ -2356,8 +2393,7 @@ public class Plane extends Vector {
final double cross2 = this.z * p.x - this.x * p.z;
final double cross3 = this.x * p.y - this.y * p.x;
//System.out.println("cross product magnitude = "+(cross1 * cross1 + cross2 * cross2 + cross3 * cross3));
- // Technically should be MINIMUM_RESOLUTION_SQUARED, but that gives us planes that are *almost* parallel, and those are problematic too
- if (cross1 * cross1 + cross2 * cross2 + cross3 * cross3 >= MINIMUM_RESOLUTION) {
+ if (cross1 * cross1 + cross2 * cross2 + cross3 * cross3 >= MINIMUM_RESOLUTION_SQUARED) {
return false;
}
/* Old method
[23/34] lucene-solr:jira/solr-12095: SOLR-12169: Fix
ComputePlanActionTest.testSelectedCollections fails on jenkins by
aggressively cleaning up trigger state left by other test methods in the test
setup
Posted by sh...@apache.org.
SOLR-12169: Fix ComputePlanActionTest.testSelectedCollections fails on jenkins by aggressively cleaning up trigger state left by other test methods in the test setup
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/bd85fd38
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/bd85fd38
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/bd85fd38
Branch: refs/heads/jira/solr-12095
Commit: bd85fd389f157796901c7b6a7b3fba467e6ad39e
Parents: 85decab
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Fri Mar 30 22:56:58 2018 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Fri Mar 30 22:56:58 2018 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +++
.../cloud/autoscaling/NodeAddedTrigger.java | 2 +-
.../solr/cloud/autoscaling/NodeLostTrigger.java | 2 +-
.../autoscaling/ComputePlanActionTest.java | 25 ++++++++++++++------
4 files changed, 23 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bd85fd38/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 6864e0b..e7349cf 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -137,6 +137,9 @@ Other Changes
* SOLR-12133: Fix race conditions that caused TriggerIntegrationTest.testEventQueue to fail. (Mark Miller, shalin)
+* SOLR-12169: Fix ComputePlanActionTest.testSelectedCollections fails on jenkins by aggressively cleaning up
+ trigger state left by other test methods in the test setup. (shalin)
+
================== 7.3.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bd85fd38/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
index ad89f2a..6190a49 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
@@ -53,7 +53,7 @@ public class NodeAddedTrigger extends TriggerBase {
SolrCloudManager cloudManager) {
super(TriggerEventType.NODEADDED, name, properties, loader, cloudManager);
lastLiveNodes = new HashSet<>(cloudManager.getClusterStateProvider().getLiveNodes());
- log.debug("Initial livenodes: {}", lastLiveNodes);
+ log.debug("NodeAddedTrigger {} - Initial livenodes: {}", name, lastLiveNodes);
log.debug("NodeAddedTrigger {} instantiated with properties: {}", name, properties);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bd85fd38/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
index 1e7aec5..2981a48 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
@@ -52,7 +52,7 @@ public class NodeLostTrigger extends TriggerBase {
SolrCloudManager dataProvider) {
super(TriggerEventType.NODELOST, name, properties, loader, dataProvider);
lastLiveNodes = new HashSet<>(dataProvider.getClusterStateProvider().getLiveNodes());
- log.debug("Initial livenodes: {}", lastLiveNodes);
+ log.debug("NodeLostTrigger {} - Initial livenodes: {}", name, lastLiveNodes);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bd85fd38/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
index 720cc4f..67b5fa0 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
@@ -71,6 +71,7 @@ public class ComputePlanActionTest extends SolrCloudTestCase {
private static CountDownLatch triggerFiredLatch = new CountDownLatch(1);
private static final AtomicReference<Map> actionContextPropsRef = new AtomicReference<>();
private static final AtomicReference<TriggerEvent> eventRef = new AtomicReference<>();
+ private static SolrCloudManager cloudManager;
@BeforeClass
public static void setupCluster() throws Exception {
@@ -83,10 +84,6 @@ public class ComputePlanActionTest extends SolrCloudTestCase {
public void setUp() throws Exception {
super.setUp();
- fired.set(false);
- triggerFiredLatch = new CountDownLatch(1);
- actionContextPropsRef.set(null);
-
// remove everything from autoscaling.json in ZK
zkClient().setData(ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, "{}".getBytes(UTF_8), true);
@@ -129,6 +126,20 @@ public class ComputePlanActionTest extends SolrCloudTestCase {
req = createAutoScalingRequest(SolrRequest.METHOD.POST, setClusterPreferencesCommand);
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
+
+ cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager();
+ deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH);
+ deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH);
+ deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH);
+ deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH);
+
+ fired.set(false);
+ triggerFiredLatch = new CountDownLatch(1);
+ actionContextPropsRef.set(null);
+ }
+
+ private void deleteChildrenRecursively(String path) throws Exception {
+ cloudManager.getDistribStateManager().removeRecursively(path, true, false);
}
@After
@@ -365,7 +376,7 @@ public class ComputePlanActionTest extends SolrCloudTestCase {
}
public static class AssertingTriggerAction implements TriggerAction {
- static String expectedNode;
+ static volatile String expectedNode;
@Override
public String getName() {
@@ -397,8 +408,8 @@ public class ComputePlanActionTest extends SolrCloudTestCase {
}
@Test
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12169")
public void testSelectedCollections() throws Exception {
+ log.info("Found number of jetties: {}", cluster.getJettySolrRunners().size());
AssertingTriggerAction.expectedNode = null;
// start 3 more nodes
@@ -467,7 +478,7 @@ public class ComputePlanActionTest extends SolrCloudTestCase {
Map context = actionContextPropsRef.get();
assertNotNull(context);
List<SolrRequest> operations = (List<SolrRequest>) context.get("operations");
- assertNotNull("The operations computed by ComputePlanAction should not be null" + getNodeStateProviderState() + context, operations);
+ assertNotNull("The operations computed by ComputePlanAction should not be null. " + getNodeStateProviderState() + context, operations);
assertEquals("ComputePlanAction should have computed exactly 2 operations", 2, operations.size());
SolrRequest request = operations.get(0);
SolrParams params = request.getParams();
[27/34] lucene-solr:jira/solr-12095: LUCENE-7642: Multi-pronged
approach to fixing this problem. Main fix is a better detection of
parallelness in polygon adjoining edge planes. We deliberately make parallel
determination less strict now than technical
Posted by sh...@apache.org.
LUCENE-7642: Multi-pronged approach to fixing this problem. Main fix is a better detection of parallelness in polygon adjoining edge planes. We deliberately make parallel determination less strict now than technically needed to avoid pathological cases. Other downstream changes devolve from that.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/590e6715
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/590e6715
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/590e6715
Branch: refs/heads/jira/solr-12095
Commit: 590e67158c1f5cba5da4e59e5f787cb2e3c943b3
Parents: 0ef68f7
Author: Karl Wright <Da...@gmail.com>
Authored: Sat Mar 31 10:03:49 2018 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Sat Mar 31 10:03:49 2018 -0400
----------------------------------------------------------------------
.../spatial3d/geom/GeoComplexPolygon.java | 6 +
.../spatial3d/geom/GeoConcavePolygon.java | 16 +-
.../lucene/spatial3d/geom/GeoConvexPolygon.java | 20 ++-
.../spatial3d/geom/GeoPolygonFactory.java | 7 +-
.../org/apache/lucene/spatial3d/geom/Plane.java | 20 ++-
.../lucene/spatial3d/geom/GeoPolygonTest.java | 159 +++++++++++++++----
6 files changed, 179 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/590e6715/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoComplexPolygon.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoComplexPolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoComplexPolygon.java
index c8d6435..d5d005e 100644
--- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoComplexPolygon.java
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoComplexPolygon.java
@@ -394,6 +394,12 @@ class GeoComplexPolygon extends GeoBasePolygon {
for (final GeoPoint point : notablePoints) {
xyzBounds.addPoint(point);
}
+ // If we have no bounds at all then the answer is "false"
+ if (xyzBounds.getMaximumX() == null || xyzBounds.getMinimumX() == null ||
+ xyzBounds.getMaximumY() == null || xyzBounds.getMinimumY() == null ||
+ xyzBounds.getMaximumZ() == null || xyzBounds.getMinimumZ() == null) {
+ return false;
+ }
// Figure out which tree likely works best
final double xDelta = xyzBounds.getMaximumX() - xyzBounds.getMinimumX();
final double yDelta = xyzBounds.getMaximumY() - xyzBounds.getMinimumY();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/590e6715/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java
index 683e11a..692d474 100644
--- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java
@@ -235,15 +235,19 @@ class GeoConcavePolygon extends GeoBasePolygon {
final SidedPlane edge = edges[edgeIndex];
final SidedPlane invertedEdge = invertedEdges[edgeIndex];
int bound1Index = legalIndex(edgeIndex+1);
- while (invertedEdges[legalIndex(bound1Index)].isNumericallyIdentical(invertedEdge)) {
- bound1Index++;
+ while (invertedEdges[bound1Index].isNumericallyIdentical(invertedEdge)) {
+ if (bound1Index == edgeIndex) {
+ throw new IllegalArgumentException("Constructed planes are all coplanar: "+points);
+ }
+ bound1Index = legalIndex(bound1Index + 1);
}
int bound2Index = legalIndex(edgeIndex-1);
- while (invertedEdges[legalIndex(bound2Index)].isNumericallyIdentical(invertedEdge)) {
- bound2Index--;
+ while (invertedEdges[bound2Index].isNumericallyIdentical(invertedEdge)) {
+ if (bound2Index == edgeIndex) {
+ throw new IllegalArgumentException("Constructed planes are all coplanar: "+points);
+ }
+ bound2Index = legalIndex(bound2Index - 1);
}
- bound1Index = legalIndex(bound1Index);
- bound2Index = legalIndex(bound2Index);
// Also confirm that all interior points are within the bounds
int startingIndex = bound2Index;
while (true) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/590e6715/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java
index a29dd0f..1fb2b0f 100755
--- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java
@@ -218,7 +218,6 @@ class GeoConvexPolygon extends GeoBasePolygon {
}
final GeoPoint check = points.get(endPointIndex);
final SidedPlane sp = new SidedPlane(check, start, end);
- //System.out.println("Created edge "+sp+" using start="+start+" end="+end+" check="+check);
edges[i] = sp;
notableEdgePoints[i] = new GeoPoint[]{start, end};
}
@@ -230,16 +229,20 @@ class GeoConvexPolygon extends GeoBasePolygon {
for (int edgeIndex = 0; edgeIndex < edges.length; edgeIndex++) {
final SidedPlane edge = edges[edgeIndex];
int bound1Index = legalIndex(edgeIndex+1);
- while (edges[legalIndex(bound1Index)].isNumericallyIdentical(edge)) {
- bound1Index++;
+ while (edges[bound1Index].isNumericallyIdentical(edge)) {
+ if (bound1Index == edgeIndex) {
+ throw new IllegalArgumentException("Constructed planes are all coplanar: "+points);
+ }
+ bound1Index = legalIndex(bound1Index + 1);
}
int bound2Index = legalIndex(edgeIndex-1);
// Look for bound2
- while (edges[legalIndex(bound2Index)].isNumericallyIdentical(edge)) {
- bound2Index--;
+ while (edges[bound2Index].isNumericallyIdentical(edge)) {
+ if (bound2Index == edgeIndex) {
+ throw new IllegalArgumentException("Constructed planes are all coplanar: "+points);
+ }
+ bound2Index = legalIndex(bound2Index - 1);
}
- bound1Index = legalIndex(bound1Index);
- bound2Index = legalIndex(bound2Index);
// Also confirm that all interior points are within the bounds
int startingIndex = bound2Index;
while (true) {
@@ -305,8 +308,9 @@ class GeoConvexPolygon extends GeoBasePolygon {
*@return the normalized index.
*/
protected int legalIndex(int index) {
- while (index >= points.size())
+ while (index >= points.size()) {
index -= points.size();
+ }
while (index < 0) {
index += points.size();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/590e6715/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java
index a5a4406..43ae32f 100755
--- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java
@@ -1219,7 +1219,7 @@ public class GeoPolygonFactory {
final GeoCompositePolygon rval,
final EdgeBuffer edgeBuffer,
final List<GeoPolygon> holes,
- final GeoPoint testPoint) {
+ final GeoPoint testPoint) throws TileException {
//System.out.println("Looking at edge "+currentEdge+" with startpoint "+currentEdge.startPoint+" endpoint "+currentEdge.endPoint);
@@ -1237,6 +1237,11 @@ public class GeoPolygonFactory {
break;
}
final Edge newLastEdge = edgeBuffer.getNext(lastEdge);
+ // Planes that are almost identical cannot be properly handled by the standard polygon logic. Detect this case and, if found,
+ // give up on the tiling -- we'll need to create a large poly instead.
+ if (lastEdge.plane.isNumericallyIdentical(newLastEdge.plane) /*isNearlyIdentical(lastEdge.plane, newLastEdge.plane) */) {
+ throw new TileException("Two adjacent edge planes are effectively parallel despite filtering; give up on tiling");
+ }
if (Plane.arePointsCoplanar(lastEdge.startPoint, lastEdge.endPoint, newLastEdge.endPoint)) {
break;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/590e6715/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java
index 5ecee28..f4d6103 100755
--- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java
@@ -1275,7 +1275,9 @@ public class Plane extends Vector {
// Since a==b==0, any plane including the Z axis suffices.
//System.err.println(" Perpendicular to z");
final GeoPoint[] points = findIntersections(planetModel, normalYPlane, NO_BOUNDS, NO_BOUNDS);
- boundsInfo.addZValue(points[0]);
+ if (points.length > 0) {
+ boundsInfo.addZValue(points[0]);
+ }
}
}
@@ -2348,15 +2350,25 @@ public class Plane extends Vector {
* @return true if the planes are numerically identical.
*/
public boolean isNumericallyIdentical(final Plane p) {
- // We can get the correlation by just doing a parallel plane check. If that passes, then compute a point on the plane
- // (using D) and see if it also on the other plane.
+ // We can get the correlation by just doing a parallel plane check. That's basically finding
+ // out if the magnitude of the cross-product is "zero".
+ final double cross1 = this.y * p.z - this.z * p.y;
+ final double cross2 = this.z * p.x - this.x * p.z;
+ final double cross3 = this.x * p.y - this.y * p.x;
+ //System.out.println("cross product magnitude = "+(cross1 * cross1 + cross2 * cross2 + cross3 * cross3));
+ // Technically should be MINIMUM_RESOLUTION_SQUARED, but that gives us planes that are *almost* parallel, and those are problematic too
+ if (cross1 * cross1 + cross2 * cross2 + cross3 * cross3 >= MINIMUM_RESOLUTION) {
+ return false;
+ }
+ /* Old method
if (Math.abs(this.y * p.z - this.z * p.y) >= MINIMUM_RESOLUTION)
return false;
if (Math.abs(this.z * p.x - this.x * p.z) >= MINIMUM_RESOLUTION)
return false;
if (Math.abs(this.x * p.y - this.y * p.x) >= MINIMUM_RESOLUTION)
return false;
-
+ */
+
// Now, see whether the parallel planes are in fact on top of one another.
// The math:
// We need a single point that fulfills:
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/590e6715/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
index e0b584b..f577901 100755
--- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
+++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
@@ -756,19 +756,9 @@ shape:
final GeoPoint point = new GeoPoint(PlanetModel.WGS84, -0.41518838180529244, 3.141592653589793);
final GeoPoint encodedPoint = new GeoPoint(-0.9155623168963972, 2.3309121299774915E-10, -0.40359240449795253);
- //System.out.println("point = "+point);
- //System.out.println("encodedPoint = "+encodedPoint);
- assertTrue(p.isWithin(point));
- assertTrue(solid.isWithin(point));
+ assertTrue(p.isWithin(point)?solid.isWithin(point):true);
- //System.out.println("bounds1 = "+bounds1);
- //System.out.println("bounds2 = "+bounds2);
- //assertTrue(poly1.isWithin(point));
- //assertTrue(poly2.isWithin(point));
- //assertTrue(solid2.isWithin(point));
-
- //assertTrue(poly2.isWithin(encodedPoint));
}
@Test
@@ -967,8 +957,10 @@ shape:
poly1List.add(new GeoPoint(PlanetModel.WGS84, 1.079437865394857, -1.720224083538152E-11));
poly1List.add(new GeoPoint(PlanetModel.WGS84, -1.5707963267948966, 0.017453291479645996));
poly1List.add(new GeoPoint(PlanetModel.WGS84, 0.017453291479645996, 2.4457272005608357E-47));
+
+ final GeoPolygonFactory.PolygonDescription pd = new GeoPolygonFactory.PolygonDescription(poly1List);
- final GeoConvexPolygon poly1 = new GeoConvexPolygon(PlanetModel.WGS84, poly1List);
+ final GeoPolygon poly1 = GeoPolygonFactory.makeGeoPolygon(PlanetModel.WGS84, pd);
/*
[junit4] 1> unquantized=[lat=-1.5316724989005415, lon=3.141592653589793([X=-0.03902652216795768, Y=4.779370545484258E-18, Z=-0.9970038705813589])]
@@ -977,17 +969,12 @@ shape:
final GeoPoint point = new GeoPoint(PlanetModel.WGS84, -1.5316724989005415, 3.141592653589793);
- assertTrue(poly1.isWithin(point));
-
final XYZBounds actualBounds1 = new XYZBounds();
poly1.getBounds(actualBounds1);
- final XYZSolid solid = XYZSolidFactory.makeXYZSolid(PlanetModel.WGS84,
- actualBounds1.getMinimumX(), actualBounds1.getMaximumX(),
- actualBounds1.getMinimumY(), actualBounds1.getMaximumY(),
- actualBounds1.getMinimumZ(), actualBounds1.getMaximumZ());
+ final XYZSolid solid = XYZSolidFactory.makeXYZSolid(PlanetModel.WGS84, actualBounds1);
- assertTrue(solid.isWithin(point));
+ assertTrue(poly1.isWithin(point)?solid.isWithin(point):true);
}
@Test
@@ -1283,10 +1270,6 @@ shape:
points.add(new GeoPoint(PlanetModel.WGS84, 0.2839194570254642, -1.2434404554202965));
GeoPolygonFactory.PolygonDescription pd = new GeoPolygonFactory.PolygonDescription(points);
- for (int i = 0; i < points.size(); i++) {
- System.out.println("Point "+i+": "+points.get(i));
- }
-
final GeoPoint unquantized = new GeoPoint(PlanetModel.WGS84, 2.4457272005608357E-47, -3.1404077424936307);
final GeoPoint quantized = new GeoPoint(-1.0011181510675629, -0.001186236379718708, 2.3309121299774915E-10);
@@ -1300,27 +1283,20 @@ shape:
// Construct a standard polygon first to see what that does. This winds up being a large polygon under the covers.
GeoPolygon standard = GeoPolygonFactory.makeGeoPolygon(PlanetModel.WGS84, pd);
- System.out.println("Shape = "+standard);
-
// This should be true, by inspection, but is false. That's the cause for the failure.
assertTrue(standard.isWithin(negativeX));
- System.out.println("Negative x pole in set? "+standard.isWithin(negativeX));
- System.out.println("Test point in set? "+standard.isWithin(testPoint));
assertTrue(standard.isWithin(testPoint));
// This is in-set because it's on an edge
- System.out.println("North pole in set? "+standard.isWithin(northPole));
assertTrue(standard.isWithin(northPole));
// This is in-set
- System.out.println("Plus-Y pole in set? "+standard.isWithin(positiveY));
assertTrue(standard.isWithin(positiveY));
final XYZBounds standardBounds = new XYZBounds();
standard.getBounds(standardBounds);
- System.out.println("Bounds = "+standardBounds);
final XYZSolid standardSolid = XYZSolidFactory.makeXYZSolid(PlanetModel.WGS84, standardBounds);
// If within shape, should be within bounds
@@ -1329,4 +1305,127 @@ shape:
}
+ @Test
+ public void testLUCENE7642() {
+ // Construct XYZ solid
+ final XYZSolid solid = XYZSolidFactory.makeXYZSolid(PlanetModel.WGS84,
+ 0.1845405855034623, 0.2730694323646922,
+ -1.398547277986495E-9, 0.020766291030223535,
+ 0.7703937553371503, 0.9977622932859774);
+
+ /*
+ [junit4] 1> individual planes
+ [junit4] 1> notableMinXPoints=[
+ [X=0.1845405855034623, Y=-1.398547277986495E-9, Z=0.9806642352600131],
+ [X=0.1845405855034623, Y=0.020766291030223535, Z=0.9804458120424796]]
+ notableMaxXPoints=[
+ [X=0.2730694323646922, Y=-1.398547277986495E-9, Z=0.959928047174481],
+ [X=0.2730694323646922, Y=0.020766291030223535, Z=0.9597049045335464]]
+ notableMinYPoints=[
+ [X=0.1845405855034623, Y=-1.398547277986495E-9, Z=0.9806642352600131],
+ [X=0.2730694323646922, Y=-1.398547277986495E-9, Z=0.959928047174481]]
+ notableMaxYPoints=[
+ [X=0.1845405855034623, Y=0.020766291030223535, Z=0.9804458120424796],
+ [X=0.2730694323646922, Y=0.020766291030223535, Z=0.9597049045335464]]
+ notableMinZPoints=[]
+ notableMaxZPoints=[]
+
+ [junit4] 1> All edge points=[
+ [X=0.1845405855034623, Y=-1.398547277986495E-9, Z=0.9806642352600131],
+ [X=0.1845405855034623, Y=0.020766291030223535, Z=0.9804458120424796],
+ [X=0.2730694323646922, Y=-1.398547277986495E-9, Z=0.959928047174481],
+ [X=0.2730694323646922, Y=0.020766291030223535, Z=0.9597049045335464]]
+
+ */
+
+ final GeoPoint edge1 = new GeoPoint(0.1845405855034623, -1.398547277986495E-9, 0.9806642352600131);
+ final GeoPoint edge2 = new GeoPoint(0.1845405855034623, 0.020766291030223535, 0.9804458120424796);
+ final GeoPoint edge3 = new GeoPoint(0.2730694323646922, -1.398547277986495E-9, 0.959928047174481);
+ final GeoPoint edge4 = new GeoPoint(0.2730694323646922, 0.020766291030223535, 0.9597049045335464);
+
+ // The above says that none of these intersect the surface: minZmaxX, minZminX, minZmaxY, minZminY, or
+ // maxZmaxX, maxZminX, maxZmaxY, maxZminY.
+
+ // So what about minZ and maxZ all by themselves?
+ //
+ // [junit4] 1> Outside world: minXminYminZ=false minXminYmaxZ=true minXmaxYminZ=false minXmaxYmaxZ=true maxXminYminZ=false
+ // maxXminYmaxZ=true maxXmaxYminZ=false maxXmaxYmaxZ=true
+ //
+ // So the minz plane does not intersect the world because it's all inside. The maxZ plane is all outside but may intersect the world still.
+ // But it doesn't because it's too far north.
+ // So it looks like these are our edge points, and they are correct.
+
+ /*
+ GeoConvexPolygon: {planetmodel=PlanetModel.WGS84, points=[
+ [lat=-1.2267098126036888, lon=3.141592653589793([X=-0.33671029227864785, Y=4.123511816790159E-17, Z=-0.9396354281810864])],
+ [lat=0.2892272352400239, lon=0.017453291479645996([X=0.9591279281485559, Y=0.01674163926221766, Z=0.28545251693892165])],
+ [lat=-1.5707963267948966, lon=1.6247683074702402E-201([X=6.109531986173988E-17, Y=9.926573944611206E-218, Z=-0.997762292022105])]], internalEdges={2}},
+ GeoConvexPolygon: {planetmodel=PlanetModel.WGS84, points=[
+ [lat=-1.2267098126036888, lon=3.141592653589793([X=-0.33671029227864785, Y=4.123511816790159E-17, Z=-0.9396354281810864])],
+ [lat=-1.5707963267948966, lon=1.6247683074702402E-201([X=6.109531986173988E-17, Y=9.926573944611206E-218, Z=-0.997762292022105])],
+ [lat=0.6723906085905078, lon=-3.0261581679831E-12([X=0.7821883235431606, Y=-2.367025584191143E-12, Z=0.6227413298552851])]], internalEdges={0}}]}
+ */
+ final List<GeoPoint> points = new ArrayList<>();
+ points.add(new GeoPoint(PlanetModel.WGS84, -1.2267098126036888, 3.141592653589793));
+ points.add(new GeoPoint(PlanetModel.WGS84, 0.2892272352400239, 0.017453291479645996));
+ points.add(new GeoPoint(PlanetModel.WGS84, -1.5707963267948966, 1.6247683074702402E-201));
+ points.add(new GeoPoint(PlanetModel.WGS84, 0.6723906085905078, -3.0261581679831E-12));
+
+ final GeoPolygonFactory.PolygonDescription pd = new GeoPolygonFactory.PolygonDescription(points);
+ final GeoPolygon shape = GeoPolygonFactory.makeGeoPolygon(PlanetModel.WGS84, pd);
+ final List<GeoPolygonFactory.PolygonDescription> pdList = new ArrayList<>(1);
+ pdList.add(pd);
+ final GeoPolygon largeShape = GeoPolygonFactory.makeLargeGeoPolygon(PlanetModel. WGS84, pdList);
+
+ /* This is the output:
+ [junit4] 1> shape = GeoCompositePolygon: {[
+ GeoConvexPolygon: {planetmodel=PlanetModel.WGS84, points=[
+ [lat=-1.2267098126036888, lon=3.141592653589793([X=-0.33671029227864785, Y=4.123511816790159E-17, Z=-0.9396354281810864])],
+ [lat=0.2892272352400239, lon=0.017453291479645996([X=0.9591279281485559, Y=0.01674163926221766, Z=0.28545251693892165])],
+ [lat=-1.5707963267948966, lon=1.6247683074702402E-201([X=6.109531986173988E-17, Y=9.926573944611206E-218, Z=-0.997762292022105])]], internalEdges={2}},
+ GeoConvexPolygon: {planetmodel=PlanetModel.WGS84, points=[
+ [lat=-1.2267098126036888, lon=3.141592653589793([X=-0.33671029227864785, Y=4.123511816790159E-17, Z=-0.9396354281810864])],
+ [lat=-1.5707963267948966, lon=1.6247683074702402E-201([X=6.109531986173988E-17, Y=9.926573944611206E-218, Z=-0.997762292022105])],
+ [lat=0.6723906085905078, lon=-3.0261581679831E-12([X=0.7821883235431606, Y=-2.367025584191143E-12, Z=0.6227413298552851])]], internalEdges={0}}]}
+ */
+
+ final GeoPoint quantized = new GeoPoint(0.24162356556559528, 2.3309121299774915E-10, 0.9682657049003708);
+ final GeoPoint unquantized = new GeoPoint(PlanetModel.WGS84, 1.3262481806651818, 2.4457272005608357E-47);
+
+ // This passes; the point is definitely within the solid.
+ assertTrue(solid.isWithin(unquantized));
+
+ // This passes, so I assume that this is the correct response.
+ assertFalse(largeShape.isWithin(unquantized));
+ // This fails because the point is within the shape but apparently shouldn't be.
+ // Instrumenting isWithin finds that the point is on three edge planes somehow:
+ /*
+ [junit4] 1> localIsWithin start for point [0.2416235655409041,5.90945326539883E-48,0.9682657046994557]
+ [junit4] 1> For edge [A=-1.224646799147353E-16, B=-1.0, C=-7.498798913309287E-33, D=0.0, side=1.0] the point evaluation is -2.959035261382389E-17
+ [junit4] 1> For edge [A=-3.0261581679831E-12, B=-0.9999999999999999, C=-1.8529874570670608E-28, D=0.0, side=1.0] the point evaluation is -7.31191126438807E-13
+ [junit4] 1> For edge [A=4.234084035470679E-12, B=1.0, C=-1.5172037954732973E-12, D=0.0, side=1.0] the point evaluation is -4.460019207463956E-13
+ */
+ // These are too close to parallel. The only solution is to prevent the poly from being created. Let's see if Geo3d thinks they are parallel.
+
+ final Plane p1 = new Plane(-1.224646799147353E-16, -1.0, -7.498798913309287E-33, 0.0);
+ final Plane p2 = new Plane(-3.0261581679831E-12, -0.9999999999999999, -1.8529874570670608E-28, 0.0);
+ final Plane p3 = new Plane(4.234084035470679E-12, 1.0, -1.5172037954732973E-12, 0.0);
+
+ assertFalse(shape.isWithin(unquantized));
+
+ // This point is indeed outside the shape but it doesn't matter
+ assertFalse(shape.isWithin(quantized));
+
+ // Sanity check with different poly implementation
+ assertTrue(shape.isWithin(edge1) == largeShape.isWithin(edge1));
+ assertTrue(shape.isWithin(edge2) == largeShape.isWithin(edge2));
+ assertTrue(shape.isWithin(edge3) == largeShape.isWithin(edge3));
+ assertTrue(shape.isWithin(edge4) == largeShape.isWithin(edge4));
+
+ // Verify both shapes give the same relationship
+ int intersection = solid.getRelationship(shape);
+ int largeIntersection = solid.getRelationship(largeShape);
+ assertTrue(intersection == largeIntersection);
+ }
+
}
[07/34] lucene-solr:jira/solr-12095: SOLR-12028: BadApple and
AwaitsFix annotations usage
Posted by sh...@apache.org.
SOLR-12028: BadApple and AwaitsFix annotations usage
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/23707314
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/23707314
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/23707314
Branch: refs/heads/jira/solr-12095
Commit: 23707314dd7fa67c7dd089d8fb6c1bece4817408
Parents: e55b7e9
Author: Erick Erickson <er...@apache.org>
Authored: Thu Mar 29 21:41:03 2018 -0700
Committer: Erick Erickson <er...@apache.org>
Committed: Thu Mar 29 21:41:03 2018 -0700
----------------------------------------------------------------------
.../src/test/org/apache/solr/cloud/DocValuesNotIndexedTest.java | 1 +
.../test/org/apache/solr/cloud/RestartWhileUpdatingTest.java | 1 +
.../src/test/org/apache/solr/cloud/TestCloudConsistency.java | 2 ++
solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java | 5 ++++-
.../core/src/test/org/apache/solr/cloud/TestSegmentSorting.java | 2 ++
.../cloud/api/collections/CollectionsAPIDistributedZkTest.java | 1 +
.../apache/solr/cloud/autoscaling/TriggerIntegrationTest.java | 2 ++
.../solr/cloud/autoscaling/sim/TestTriggerIntegration.java | 1 +
.../org/apache/solr/cloud/hdfs/HdfsBasicDistributedZk2Test.java | 2 ++
.../solr/handler/admin/AutoscalingHistoryHandlerTest.java | 3 ++-
10 files changed, 18 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23707314/solr/core/src/test/org/apache/solr/cloud/DocValuesNotIndexedTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/DocValuesNotIndexedTest.java b/solr/core/src/test/org/apache/solr/cloud/DocValuesNotIndexedTest.java
index 058cd0e..13ecea2 100644
--- a/solr/core/src/test/org/apache/solr/cloud/DocValuesNotIndexedTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/DocValuesNotIndexedTest.java
@@ -224,6 +224,7 @@ public class DocValuesNotIndexedTest extends SolrCloudTestCase {
// We should be able to sort thing with missing first/last and that are _NOT_ present at all on one server.
@Test
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 26-Mar-2018
public void testGroupingSorting() throws IOException, SolrServerException {
CloudSolrClient client = cluster.getSolrClient();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23707314/solr/core/src/test/org/apache/solr/cloud/RestartWhileUpdatingTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/RestartWhileUpdatingTest.java b/solr/core/src/test/org/apache/solr/cloud/RestartWhileUpdatingTest.java
index 095d3ed..5a4734b 100644
--- a/solr/core/src/test/org/apache/solr/cloud/RestartWhileUpdatingTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/RestartWhileUpdatingTest.java
@@ -76,6 +76,7 @@ public class RestartWhileUpdatingTest extends AbstractFullDistribZkTestBase {
}
@Test
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 26-Mar-2018
public void test() throws Exception {
handle.clear();
handle.put("timestamp", SKIPVAL);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23707314/solr/core/src/test/org/apache/solr/cloud/TestCloudConsistency.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudConsistency.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudConsistency.java
index b70a5a5..489e7c2 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestCloudConsistency.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudConsistency.java
@@ -88,11 +88,13 @@ public class TestCloudConsistency extends SolrCloudTestCase {
}
@Test
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 26-Mar-2018
public void testOutOfSyncReplicasCannotBecomeLeader() throws Exception {
testOutOfSyncReplicasCannotBecomeLeader(false);
}
@Test
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 26-Mar-2018
public void testOutOfSyncReplicasCannotBecomeLeaderAfterRestart() throws Exception {
testOutOfSyncReplicasCannotBecomeLeader(true);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23707314/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java b/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java
index c10ec0f..3de8c02 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java
@@ -60,6 +60,7 @@ import org.apache.zookeeper.KeeperException;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Ignore;
+import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -300,7 +301,9 @@ public class TestPullReplica extends SolrCloudTestCase {
public void testRemoveAllWriterReplicas() throws Exception {
doTestNoLeader(true);
}
-
+
+ @Test
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 26-Mar-2018
public void testKillLeader() throws Exception {
doTestNoLeader(false);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23707314/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java b/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java
index 1f4433b..46eb266 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java
@@ -122,6 +122,8 @@ public class TestSegmentSorting extends SolrCloudTestCase {
* In this situation, the updates should *NOT* be done inplace, because that would
* break the index sorting
*/
+ @Test
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 26-Mar-2018
public void testAtomicUpdateOfSegmentSortField() throws Exception {
final CloudSolrClient cloudSolrClient = cluster.getSolrClient();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23707314/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java
index 2901961..f970d4b 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java
@@ -483,6 +483,7 @@ public class CollectionsAPIDistributedZkTest extends SolrCloudTestCase {
}
@Test
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 26-Mar-2018
public void testCollectionReload() throws Exception {
final String collectionName = "reloaded_collection";
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23707314/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
index f29280b..15f65e7 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
@@ -1659,6 +1659,8 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
assertEquals(5, docCollection.getReplicas().size());
}
+ @Test
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 26-Mar-2018
public void testScheduledTrigger() throws Exception {
CloudSolrClient solrClient = cluster.getSolrClient();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23707314/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestTriggerIntegration.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestTriggerIntegration.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestTriggerIntegration.java
index 974e672..fc06b50 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestTriggerIntegration.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestTriggerIntegration.java
@@ -436,6 +436,7 @@ public class TestTriggerIntegration extends SimSolrCloudTestCase {
}
@Test
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 26-Mar-2018
public void testNodeLostTrigger() throws Exception {
SolrClient solrClient = cluster.simGetSolrClient();
String setTriggerCommand = "{" +
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23707314/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZk2Test.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZk2Test.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZk2Test.java
index a0b4a42..b1cc2fe 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZk2Test.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZk2Test.java
@@ -19,6 +19,7 @@ package org.apache.solr.cloud.hdfs;
import java.io.IOException;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.cloud.BasicDistributedZk2Test;
import org.apache.solr.util.BadHdfsThreadsFilter;
@@ -33,6 +34,7 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
@ThreadLeakFilters(defaultFilters = true, filters = {
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
+@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 26-Mar-2018
public class HdfsBasicDistributedZk2Test extends BasicDistributedZk2Test {
private static MiniDFSCluster dfsCluster;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/23707314/solr/core/src/test/org/apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java
index 1133684..9597ac7 100644
--- a/solr/core/src/test/org/apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java
@@ -242,7 +242,8 @@ public class AutoscalingHistoryHandlerTest extends SolrCloudTestCase {
}
@Test
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
+ // commented out 26-Mar-2018
+ //@BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
public void testHistory() throws Exception {
waitForState("Timed out wait for collection be active", COLL_NAME,
clusterShape(1, 3));
[05/34] lucene-solr:jira/solr-12095: SOLR-12136: fix bad links
breaking the build
Posted by sh...@apache.org.
SOLR-12136: fix bad links breaking the build
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b5a36785
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b5a36785
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b5a36785
Branch: refs/heads/jira/solr-12095
Commit: b5a36785738a299cb00933c2d55c587917a2d9ab
Parents: 1ce7253
Author: Cassandra Targett <ct...@apache.org>
Authored: Thu Mar 29 14:18:45 2018 -0500
Committer: Cassandra Targett <ct...@apache.org>
Committed: Thu Mar 29 14:18:45 2018 -0500
----------------------------------------------------------------------
solr/solr-ref-guide/src/highlighting.adoc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b5a36785/solr/solr-ref-guide/src/highlighting.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/highlighting.adoc b/solr/solr-ref-guide/src/highlighting.adoc
index b6c9bf2..1990c6c 100644
--- a/solr/solr-ref-guide/src/highlighting.adoc
+++ b/solr/solr-ref-guide/src/highlighting.adoc
@@ -42,7 +42,7 @@ When using `*`, consider adding `hl.requireFieldMatch=true`.
+
Note that the field(s) listed here ought to have compatible text-analysis (defined in the schema) with field(s) referenced in the query to be highlighted.
It may be necessary to modify `hl.q` and `hl.qparser` and/or modify the text analysis.
-The following example uses the <<local-parameters-in-queries.adoc,local-params>> syntax and <<the-extended-dismax-query-parser.adoc,the edismax parser>> to highlight fields in `hl.fl`:
+The following example uses the <<local-parameters-in-queries.adoc#local-parameters-in-queries,local-params>> syntax and <<the-extended-dismax-query-parser.adoc#the-extended-dismax-query-parser,the edismax parser>> to highlight fields in `hl.fl`:
`&hl.fl=field1 field2&hl.q={!edismax qf=$hl.fl v=$q}&hl.qparser=lucene&hl.requireFieldMatch=true` (along with other applicable parameters, of course).
+
The default is the value of the `df` parameter which in turn has no default.
[08/34] lucene-solr:jira/solr-12095: SOLR-12152: Split up
TriggerIntegrationTest into multiple tests to isolate and increase
reliability
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ed9e5eb7/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
index 15f65e7..f536633 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
@@ -19,52 +19,34 @@ package org.apache.solr.cloud.autoscaling;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Date;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.SortedSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReentrantLock;
-import com.google.common.util.concurrent.AtomicDouble;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
-import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
+import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage;
-import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.cloud.Overseer;
import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.LiveNodesListener;
-import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.params.AutoScalingParams;
-import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.TimeSource;
import org.apache.solr.common.util.Utils;
-import org.apache.solr.core.SolrResourceLoader;
-import org.apache.solr.metrics.SolrCoreMetricManager;
import org.apache.solr.util.LogLevel;
-import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.data.Stat;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -73,7 +55,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.createAutoScalingRequest;
-import static org.apache.solr.cloud.autoscaling.ScheduledTriggers.DEFAULT_SCHEDULED_TRIGGER_DELAY_SECONDS;
import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH;
/**
@@ -92,20 +73,18 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
private static CountDownLatch actionCompleted;
private static AtomicBoolean triggerFired;
private static Set<TriggerEvent> events = ConcurrentHashMap.newKeySet();
- private static ZkStateReader zkStateReader;
private static SolrCloudManager cloudManager;
// use the same time source as triggers use
- private static final TimeSource timeSource = TimeSource.CURRENT_TIME;
+ static final TimeSource timeSource = TimeSource.CURRENT_TIME;
- private static final long WAIT_FOR_DELTA_NANOS = TimeUnit.MILLISECONDS.toNanos(5);
+ static final long WAIT_FOR_DELTA_NANOS = TimeUnit.MILLISECONDS.toNanos(5);
@BeforeClass
public static void setupCluster() throws Exception {
configureCluster(2)
.addConfig("conf", configset("cloud-minimal"))
.configure();
- zkStateReader = cluster.getSolrClient().getZkStateReader();
// disable .scheduled_maintenance
String suspendTriggerCommand = "{" +
"'suspend-trigger' : {'name' : '.scheduled_maintenance'}" +
@@ -344,250 +323,6 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
}
@Test
- public void testNodeLostTriggerRestoreState() throws Exception {
- // for this test we want to update the trigger so we must assert that the actions were created twice
- TriggerIntegrationTest.actionInitCalled = new CountDownLatch(2);
-
- // start a new node
- JettySolrRunner newNode = cluster.startJettySolrRunner();
- String nodeName = newNode.getNodeName();
-
- CloudSolrClient solrClient = cluster.getSolrClient();
- waitForSeconds = 5;
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_lost_restore_trigger'," +
- "'event' : 'nodeLost'," +
- "'waitFor' : '5s'," + // should be enough for us to update the trigger
- "'enabled' : true," +
- "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
- "}}";
- SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- TimeOut timeOut = new TimeOut(2, TimeUnit.SECONDS, cloudManager.getTimeSource());
- while (actionInitCalled.getCount() == 0 && !timeOut.hasTimedOut()) {
- Thread.sleep(200);
- }
- assertTrue("The action specified in node_lost_restore_trigger was not instantiated even after 2 seconds", actionInitCalled.getCount() > 0);
-
- List<JettySolrRunner> jettySolrRunners = cluster.getJettySolrRunners();
- int index = -1;
- for (int i = 0; i < jettySolrRunners.size(); i++) {
- JettySolrRunner runner = jettySolrRunners.get(i);
- if (runner == newNode) index = i;
- }
- assertFalse(index == -1);
- cluster.stopJettySolrRunner(index);
-
- // ensure that the old trigger sees the stopped node, todo find a better way to do this
- Thread.sleep(500 + TimeUnit.SECONDS.toMillis(DEFAULT_SCHEDULED_TRIGGER_DELAY_SECONDS));
-
- waitForSeconds = 0;
- setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_lost_restore_trigger'," +
- "'event' : 'nodeLost'," +
- "'waitFor' : '0s'," + // update a property so that it replaces the old trigger, also we want it to fire immediately
- "'enabled' : true," +
- "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
- "}}";
- req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- // wait until the second instance of action is created
- if (!actionInitCalled.await(3, TimeUnit.SECONDS)) {
- fail("Two TriggerAction instances should have been created by now");
- }
-
- boolean await = triggerFiredLatch.await(5, TimeUnit.SECONDS);
- assertTrue("The trigger did not fire at all", await);
- assertTrue(triggerFired.get());
- NodeLostTrigger.NodeLostEvent nodeLostEvent = (NodeLostTrigger.NodeLostEvent) events.iterator().next();
- assertNotNull(nodeLostEvent);
- List<String> nodeNames = (List<String>)nodeLostEvent.getProperty(TriggerEvent.NODE_NAMES);
- assertTrue(nodeNames.contains(nodeName));
- }
-
- @Test
- public void testNodeAddedTriggerRestoreState() throws Exception {
- // for this test we want to update the trigger so we must assert that the actions were created twice
- TriggerIntegrationTest.actionInitCalled = new CountDownLatch(2);
-
- CloudSolrClient solrClient = cluster.getSolrClient();
- waitForSeconds = 5;
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_added_restore_trigger'," +
- "'event' : 'nodeAdded'," +
- "'waitFor' : '5s'," + // should be enough for us to update the trigger
- "'enabled' : true," +
- "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
- "}}";
- SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- TimeOut timeOut = new TimeOut(2, TimeUnit.SECONDS, cloudManager.getTimeSource());
- while (actionInitCalled.getCount() == 0 && !timeOut.hasTimedOut()) {
- Thread.sleep(200);
- }
- assertTrue("The action specified in node_added_restore_trigger was not instantiated even after 2 seconds", actionInitCalled.getCount() > 0);
-
- // start a new node
- JettySolrRunner newNode = cluster.startJettySolrRunner();
-
- // ensure that the old trigger sees the new node, todo find a better way to do this
- Thread.sleep(500 + TimeUnit.SECONDS.toMillis(DEFAULT_SCHEDULED_TRIGGER_DELAY_SECONDS));
-
- waitForSeconds = 0;
- setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_added_restore_trigger'," +
- "'event' : 'nodeAdded'," +
- "'waitFor' : '0s'," + // update a property so that it replaces the old trigger, also we want it to fire immediately
- "'enabled' : true," +
- "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
- "}}";
- req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- // wait until the second instance of action is created
- if (!actionInitCalled.await(3, TimeUnit.SECONDS)) {
- fail("Two TriggerAction instances should have been created by now");
- }
-
- boolean await = triggerFiredLatch.await(5, TimeUnit.SECONDS);
- assertTrue("The trigger did not fire at all", await);
- assertTrue(triggerFired.get());
- NodeAddedTrigger.NodeAddedEvent nodeAddedEvent = (NodeAddedTrigger.NodeAddedEvent) events.iterator().next();
- assertNotNull(nodeAddedEvent);
- List<String> nodeNames = (List<String>)nodeAddedEvent.getProperty(TriggerEvent.NODE_NAMES);
- assertTrue(nodeNames.contains(newNode.getNodeName()));
- }
-
- @Test
- public void testNodeAddedTrigger() throws Exception {
- CloudSolrClient solrClient = cluster.getSolrClient();
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_added_trigger'," +
- "'event' : 'nodeAdded'," +
- "'waitFor' : '" + waitForSeconds + "s'," +
- "'enabled' : true," +
- "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
- "}}";
- SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- if (!actionInitCalled.await(3, TimeUnit.SECONDS)) {
- fail("The TriggerAction should have been created by now");
- }
-
- JettySolrRunner newNode = cluster.startJettySolrRunner();
- boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
- assertTrue("The trigger did not fire at all", await);
- assertTrue(triggerFired.get());
- NodeAddedTrigger.NodeAddedEvent nodeAddedEvent = (NodeAddedTrigger.NodeAddedEvent) events.iterator().next();
- assertNotNull(nodeAddedEvent);
- List<String> nodeNames = (List<String>)nodeAddedEvent.getProperty(TriggerEvent.NODE_NAMES);
- assertTrue(nodeNames.contains(newNode.getNodeName()));
-
- // reset
- actionConstructorCalled = new CountDownLatch(1);
- actionInitCalled = new CountDownLatch(1);
-
- // update the trigger with exactly the same data
- setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_added_trigger'," +
- "'event' : 'nodeAdded'," +
- "'waitFor' : '" + waitForSeconds + "s'," +
- "'enabled' : true," +
- "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
- "}}";
- req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- // this should be a no-op so the action should have been created but init should not be called
- if (!actionConstructorCalled.await(3, TimeUnit.SECONDS)) {
- fail("The TriggerAction should have been created by now");
- }
-
- assertFalse(actionInitCalled.await(2, TimeUnit.SECONDS));
- }
-
- @Test
- public void testNodeLostTrigger() throws Exception {
- CloudSolrClient solrClient = cluster.getSolrClient();
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_lost_trigger'," +
- "'event' : 'nodeLost'," +
- "'waitFor' : '" + waitForSeconds + "s'," +
- "'enabled' : true," +
- "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
- "}}";
- NamedList<Object> overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
- String overseerLeader = (String) overSeerStatus.get("leader");
- int nonOverseerLeaderIndex = 0;
- for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
- JettySolrRunner jetty = cluster.getJettySolrRunner(i);
- if (!jetty.getNodeName().equals(overseerLeader)) {
- nonOverseerLeaderIndex = i;
- }
- }
- SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- if (!actionInitCalled.await(3, TimeUnit.SECONDS)) {
- fail("The TriggerAction should have been created by now");
- }
-
- triggerFired.set(false);
- triggerFiredLatch = new CountDownLatch(1);
- String lostNodeName = cluster.getJettySolrRunner(nonOverseerLeaderIndex).getNodeName();
- cluster.stopJettySolrRunner(nonOverseerLeaderIndex);
- boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
- assertTrue("The trigger did not fire at all", await);
- assertTrue(triggerFired.get());
- NodeLostTrigger.NodeLostEvent nodeLostEvent = (NodeLostTrigger.NodeLostEvent) events.iterator().next();
- assertNotNull(nodeLostEvent);
- List<String> nodeNames = (List<String>)nodeLostEvent.getProperty(TriggerEvent.NODE_NAMES);
- assertTrue(nodeNames.contains(lostNodeName));
-
- // reset
- actionConstructorCalled = new CountDownLatch(1);
- actionInitCalled = new CountDownLatch(1);
-
- // update the trigger with exactly the same data
- setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_lost_trigger'," +
- "'event' : 'nodeLost'," +
- "'waitFor' : '" + waitForSeconds + "s'," +
- "'enabled' : true," +
- "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
- "}}";
- req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- // this should be a no-op so the action should have been created but init should not be called
- if (!actionConstructorCalled.await(3, TimeUnit.SECONDS)) {
- fail("The TriggerAction should have been created by now");
- }
-
- assertFalse(actionInitCalled.await(2, TimeUnit.SECONDS));
- }
-
- @Test
public void testContinueTriggersOnOverseerRestart() throws Exception {
CollectionAdminRequest.OverseerStatus status = new CollectionAdminRequest.OverseerStatus();
CloudSolrClient solrClient = cluster.getSolrClient();
@@ -814,194 +549,6 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
assertTrue(triggerFired.get());
}
- private static class TestLiveNodesListener implements LiveNodesListener {
- Set<String> lostNodes = new HashSet<>();
- Set<String> addedNodes = new HashSet<>();
- CountDownLatch onChangeLatch = new CountDownLatch(1);
-
- public void reset() {
- lostNodes.clear();
- addedNodes.clear();
- onChangeLatch = new CountDownLatch(1);
- }
-
- @Override
- public void onChange(SortedSet<String> oldLiveNodes, SortedSet<String> newLiveNodes) {
- onChangeLatch.countDown();
- Set<String> old = new HashSet<>(oldLiveNodes);
- old.removeAll(newLiveNodes);
- if (!old.isEmpty()) {
- lostNodes.addAll(old);
- }
- newLiveNodes.removeAll(oldLiveNodes);
- if (!newLiveNodes.isEmpty()) {
- addedNodes.addAll(newLiveNodes);
- }
- }
- }
-
- private TestLiveNodesListener registerLiveNodesListener() {
- TestLiveNodesListener listener = new TestLiveNodesListener();
- zkStateReader.registerLiveNodesListener(listener);
- return listener;
- }
-
- public static class TestEventMarkerAction extends TriggerActionBase {
-
- public TestEventMarkerAction() {
- actionConstructorCalled.countDown();
- }
-
- @Override
- public void process(TriggerEvent event, ActionContext actionContext) {
- boolean locked = lock.tryLock();
- if (!locked) {
- log.info("We should never have a tryLock fail because actions are never supposed to be executed concurrently");
- return;
- }
- try {
- events.add(event);
- getTriggerFiredLatch().countDown();
- } catch (Throwable t) {
- log.debug("--throwable", t);
- throw t;
- } finally {
- lock.unlock();
- }
- }
-
- @Override
- public void init(Map<String, String> args) {
- log.info("TestEventMarkerAction init");
- actionInitCalled.countDown();
- super.init(args);
- }
- }
-
- @Test
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
- public void testNodeMarkersRegistration() throws Exception {
- // for this test we want to create two triggers so we must assert that the actions were created twice
- actionInitCalled = new CountDownLatch(2);
- // similarly we want both triggers to fire
- triggerFiredLatch = new CountDownLatch(2);
- TestLiveNodesListener listener = registerLiveNodesListener();
-
- NamedList<Object> overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
- String overseerLeader = (String) overSeerStatus.get("leader");
- int overseerLeaderIndex = 0;
- for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
- JettySolrRunner jetty = cluster.getJettySolrRunner(i);
- if (jetty.getNodeName().equals(overseerLeader)) {
- overseerLeaderIndex = i;
- break;
- }
- }
- // add a node
- JettySolrRunner node = cluster.startJettySolrRunner();
- if (!listener.onChangeLatch.await(10, TimeUnit.SECONDS)) {
- fail("onChange listener didn't execute on cluster change");
- }
- assertEquals(1, listener.addedNodes.size());
- assertEquals(node.getNodeName(), listener.addedNodes.iterator().next());
- // verify that a znode doesn't exist (no trigger)
- String pathAdded = ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH + "/" + node.getNodeName();
- assertFalse("Path " + pathAdded + " was created but there are no nodeAdded triggers", zkClient().exists(pathAdded, true));
- listener.reset();
- // stop overseer
- log.info("====== KILL OVERSEER 1");
- cluster.stopJettySolrRunner(overseerLeaderIndex);
- if (!listener.onChangeLatch.await(10, TimeUnit.SECONDS)) {
- fail("onChange listener didn't execute on cluster change");
- }
- assertEquals(1, listener.lostNodes.size());
- assertEquals(overseerLeader, listener.lostNodes.iterator().next());
- assertEquals(0, listener.addedNodes.size());
- // wait until the new overseer is up
- Thread.sleep(5000);
- // verify that a znode does NOT exist - there's no nodeLost trigger,
- // so the new overseer cleaned up existing nodeLost markers
- String pathLost = ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH + "/" + overseerLeader;
- assertFalse("Path " + pathLost + " exists", zkClient().exists(pathLost, true));
-
- listener.reset();
-
- // set up triggers
- CloudSolrClient solrClient = cluster.getSolrClient();
-
- log.info("====== ADD TRIGGERS");
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_added_triggerMR'," +
- "'event' : 'nodeAdded'," +
- "'waitFor' : '1s'," +
- "'enabled' : true," +
- "'actions' : [{'name':'test','class':'" + TestEventMarkerAction.class.getName() + "'}]" +
- "}}";
- SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_lost_triggerMR'," +
- "'event' : 'nodeLost'," +
- "'waitFor' : '1s'," +
- "'enabled' : true," +
- "'actions' : [{'name':'test','class':'" + TestEventMarkerAction.class.getName() + "'}]" +
- "}}";
- req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
- overseerLeader = (String) overSeerStatus.get("leader");
- overseerLeaderIndex = 0;
- for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
- JettySolrRunner jetty = cluster.getJettySolrRunner(i);
- if (jetty.getNodeName().equals(overseerLeader)) {
- overseerLeaderIndex = i;
- break;
- }
- }
-
- // create another node
- log.info("====== ADD NODE 1");
- JettySolrRunner node1 = cluster.startJettySolrRunner();
- if (!listener.onChangeLatch.await(10, TimeUnit.SECONDS)) {
- fail("onChange listener didn't execute on cluster change");
- }
- assertEquals(1, listener.addedNodes.size());
- assertEquals(node1.getNodeName(), listener.addedNodes.iterator().next());
- // verify that a znode exists
- pathAdded = ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH + "/" + node1.getNodeName();
- assertTrue("Path " + pathAdded + " wasn't created", zkClient().exists(pathAdded, true));
-
- Thread.sleep(5000);
- // nodeAdded marker should be consumed now by nodeAdded trigger
- assertFalse("Path " + pathAdded + " should have been deleted", zkClient().exists(pathAdded, true));
-
- listener.reset();
- events.clear();
- triggerFiredLatch = new CountDownLatch(1);
- // kill overseer again
- log.info("====== KILL OVERSEER 2");
- cluster.stopJettySolrRunner(overseerLeaderIndex);
- if (!listener.onChangeLatch.await(10, TimeUnit.SECONDS)) {
- fail("onChange listener didn't execute on cluster change");
- }
-
-
- if (!triggerFiredLatch.await(20, TimeUnit.SECONDS)) {
- fail("Trigger should have fired by now");
- }
- assertEquals(1, events.size());
- TriggerEvent ev = events.iterator().next();
- List<String> nodeNames = (List<String>)ev.getProperty(TriggerEvent.NODE_NAMES);
- assertTrue(nodeNames.contains(overseerLeader));
- assertEquals(TriggerEventType.NODELOST, ev.getEventType());
- }
-
static Map<String, List<CapturedEvent>> listenerEvents = new HashMap<>();
static CountDownLatch listenerCreated = new CountDownLatch(1);
static boolean failDummyAction = false;
@@ -1175,551 +722,4 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
assertEquals(TriggerEventProcessorStage.FAILED, capturedEvents.get(3).stage);
assertEquals("test1", capturedEvents.get(3).actionName);
}
-
- @Test
- public void testCooldown() throws Exception {
- CloudSolrClient solrClient = cluster.getSolrClient();
- failDummyAction = false;
- waitForSeconds = 1;
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_added_cooldown_trigger'," +
- "'event' : 'nodeAdded'," +
- "'waitFor' : '" + waitForSeconds + "s'," +
- "'enabled' : true," +
- "'actions' : [" +
- "{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}" +
- "]" +
- "}}";
- SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- String setListenerCommand1 = "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'bar'," +
- "'trigger' : 'node_added_cooldown_trigger'," +
- "'stage' : ['FAILED','SUCCEEDED', 'IGNORED']," +
- "'class' : '" + TestTriggerListener.class.getName() + "'" +
- "}" +
- "}";
- req = createAutoScalingRequest(SolrRequest.METHOD.POST, setListenerCommand1);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- listenerCreated = new CountDownLatch(1);
- listenerEvents.clear();
-
- JettySolrRunner newNode = cluster.startJettySolrRunner();
- boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
- assertTrue("The trigger did not fire at all", await);
- assertTrue(triggerFired.get());
- // wait for listener to capture the SUCCEEDED stage
- Thread.sleep(1000);
-
- List<CapturedEvent> capturedEvents = listenerEvents.get("bar");
- // we may get a few IGNORED events if other tests caused events within cooldown period
- assertTrue(capturedEvents.toString(), capturedEvents.size() > 0);
- long prevTimestamp = capturedEvents.get(capturedEvents.size() - 1).timestamp;
-
- // reset the trigger and captured events
- listenerEvents.clear();
- triggerFiredLatch = new CountDownLatch(1);
- triggerFired.compareAndSet(true, false);
-
- JettySolrRunner newNode2 = cluster.startJettySolrRunner();
- await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
- assertTrue("The trigger did not fire at all", await);
- // wait for listener to capture the SUCCEEDED stage
- Thread.sleep(2000);
-
- // there must be at least one IGNORED event due to cooldown, and one SUCCEEDED event
- capturedEvents = listenerEvents.get("bar");
- assertEquals(capturedEvents.toString(),1, capturedEvents.size());
- CapturedEvent ev = capturedEvents.get(0);
- assertEquals(ev.toString(), TriggerEventProcessorStage.SUCCEEDED, ev.stage);
- // the difference between timestamps of the first SUCCEEDED and the last SUCCEEDED
- // must be larger than cooldown period
- assertTrue("timestamp delta is less than default cooldown period", ev.timestamp - prevTimestamp > TimeUnit.SECONDS.toNanos(ScheduledTriggers.DEFAULT_COOLDOWN_PERIOD_SECONDS));
- prevTimestamp = ev.timestamp;
-
- // this also resets the cooldown period
- long modifiedCooldownPeriodSeconds = 7;
- String setPropertiesCommand = "{\n" +
- "\t\"set-properties\" : {\n" +
- "\t\t\"" + AutoScalingParams.TRIGGER_COOLDOWN_PERIOD_SECONDS + "\" : " + modifiedCooldownPeriodSeconds + "\n" +
- "\t}\n" +
- "}";
- solrClient.request(createAutoScalingRequest(SolrRequest.METHOD.POST, setPropertiesCommand));
- req = createAutoScalingRequest(SolrRequest.METHOD.GET, null);
- response = solrClient.request(req);
-
- // reset the trigger and captured events
- listenerEvents.clear();
- triggerFiredLatch = new CountDownLatch(1);
- triggerFired.compareAndSet(true, false);
-
- JettySolrRunner newNode3 = cluster.startJettySolrRunner();
- await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
- assertTrue("The trigger did not fire at all", await);
- triggerFiredLatch = new CountDownLatch(1);
- triggerFired.compareAndSet(true, false);
- // add another node
- JettySolrRunner newNode4 = cluster.startJettySolrRunner();
- await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
- assertTrue("The trigger did not fire at all", await);
- // wait for listener to capture the SUCCEEDED stage
- Thread.sleep(2000);
-
- // there must be two SUCCEEDED (due to newNode3 and newNode4) and maybe some ignored events
- capturedEvents = listenerEvents.get("bar");
- assertTrue(capturedEvents.toString(), capturedEvents.size() >= 2);
- // first event should be SUCCEEDED
- ev = capturedEvents.get(0);
- assertEquals(ev.toString(), TriggerEventProcessorStage.SUCCEEDED, ev.stage);
-
- ev = capturedEvents.get(capturedEvents.size() - 1);
- assertEquals(ev.toString(), TriggerEventProcessorStage.SUCCEEDED, ev.stage);
- // the difference between timestamps of the first SUCCEEDED and the last SUCCEEDED
- // must be larger than the modified cooldown period
- assertTrue("timestamp delta is less than default cooldown period", ev.timestamp - prevTimestamp > TimeUnit.SECONDS.toNanos(modifiedCooldownPeriodSeconds));
- }
-
- public void testSetProperties() throws Exception {
- JettySolrRunner runner = cluster.getJettySolrRunner(0);
- SolrResourceLoader resourceLoader = runner.getCoreContainer().getResourceLoader();
- SolrCloudManager solrCloudManager = runner.getCoreContainer().getZkController().getSolrCloudManager();
- AtomicLong diff = new AtomicLong(0);
- triggerFiredLatch = new CountDownLatch(2); // have the trigger run twice to capture time difference
- try (ScheduledTriggers scheduledTriggers = new ScheduledTriggers(resourceLoader, solrCloudManager)) {
- AutoScalingConfig config = new AutoScalingConfig(Collections.emptyMap());
- scheduledTriggers.setAutoScalingConfig(config);
- scheduledTriggers.add(new TriggerBase(TriggerEventType.NODELOST, "x", Collections.emptyMap(), resourceLoader, solrCloudManager) {
- @Override
- protected Map<String, Object> getState() {
- return Collections.singletonMap("x","y");
- }
-
- @Override
- protected void setState(Map<String, Object> state) {
-
- }
-
- @Override
- public void restoreState(AutoScaling.Trigger old) {
-
- }
-
- @Override
- public void run() {
- if (getTriggerFiredLatch().getCount() == 0) return;
- long l = diff.get();
- diff.set(timeSource.getTimeNs() - l);
- getTriggerFiredLatch().countDown();
- }
- });
- assertTrue(getTriggerFiredLatch().await(4, TimeUnit.SECONDS));
- assertTrue(diff.get() - TimeUnit.SECONDS.toNanos(ScheduledTriggers.DEFAULT_SCHEDULED_TRIGGER_DELAY_SECONDS) >= 0);
-
- // change schedule delay
- config = config.withProperties(Collections.singletonMap(AutoScalingParams.TRIGGER_SCHEDULE_DELAY_SECONDS, 4));
- scheduledTriggers.setAutoScalingConfig(config);
- triggerFiredLatch = new CountDownLatch(2);
- assertTrue("Timed out waiting for latch to fire", getTriggerFiredLatch().await(10, TimeUnit.SECONDS));
- assertTrue(diff.get() - TimeUnit.SECONDS.toNanos(4) >= 0);
-
- // reset with default properties
- scheduledTriggers.remove("x"); // remove the old trigger
- config = config.withProperties(ScheduledTriggers.DEFAULT_PROPERTIES);
- scheduledTriggers.setAutoScalingConfig(config);
-
- // test core thread count
- List<AutoScaling.Trigger> triggerList = new ArrayList<>();
- final Set<String> threadNames = Collections.synchronizedSet(new HashSet<>());
- final Set<String> triggerNames = Collections.synchronizedSet(new HashSet<>());
- triggerFiredLatch = new CountDownLatch(8);
- for (int i = 0; i < 8; i++) {
- triggerList.add(new MockTrigger(TriggerEventType.NODELOST, "x" + i, Collections.emptyMap(), resourceLoader, solrCloudManager) {
- @Override
- public void run() {
- try {
- // If core pool size is increased then new threads won't be started if existing threads
- // aren't busy with tasks. So we make this thread wait longer than necessary
- // so that the pool is forced to start threads for other triggers
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- }
- if (triggerNames.add(getName())) {
- getTriggerFiredLatch().countDown();
- threadNames.add(Thread.currentThread().getName());
- }
- }
- });
- scheduledTriggers.add(triggerList.get(i));
- }
- assertTrue("Timed out waiting for latch to fire", getTriggerFiredLatch().await(20, TimeUnit.SECONDS));
- assertEquals("Expected 8 triggers but found: " + triggerNames,8, triggerNames.size());
- assertEquals("Expected " + ScheduledTriggers.DEFAULT_TRIGGER_CORE_POOL_SIZE
- + " threads but found: " + threadNames,
- ScheduledTriggers.DEFAULT_TRIGGER_CORE_POOL_SIZE, threadNames.size());
-
- // change core pool size
- config = config.withProperties(Collections.singletonMap(AutoScalingParams.TRIGGER_CORE_POOL_SIZE, 6));
- scheduledTriggers.setAutoScalingConfig(config);
- triggerFiredLatch = new CountDownLatch(8);
- threadNames.clear();
- triggerNames.clear();
- assertTrue(getTriggerFiredLatch().await(20, TimeUnit.SECONDS));
- assertEquals("Expected 8 triggers but found: " + triggerNames,8, triggerNames.size());
- assertEquals("Expected 6 threads but found: " + threadNames,6, threadNames.size());
-
- // reset
- for (int i = 0; i < 8; i++) {
- scheduledTriggers.remove(triggerList.get(i).getName());
- }
- }
- }
-
- public static class MockTrigger extends TriggerBase {
-
- public MockTrigger(TriggerEventType eventType, String name, Map<String, Object> properties, SolrResourceLoader loader, SolrCloudManager cloudManager) {
- super(eventType, name, properties, loader, cloudManager);
- }
-
- @Override
- protected Map<String, Object> getState() {
- return Collections.emptyMap();
- }
-
- @Override
- protected void setState(Map<String, Object> state) {
-
- }
-
- @Override
- public void restoreState(AutoScaling.Trigger old) {
-
- }
-
- @Override
- public void run() {
-
- }
- }
-
- public static class TestSearchRateAction extends TriggerActionBase {
-
- @Override
- public void process(TriggerEvent event, ActionContext context) throws Exception {
- try {
- events.add(event);
- long currentTimeNanos = timeSource.getTimeNs();
- long eventTimeNanos = event.getEventTime();
- long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS;
- if (currentTimeNanos - eventTimeNanos <= waitForNanos) {
- fail(event.source + " was fired before the configured waitFor period");
- }
- getTriggerFiredLatch().countDown();
- } catch (Throwable t) {
- log.debug("--throwable", t);
- throw t;
- }
- }
- }
-
- @Test
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
- public void testSearchRate() throws Exception {
- // start a few more jetty-s
- for (int i = 0; i < 3; i++) {
- cluster.startJettySolrRunner();
- }
- CloudSolrClient solrClient = cluster.getSolrClient();
- String COLL1 = "collection1";
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(COLL1,
- "conf", 1, 2);
- create.process(solrClient);
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'search_rate_trigger'," +
- "'event' : 'searchRate'," +
- "'waitFor' : '" + waitForSeconds + "s'," +
- "'enabled' : true," +
- "'rate' : 1.0," +
- "'actions' : [" +
- "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," +
- "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}," +
- "{'name':'test','class':'" + TestSearchRateAction.class.getName() + "'}" +
- "]" +
- "}}";
- SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- String setListenerCommand1 = "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'srt'," +
- "'trigger' : 'search_rate_trigger'," +
- "'stage' : ['FAILED','SUCCEEDED']," +
- "'afterAction': ['compute', 'execute', 'test']," +
- "'class' : '" + TestTriggerListener.class.getName() + "'" +
- "}" +
- "}";
- req = createAutoScalingRequest(SolrRequest.METHOD.POST, setListenerCommand1);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
- SolrParams query = params(CommonParams.Q, "*:*");
- for (int i = 0; i < 500; i++) {
- solrClient.query(COLL1, query);
- }
- boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
- assertTrue("The trigger did not fire at all", await);
- // wait for listener to capture the SUCCEEDED stage
- Thread.sleep(5000);
- List<CapturedEvent> events = listenerEvents.get("srt");
- assertEquals(listenerEvents.toString(), 4, events.size());
- assertEquals("AFTER_ACTION", events.get(0).stage.toString());
- assertEquals("compute", events.get(0).actionName);
- assertEquals("AFTER_ACTION", events.get(1).stage.toString());
- assertEquals("execute", events.get(1).actionName);
- assertEquals("AFTER_ACTION", events.get(2).stage.toString());
- assertEquals("test", events.get(2).actionName);
- assertEquals("SUCCEEDED", events.get(3).stage.toString());
- assertNull(events.get(3).actionName);
-
- CapturedEvent ev = events.get(0);
- long now = timeSource.getTimeNs();
- // verify waitFor
- assertTrue(TimeUnit.SECONDS.convert(waitForSeconds, TimeUnit.NANOSECONDS) - WAIT_FOR_DELTA_NANOS <= now - ev.event.getEventTime());
- Map<String, Double> nodeRates = (Map<String, Double>)ev.event.getProperties().get("node");
- assertNotNull("nodeRates", nodeRates);
- assertTrue(nodeRates.toString(), nodeRates.size() > 0);
- AtomicDouble totalNodeRate = new AtomicDouble();
- nodeRates.forEach((n, r) -> totalNodeRate.addAndGet(r));
- List<ReplicaInfo> replicaRates = (List<ReplicaInfo>)ev.event.getProperties().get("replica");
- assertNotNull("replicaRates", replicaRates);
- assertTrue(replicaRates.toString(), replicaRates.size() > 0);
- AtomicDouble totalReplicaRate = new AtomicDouble();
- replicaRates.forEach(r -> {
- assertTrue(r.toString(), r.getVariable("rate") != null);
- totalReplicaRate.addAndGet((Double)r.getVariable("rate"));
- });
- Map<String, Object> shardRates = (Map<String, Object>)ev.event.getProperties().get("shard");
- assertNotNull("shardRates", shardRates);
- assertEquals(shardRates.toString(), 1, shardRates.size());
- shardRates = (Map<String, Object>)shardRates.get(COLL1);
- assertNotNull("shardRates", shardRates);
- assertEquals(shardRates.toString(), 1, shardRates.size());
- AtomicDouble totalShardRate = new AtomicDouble();
- shardRates.forEach((s, r) -> totalShardRate.addAndGet((Double)r));
- Map<String, Double> collectionRates = (Map<String, Double>)ev.event.getProperties().get("collection");
- assertNotNull("collectionRates", collectionRates);
- assertEquals(collectionRates.toString(), 1, collectionRates.size());
- Double collectionRate = collectionRates.get(COLL1);
- assertNotNull(collectionRate);
- assertTrue(collectionRate > 5.0);
- assertEquals(collectionRate, totalNodeRate.get(), 5.0);
- assertEquals(collectionRate, totalShardRate.get(), 5.0);
- assertEquals(collectionRate, totalReplicaRate.get(), 5.0);
-
- // check operations
- List<Map<String, Object>> ops = (List<Map<String, Object>>)ev.context.get("properties.operations");
- assertNotNull(ops);
- assertTrue(ops.size() > 1);
- for (Map<String, Object> m : ops) {
- assertEquals("ADDREPLICA", m.get("params.action"));
- }
- }
-
- @Test
- public void testMetricTrigger() throws Exception {
- cluster.waitForAllNodes(5);
-
- String collectionName = "testMetricTrigger";
- CloudSolrClient solrClient = cluster.getSolrClient();
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,
- "conf", 2, 2).setMaxShardsPerNode(2);
- create.process(solrClient);
- solrClient.setDefaultCollection(collectionName);
-
- waitForState("Timed out waiting for collection:" + collectionName + " to become active", collectionName, clusterShape(2, 2));
-
- DocCollection docCollection = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
- String shardId = "shard1";
- Replica replica = docCollection.getSlice(shardId).getReplicas().iterator().next();
- String coreName = replica.getCoreName();
- String replicaName = Utils.parseMetricsReplicaName(collectionName, coreName);
- long waitForSeconds = 2 + random().nextInt(5);
- String registry = SolrCoreMetricManager.createRegistryName(true, collectionName, shardId, replicaName, null);
- String tag = "metrics:" + registry + ":INDEX.sizeInBytes";
-
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'metric_trigger'," +
- "'event' : 'metric'," +
- "'waitFor' : '" + waitForSeconds + "s'," +
- "'enabled' : true," +
- "'metric': '" + tag + "'" +
- "'above' : 100.0," +
- "'collection': '" + collectionName + "'" +
- "'shard':'" + shardId + "'" +
- "'actions' : [" +
- "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," +
- "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}," +
- "{'name':'test','class':'" + TestSearchRateAction.class.getName() + "'}" +
- "]" +
- "}}";
- SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- String setListenerCommand1 = "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'srt'," +
- "'trigger' : 'metric_trigger'," +
- "'stage' : ['FAILED','SUCCEEDED']," +
- "'afterAction': ['compute', 'execute', 'test']," +
- "'class' : '" + TestTriggerListener.class.getName() + "'" +
- "}" +
- "}";
- req = createAutoScalingRequest(SolrRequest.METHOD.POST, setListenerCommand1);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- // start more nodes so that we have at least 4
- for (int i = cluster.getJettySolrRunners().size(); i < 4; i++) {
- cluster.startJettySolrRunner();
- }
- cluster.waitForAllNodes(10);
-
- List<SolrInputDocument> docs = new ArrayList<>(500);
- for (int i = 0; i < 500; i++) {
- docs.add(new SolrInputDocument("id", String.valueOf(i), "x_s", "x" + i));
- }
- solrClient.add(docs);
- solrClient.commit();
-
- boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
- assertTrue("The trigger did not fire at all", await);
- // wait for listener to capture the SUCCEEDED stage
- Thread.sleep(2000);
- assertEquals(listenerEvents.toString(), 4, listenerEvents.get("srt").size());
- CapturedEvent ev = listenerEvents.get("srt").get(0);
- long now = timeSource.getTimeNs();
- // verify waitFor
- assertTrue(TimeUnit.SECONDS.convert(waitForSeconds, TimeUnit.NANOSECONDS) - WAIT_FOR_DELTA_NANOS <= now - ev.event.getEventTime());
- assertEquals(collectionName, ev.event.getProperties().get("collection"));
-
- // find a new replica and create its metric name
- docCollection = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
- replica = docCollection.getSlice(shardId).getReplicas().iterator().next();
- coreName = replica.getCoreName();
- replicaName = Utils.parseMetricsReplicaName(collectionName, coreName);
- registry = SolrCoreMetricManager.createRegistryName(true, collectionName, shardId, replicaName, null);
- tag = "metrics:" + registry + ":INDEX.sizeInBytes";
-
- triggerFiredLatch = new CountDownLatch(1);
- listenerEvents.clear();
-
- setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'metric_trigger'," +
- "'event' : 'metric'," +
- "'waitFor' : '" + waitForSeconds + "s'," +
- "'enabled' : true," +
- "'metric': '" + tag + "'" +
- "'above' : 100.0," +
- "'collection': '" + collectionName + "'" +
- "'shard':'" + shardId + "'" +
- "'preferredOperation':'addreplica'" +
- "'actions' : [" +
- "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," +
- "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}," +
- "{'name':'test','class':'" + TestSearchRateAction.class.getName() + "'}" +
- "]" +
- "}}";
- req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
- assertTrue("The trigger did not fire at all", await);
- // wait for listener to capture the SUCCEEDED stage
- Thread.sleep(2000);
- assertEquals(listenerEvents.toString(), 4, listenerEvents.get("srt").size());
- ev = listenerEvents.get("srt").get(0);
- now = timeSource.getTimeNs();
- // verify waitFor
- assertTrue(TimeUnit.SECONDS.convert(waitForSeconds, TimeUnit.NANOSECONDS) - WAIT_FOR_DELTA_NANOS <= now - ev.event.getEventTime());
- assertEquals(collectionName, ev.event.getProperties().get("collection"));
- docCollection = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
- assertEquals(5, docCollection.getReplicas().size());
- }
-
- @Test
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 26-Mar-2018
- public void testScheduledTrigger() throws Exception {
- CloudSolrClient solrClient = cluster.getSolrClient();
-
- // this collection will place 2 cores on 1st node and 1 core on 2nd node
- String collectionName = "testScheduledTrigger";
- CollectionAdminRequest.createCollection(collectionName, 1, 3)
- .setMaxShardsPerNode(5).process(solrClient);
- waitForState("", collectionName, clusterShape(1, 3));
-
- // create a policy which allows only 1 core per node thereby creating a violation for the above collection
- String setClusterPolicy = "{\n" +
- " \"set-cluster-policy\" : [\n" +
- " {\"cores\" : \"<2\", \"node\" : \"#EACH\"}\n" +
- " ]\n" +
- "}";
- SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setClusterPolicy);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- // start a new node which can be used to balance the cluster as per policy
- JettySolrRunner newNode = cluster.startJettySolrRunner();
- cluster.waitForAllNodes(10);
-
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'sched_trigger_integration1'," +
- "'event' : 'scheduled'," +
- "'startTime' : '" + new Date().toInstant().toString() + "'" +
- "'every' : '+3SECONDS'" +
- "'actions' : [" +
- "{'name' : 'compute','class':'" + ComputePlanAction.class.getName() + "'}," +
- "{'name' : 'execute','class':'" + ExecutePlanAction.class.getName() + "'}," +
- "{'name' : 'recorder', 'class': '" + ContextPropertiesRecorderAction.class.getName() + "'}" +
- "]}}";
- req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- assertTrue("ScheduledTrigger did not fire within 20 seconds", triggerFiredLatch.await(20, TimeUnit.SECONDS));
- assertEquals(1, events.size());
- Map<String, Object> actionContextProps = actionContextPropertiesRef.get();
- assertNotNull(actionContextProps);
- TriggerEvent event = events.iterator().next();
- List<SolrRequest> operations = (List<SolrRequest>) actionContextProps.get("operations");
- assertNotNull(operations);
- assertEquals(1, operations.size());
- for (SolrRequest operation : operations) {
- SolrParams params = operation.getParams();
- assertEquals(newNode.getNodeName(), params.get("targetNode"));
- }
- }
-
- private static AtomicReference<Map<String, Object>> actionContextPropertiesRef = new AtomicReference<>();
-
- public static class ContextPropertiesRecorderAction extends TestEventMarkerAction {
- @Override
- public void process(TriggerEvent event, ActionContext actionContext) {
- actionContextPropertiesRef.set(actionContext.getProperties());
- super.process(event, actionContext);
- }
- }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ed9e5eb7/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerSetPropertiesIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerSetPropertiesIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerSetPropertiesIntegrationTest.java
new file mode 100644
index 0000000..47ac227
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerSetPropertiesIntegrationTest.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud.autoscaling;
+
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.cloud.SolrCloudManager;
+import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
+import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.params.AutoScalingParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.util.LogLevel;
+import org.junit.BeforeClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.createAutoScalingRequest;
+import static org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.timeSource;
+
+@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG")
+public class TriggerSetPropertiesIntegrationTest extends SolrCloudTestCase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private static CountDownLatch triggerFiredLatch = new CountDownLatch(1);
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(5)
+ .addConfig("conf", configset("cloud-minimal"))
+ .configure();
+ // disable .scheduled_maintenance
+ String suspendTriggerCommand = "{" +
+ "'suspend-trigger' : {'name' : '.scheduled_maintenance'}" +
+ "}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
+ SolrClient solrClient = cluster.getSolrClient();
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ }
+
+ private static CountDownLatch getTriggerFiredLatch() {
+ return triggerFiredLatch;
+ }
+
+ public void testSetProperties() throws Exception {
+ JettySolrRunner runner = cluster.getJettySolrRunner(0);
+ SolrResourceLoader resourceLoader = runner.getCoreContainer().getResourceLoader();
+ SolrCloudManager solrCloudManager = runner.getCoreContainer().getZkController().getSolrCloudManager();
+ AtomicLong diff = new AtomicLong(0);
+ triggerFiredLatch = new CountDownLatch(2); // have the trigger run twice to capture time difference
+ try (ScheduledTriggers scheduledTriggers = new ScheduledTriggers(resourceLoader, solrCloudManager)) {
+ AutoScalingConfig config = new AutoScalingConfig(Collections.emptyMap());
+ scheduledTriggers.setAutoScalingConfig(config);
+ scheduledTriggers.add(new TriggerBase(TriggerEventType.NODELOST, "x", Collections.emptyMap(), resourceLoader, solrCloudManager) {
+ @Override
+ protected Map<String, Object> getState() {
+ return Collections.singletonMap("x", "y");
+ }
+
+ @Override
+ protected void setState(Map<String, Object> state) {
+
+ }
+
+ @Override
+ public void restoreState(AutoScaling.Trigger old) {
+
+ }
+
+ @Override
+ public void run() {
+ if (getTriggerFiredLatch().getCount() == 0) return;
+ long l = diff.get();
+ diff.set(timeSource.getTimeNs() - l);
+ getTriggerFiredLatch().countDown();
+ }
+ });
+ assertTrue(getTriggerFiredLatch().await(4, TimeUnit.SECONDS));
+ assertTrue(diff.get() - TimeUnit.SECONDS.toNanos(ScheduledTriggers.DEFAULT_SCHEDULED_TRIGGER_DELAY_SECONDS) >= 0);
+
+ // change schedule delay
+ config = config.withProperties(Collections.singletonMap(AutoScalingParams.TRIGGER_SCHEDULE_DELAY_SECONDS, 4));
+ scheduledTriggers.setAutoScalingConfig(config);
+ triggerFiredLatch = new CountDownLatch(2);
+ assertTrue("Timed out waiting for latch to fire", getTriggerFiredLatch().await(10, TimeUnit.SECONDS));
+ assertTrue(diff.get() - TimeUnit.SECONDS.toNanos(4) >= 0);
+
+ // reset with default properties
+ scheduledTriggers.remove("x"); // remove the old trigger
+ config = config.withProperties(ScheduledTriggers.DEFAULT_PROPERTIES);
+ scheduledTriggers.setAutoScalingConfig(config);
+
+ // test core thread count
+ List<AutoScaling.Trigger> triggerList = new ArrayList<>();
+ final Set<String> threadNames = Collections.synchronizedSet(new HashSet<>());
+ final Set<String> triggerNames = Collections.synchronizedSet(new HashSet<>());
+ triggerFiredLatch = new CountDownLatch(8);
+ for (int i = 0; i < 8; i++) {
+ triggerList.add(new MockTrigger(TriggerEventType.NODELOST, "x" + i, Collections.emptyMap(), resourceLoader, solrCloudManager) {
+ @Override
+ public void run() {
+ try {
+ // If core pool size is increased then new threads won't be started if existing threads
+ // aren't busy with tasks. So we make this thread wait longer than necessary
+ // so that the pool is forced to start threads for other triggers
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ }
+ if (triggerNames.add(getName())) {
+ getTriggerFiredLatch().countDown();
+ threadNames.add(Thread.currentThread().getName());
+ }
+ }
+ });
+ scheduledTriggers.add(triggerList.get(i));
+ }
+ assertTrue("Timed out waiting for latch to fire", getTriggerFiredLatch().await(20, TimeUnit.SECONDS));
+ assertEquals("Expected 8 triggers but found: " + triggerNames, 8, triggerNames.size());
+ assertEquals("Expected " + ScheduledTriggers.DEFAULT_TRIGGER_CORE_POOL_SIZE
+ + " threads but found: " + threadNames,
+ ScheduledTriggers.DEFAULT_TRIGGER_CORE_POOL_SIZE, threadNames.size());
+
+ // change core pool size
+ config = config.withProperties(Collections.singletonMap(AutoScalingParams.TRIGGER_CORE_POOL_SIZE, 6));
+ scheduledTriggers.setAutoScalingConfig(config);
+ triggerFiredLatch = new CountDownLatch(8);
+ threadNames.clear();
+ triggerNames.clear();
+ assertTrue(getTriggerFiredLatch().await(20, TimeUnit.SECONDS));
+ assertEquals("Expected 8 triggers but found: " + triggerNames, 8, triggerNames.size());
+ assertEquals("Expected 6 threads but found: " + threadNames, 6, threadNames.size());
+
+ // reset
+ for (int i = 0; i < 8; i++) {
+ scheduledTriggers.remove(triggerList.get(i).getName());
+ }
+ }
+ }
+
+ public static class MockTrigger extends TriggerBase {
+
+ public MockTrigger(TriggerEventType eventType, String name, Map<String, Object> properties, SolrResourceLoader loader, SolrCloudManager cloudManager) {
+ super(eventType, name, properties, loader, cloudManager);
+ }
+
+ @Override
+ protected Map<String, Object> getState() {
+ return Collections.emptyMap();
+ }
+
+ @Override
+ protected void setState(Map<String, Object> state) {
+
+ }
+
+ @Override
+ public void restoreState(AutoScaling.Trigger old) {
+
+ }
+
+ @Override
+ public void run() {
+
+ }
+ }
+}
[18/34] lucene-solr:jira/solr-12095: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/lucene-solr
Posted by sh...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/lucene-solr
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0ef68f7d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0ef68f7d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0ef68f7d
Branch: refs/heads/jira/solr-12095
Commit: 0ef68f7d7ce17601ff43f76a670f3360ecd884c6
Parents: e06554a b78d980
Author: Karl Wright <Da...@gmail.com>
Authored: Fri Mar 30 08:37:06 2018 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Fri Mar 30 08:37:06 2018 -0400
----------------------------------------------------------------------
.../lucene/spatial3d/geom/GeoPolygonTest.java | 1 -
.../cloud/autoscaling/ScheduledTriggers.java | 13 +++++++++
.../autoscaling/TriggerIntegrationTest.java | 28 +++++++++++---------
.../apache/solr/common/util/ExecutorUtil.java | 4 +++
4 files changed, 32 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0ef68f7d/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
----------------------------------------------------------------------
[32/34] lucene-solr:jira/solr-12095: SOLR-7736: break
OverseerTriggerThread when interrupted.
Posted by sh...@apache.org.
SOLR-7736: break OverseerTriggerThread when interrupted.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a4789db4
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a4789db4
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a4789db4
Branch: refs/heads/jira/solr-12095
Commit: a4789db47788daeef0ba2ab426b4047d2fa47070
Parents: 7a920cb
Author: Mikhail Khludnev <mk...@apache.org>
Authored: Sun Apr 1 18:37:18 2018 +0300
Committer: Mikhail Khludnev <mk...@apache.org>
Committed: Sun Apr 1 18:37:18 2018 +0300
----------------------------------------------------------------------
.../org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a4789db4/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
index f97372d..874122d 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
@@ -124,6 +124,10 @@ public class OverseerTriggerThread implements Runnable, SolrCloseable {
// we also automatically add a scheduled maintenance trigger
while (!isClosed) {
try {
+ if (Thread.currentThread().isInterrupted()) {
+ log.warn("Interrupted");
+ break;
+ }
AutoScalingConfig autoScalingConfig = cloudManager.getDistribStateManager().getAutoScalingConfig();
AutoScalingConfig updatedConfig = withAutoAddReplicasTrigger(autoScalingConfig);
updatedConfig = withScheduledMaintenanceTrigger(updatedConfig);
[31/34] lucene-solr:jira/solr-12095: SOLR-11673: Slave doesn't commit
empty index if new index appears on master by deafult.
Posted by sh...@apache.org.
SOLR-11673: Slave doesn't commit empty index if new index appears on master by deafult.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/7a920cb0
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/7a920cb0
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/7a920cb0
Branch: refs/heads/jira/solr-12095
Commit: 7a920cb0f64ee3de058a980e7564685cdc311c06
Parents: d98138a
Author: Mikhail Khludnev <mk...@apache.org>
Authored: Sun Apr 1 16:47:23 2018 +0300
Committer: Mikhail Khludnev <mk...@apache.org>
Committed: Sun Apr 1 16:47:23 2018 +0300
----------------------------------------------------------------------
solr/CHANGES.txt | 7 ++++
.../org/apache/solr/handler/IndexFetcher.java | 35 ++++++++++++++++----
.../org/apache/solr/util/TestInjection.java | 15 +++++++++
.../solr/handler/TestReplicationHandler.java | 7 ++--
4 files changed, 55 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7a920cb0/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index e7349cf..0dfb90b 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -61,6 +61,10 @@ Upgrade Notes
This is a server side change only and clients using SolrJ won't need any changes. Clients can still use any logging
implementation which is compatible with SLF4J.
+* SOLR-11673: Slave doesn't commit empty index when completely new index is detected on master during replication.
+ To return the previous behavior pass false to skipCommitOnMasterVersionZero in slave section of replication
+ handler configuration, or pass it to the fetchindex command.
+
New Features
----------------------
@@ -95,6 +99,9 @@ Bug Fixes
document. Multithreaded test for AtomicUpdateRequestProcessor was also beefed up and fixed.
(Ishan Chattopadhyaya, Noble Paul, Amrit Sarkar, shalin)
+* SOLR-11673: By default slave doesn't commit empty index when completely new index appears on master.
+ See Upgrade Notes to find a way to get back to the previous behavior. (Mikhail Khludnev)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7a920cb0/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
index 2476a81..3c6859b 100644
--- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
+++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
@@ -16,6 +16,31 @@
*/
package org.apache.solr.handler;
+import static org.apache.solr.common.params.CommonParams.JAVABIN;
+import static org.apache.solr.common.params.CommonParams.NAME;
+import static org.apache.solr.handler.ReplicationHandler.ALIAS;
+import static org.apache.solr.handler.ReplicationHandler.CHECKSUM;
+import static org.apache.solr.handler.ReplicationHandler.CMD_DETAILS;
+import static org.apache.solr.handler.ReplicationHandler.CMD_GET_FILE;
+import static org.apache.solr.handler.ReplicationHandler.CMD_GET_FILE_LIST;
+import static org.apache.solr.handler.ReplicationHandler.CMD_INDEX_VERSION;
+import static org.apache.solr.handler.ReplicationHandler.COMMAND;
+import static org.apache.solr.handler.ReplicationHandler.COMPRESSION;
+import static org.apache.solr.handler.ReplicationHandler.CONF_FILES;
+import static org.apache.solr.handler.ReplicationHandler.CONF_FILE_SHORT;
+import static org.apache.solr.handler.ReplicationHandler.EXTERNAL;
+import static org.apache.solr.handler.ReplicationHandler.FETCH_FROM_LEADER;
+import static org.apache.solr.handler.ReplicationHandler.FILE;
+import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM;
+import static org.apache.solr.handler.ReplicationHandler.GENERATION;
+import static org.apache.solr.handler.ReplicationHandler.INTERNAL;
+import static org.apache.solr.handler.ReplicationHandler.MASTER_URL;
+import static org.apache.solr.handler.ReplicationHandler.OFFSET;
+import static org.apache.solr.handler.ReplicationHandler.SIZE;
+import static org.apache.solr.handler.ReplicationHandler.SKIP_COMMIT_ON_MASTER_VERSION_ZERO;
+import static org.apache.solr.handler.ReplicationHandler.TLOG_FILE;
+import static org.apache.solr.handler.ReplicationHandler.TLOG_FILES;
+
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
@@ -86,7 +111,7 @@ import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.core.IndexDeletionPolicyWrapper;
import org.apache.solr.core.SolrCore;
-import org.apache.solr.handler.ReplicationHandler.*;
+import org.apache.solr.handler.ReplicationHandler.FileInfo;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.search.SolrIndexSearcher;
@@ -99,13 +124,10 @@ import org.apache.solr.util.FileUtils;
import org.apache.solr.util.PropertiesOutputStream;
import org.apache.solr.util.RTimer;
import org.apache.solr.util.RefCounted;
+import org.apache.solr.util.TestInjection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.solr.common.params.CommonParams.JAVABIN;
-import static org.apache.solr.common.params.CommonParams.NAME;
-import static org.apache.solr.handler.ReplicationHandler.*;
-
import com.google.common.base.Strings;
/**
@@ -167,7 +189,7 @@ public class IndexFetcher {
private boolean downloadTlogFiles = false;
- private boolean skipCommitOnMasterVersionZero;
+ private boolean skipCommitOnMasterVersionZero = true;
private static final String INTERRUPT_RESPONSE_MESSAGE = "Interrupted while waiting for modify lock";
@@ -453,6 +475,7 @@ public class IndexFetcher {
} finally {
iw.decref();
}
+ assert TestInjection.injectDelayBeforeSlaveCommitRefresh();
if (skipCommitOnMasterVersionZero) {
openNewSearcherAndUpdateCommitPoint();
} else {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7a920cb0/solr/core/src/java/org/apache/solr/util/TestInjection.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/TestInjection.java b/solr/core/src/java/org/apache/solr/util/TestInjection.java
index 422de73..821b37e 100644
--- a/solr/core/src/java/org/apache/solr/util/TestInjection.java
+++ b/solr/core/src/java/org/apache/solr/util/TestInjection.java
@@ -144,6 +144,8 @@ public class TestInjection {
private static AtomicInteger countPrepRecoveryOpPauseForever = new AtomicInteger(0);
+ public static Integer delayBeforeSlaveCommitRefresh=null;
+
public static void reset() {
nonGracefullClose = null;
failReplicaRequests = null;
@@ -158,6 +160,7 @@ public class TestInjection {
waitForReplicasInSync = "true:60";
failIndexFingerprintRequests = null;
wrongIndexFingerprint = null;
+ delayBeforeSlaveCommitRefresh = null;
for (Timer timer : timers) {
timer.cancel();
@@ -455,4 +458,16 @@ public class TestInjection {
return new Pair<>(Boolean.parseBoolean(val), Integer.parseInt(percent));
}
+ public static boolean injectDelayBeforeSlaveCommitRefresh() {
+ if (delayBeforeSlaveCommitRefresh!=null) {
+ try {
+ log.info("Pausing IndexFetcher for {}ms", delayBeforeSlaveCommitRefresh);
+ Thread.sleep(delayBeforeSlaveCommitRefresh);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ return true;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7a920cb0/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
index ddc5b60..e8caf99 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
@@ -70,6 +70,7 @@ import org.apache.solr.core.SolrCore;
import org.apache.solr.core.StandardDirectoryFactory;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager;
import org.apache.solr.util.FileUtils;
+import org.apache.solr.util.TestInjection;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -494,8 +495,10 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
}
@Test
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-11673")
public void doTestIndexAndConfigReplication() throws Exception {
+
+ TestInjection.delayBeforeSlaveCommitRefresh = random().nextInt(10);
+
clearIndexWithReplication();
nDocs--;
@@ -550,7 +553,6 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
slaveJetty = createJetty(slave);
slaveClient.close();
slaveClient = createNewSolrClient(slaveJetty.getLocalPort());
-
//add a doc with new field and commit on master to trigger index fetch from slave.
index(masterClient, "id", "2000", "name", "name = " + 2000, "newname", "newname = " + 2000);
masterClient.commit();
@@ -567,7 +569,6 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
checkForSingleIndex(masterJetty);
checkForSingleIndex(slaveJetty, true);
-
}
@Test
[13/34] lucene-solr:jira/solr-12095: LUCENE-8227: Redevelop path
iterator implementations to make them robust against edges on paths.
Posted by sh...@apache.org.
LUCENE-8227: Redevelop path iterator implementations to make them robust against edges on paths.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/bc40f6c7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/bc40f6c7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/bc40f6c7
Branch: refs/heads/jira/solr-12095
Commit: bc40f6c7e219c3def81e5d3bee6d5123cc4141e6
Parents: 358e595
Author: Karl Wright <Da...@gmail.com>
Authored: Fri Mar 30 06:43:42 2018 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Fri Mar 30 06:43:42 2018 -0400
----------------------------------------------------------------------
.../spatial3d/geom/GeoComplexPolygon.java | 680 ++++++-------------
.../apache/lucene/spatial3d/TestGeo3DPoint.java | 11 +-
.../lucene/spatial3d/geom/GeoPolygonTest.java | 21 +-
3 files changed, 223 insertions(+), 489 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bc40f6c7/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoComplexPolygon.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoComplexPolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoComplexPolygon.java
index de12348..c8d6435 100644
--- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoComplexPolygon.java
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoComplexPolygon.java
@@ -19,6 +19,8 @@ package org.apache.lucene.spatial3d.geom;
import java.util.Arrays;
import java.util.List;
import java.util.ArrayList;
+import java.util.Set;
+import java.util.HashSet;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.IOException;
@@ -157,6 +159,7 @@ class GeoComplexPolygon extends GeoBasePolygon {
@Override
public boolean isWithin(final double x, final double y, final double z) {
+ //System.out.println("\nIswithin called for ["+x+","+y+","+z+"]");
// If we're right on top of the point, we know the answer.
if (testPoint.isNumericallyIdentical(x, y, z)) {
return testPointInSet;
@@ -366,11 +369,12 @@ class GeoComplexPolygon extends GeoBasePolygon {
if (!firstLegTree.traverse(edgeIterator, firstLegValue)) {
return true;
}
- edgeIterator.setSecondLeg();
+ //edgeIterator.setSecondLeg();
if (!secondLegTree.traverse(edgeIterator, secondLegValue)) {
return true;
}
- return ((edgeIterator.crossingCount & 1) == 0)?testPointInSet:!testPointInSet;
+ //System.out.println("Polarity vs. test point: "+(((edgeIterator.getCrossingCount() & 1) == 0)?"same":"different")+"; testPointInSet: "+testPointInSet);
+ return ((edgeIterator.getCrossingCount() & 1) == 0)?testPointInSet:!testPointInSet;
}
}
@@ -502,6 +506,8 @@ class GeoComplexPolygon extends GeoBasePolygon {
this.planeBounds.addPlane(pm, this.plane, this.startPlane, this.endPlane);
//System.err.println("Recording edge "+this+" from "+startPoint+" to "+endPoint+"; bounds = "+planeBounds);
}
+
+ // Hashcode and equals are system default!!
}
/**
@@ -798,7 +804,8 @@ class GeoComplexPolygon extends GeoBasePolygon {
private final double thePointY;
private final double thePointZ;
- private int crossingCount = 0;
+ private int aboveCrossingCount = 0;
+ private int belowCrossingCount = 0;
public FullLinearCrossingEdgeIterator(final Plane plane, final Plane abovePlane, final Plane belowPlane, final double thePointX, final double thePointY, final double thePointZ) {
this.plane = plane;
@@ -813,7 +820,11 @@ class GeoComplexPolygon extends GeoBasePolygon {
@Override
public int getCrossingCount() {
- return crossingCount;
+ if (aboveCrossingCount < belowCrossingCount) {
+ return aboveCrossingCount;
+ } else {
+ return belowCrossingCount;
+ }
}
@Override
@@ -822,131 +833,29 @@ class GeoComplexPolygon extends GeoBasePolygon {
if (edge.plane.evaluateIsZero(thePointX, thePointY, thePointZ) && edge.startPlane.isWithin(thePointX, thePointY, thePointZ) && edge.endPlane.isWithin(thePointX, thePointY, thePointZ)) {
return false;
}
- final GeoPoint[] crossingPoints = plane.findCrossings(planetModel, edge.plane, bound, edge.startPlane, edge.endPlane);
- if (crossingPoints != null) {
- // We need to handle the endpoint case, which is quite tricky.
- for (final GeoPoint crossingPoint : crossingPoints) {
- countCrossingPoint(crossingPoint, edge);
- }
+
+ // This should precisely mirror what is in DualCrossingIterator, but without the dual crossings.
+ // Some edges are going to be given to us even when there's no real intersection, so do that as a sanity check, first.
+ final GeoPoint[] planeCrossings = plane.findIntersections(planetModel, edge.plane, bound, edge.startPlane, edge.endPlane);
+ if (planeCrossings != null && planeCrossings.length == 0) {
+ // No actual crossing
+ return true;
}
+
+ // Determine crossings of this edge against all inside/outside planes. There's no further need to look at the actual travel plane itself.
+ final GeoPoint[] aboveCrossings = abovePlane.findCrossings(planetModel, edge.plane, bound, edge.startPlane, edge.endPlane);
+ final GeoPoint[] belowCrossings = belowPlane.findCrossings(planetModel, edge.plane, bound, edge.startPlane, edge.endPlane);
+
+ if (aboveCrossings != null) {
+ aboveCrossingCount += aboveCrossings.length;
+ }
+ if (belowCrossings != null) {
+ belowCrossingCount += belowCrossings.length;
+ }
+
return true;
}
- private void countCrossingPoint(final GeoPoint crossingPoint, final Edge edge) {
- if (crossingPoint.isNumericallyIdentical(edge.startPoint)) {
- // We have to figure out if this crossing should be counted.
-
- // Does the crossing for this edge go up, or down? Or can't we tell?
- final GeoPoint[] aboveIntersections = abovePlane.findIntersections(planetModel, edge.plane, edge.startPlane, edge.endPlane);
- final GeoPoint[] belowIntersections = belowPlane.findIntersections(planetModel, edge.plane, edge.startPlane, edge.endPlane);
-
- assert !(aboveIntersections.length > 0 && belowIntersections.length > 0) : "edge that ends in a crossing can't both up and down";
-
- if (aboveIntersections.length == 0 && belowIntersections.length == 0) {
- return;
- }
-
- final boolean edgeCrossesAbove = aboveIntersections.length > 0;
-
- // This depends on the previous edge that first departs from identicalness.
- Edge assessEdge = edge;
- GeoPoint[] assessAboveIntersections;
- GeoPoint[] assessBelowIntersections;
- while (true) {
- assessEdge = assessEdge.previous;
- assessAboveIntersections = abovePlane.findIntersections(planetModel, assessEdge.plane, assessEdge.startPlane, assessEdge.endPlane);
- assessBelowIntersections = belowPlane.findIntersections(planetModel, assessEdge.plane, assessEdge.startPlane, assessEdge.endPlane);
-
- assert !(assessAboveIntersections.length > 0 && assessBelowIntersections.length > 0) : "assess edge that ends in a crossing can't both up and down";
-
- if (assessAboveIntersections.length == 0 && assessBelowIntersections.length == 0) {
- continue;
- }
- break;
- }
-
- // Basically, we now want to assess whether both edges that come together at this endpoint leave the plane in opposite
- // directions. If they do, then we should count it as a crossing; if not, we should not. We also have to remember that
- // each edge we look at can also be looked at again if it, too, seems to cross the plane.
-
- // To handle the latter situation, we need to know if the other edge will be looked at also, and then we can make
- // a decision whether to count or not based on that.
-
- // Compute the crossing points of this other edge.
- final GeoPoint[] otherCrossingPoints = plane.findCrossings(planetModel, assessEdge.plane, bound, assessEdge.startPlane, assessEdge.endPlane);
-
- // Look for a matching endpoint. If the other endpoint doesn't show up, it is either out of bounds (in which case the
- // transition won't be counted for that edge), or it is not a crossing for that edge (so, same conclusion).
- for (final GeoPoint otherCrossingPoint : otherCrossingPoints) {
- if (otherCrossingPoint.isNumericallyIdentical(assessEdge.endPoint)) {
- // Found it!
- // Both edges will try to contribute to the crossing count. By convention, we'll only include the earlier one.
- // Since we're the latter point, we exit here in that case.
- return;
- }
- }
-
- // Both edges will not count the same point, so we can proceed. We need to determine the direction of both edges at the
- // point where they hit the plane. This may be complicated by the 3D geometry; it may not be safe just to look at the endpoints of the edges
- // and make an assessment that way, since a single edge can intersect the plane at more than one point.
-
- final boolean assessEdgeAbove = assessAboveIntersections.length > 0;
- if (assessEdgeAbove != edgeCrossesAbove) {
- crossingCount++;
- }
-
- } else if (crossingPoint.isNumericallyIdentical(edge.endPoint)) {
- // Figure out if the crossing should be counted.
-
- // Does the crossing for this edge go up, or down? Or can't we tell?
- final GeoPoint[] aboveIntersections = abovePlane.findIntersections(planetModel, edge.plane, edge.startPlane, edge.endPlane);
- final GeoPoint[] belowIntersections = belowPlane.findIntersections(planetModel, edge.plane, edge.startPlane, edge.endPlane);
-
- assert !(aboveIntersections.length > 0 && belowIntersections.length > 0) : "edge that ends in a crossing can't both up and down";
-
- if (aboveIntersections.length == 0 && belowIntersections.length == 0) {
- return;
- }
-
- final boolean edgeCrossesAbove = aboveIntersections.length > 0;
-
- // This depends on the previous edge that first departs from identicalness.
- Edge assessEdge = edge;
- GeoPoint[] assessAboveIntersections;
- GeoPoint[] assessBelowIntersections;
- while (true) {
- assessEdge = assessEdge.next;
- assessAboveIntersections = abovePlane.findIntersections(planetModel, assessEdge.plane, assessEdge.startPlane, assessEdge.endPlane);
- assessBelowIntersections = belowPlane.findIntersections(planetModel, assessEdge.plane, assessEdge.startPlane, assessEdge.endPlane);
-
- assert !(assessAboveIntersections.length > 0 && assessBelowIntersections.length > 0) : "assess edge that ends in a crossing can't both up and down";
-
- if (assessAboveIntersections.length == 0 && assessBelowIntersections.length == 0) {
- continue;
- }
- break;
- }
-
- // Basically, we now want to assess whether both edges that come together at this endpoint leave the plane in opposite
- // directions. If they do, then we should count it as a crossing; if not, we should not. We also have to remember that
- // each edge we look at can also be looked at again if it, too, seems to cross the plane.
-
- // By definition, we're the earlier plane in this case, so any crossing we detect we must count, by convention. It is unnecessary
- // to consider what the other edge does, because when we get to it, it will look back and figure out what we did for this one.
-
- // We need to determine the direction of both edges at the
- // point where they hit the plane. This may be complicated by the 3D geometry; it may not be safe just to look at the endpoints of the edges
- // and make an assessment that way, since a single edge can intersect the plane at more than one point.
-
- final boolean assessEdgeAbove = assessAboveIntersections.length > 0;
- if (assessEdgeAbove != edgeCrossesAbove) {
- crossingCount++;
- }
-
- } else {
- crossingCount++;
- }
- }
}
/** Create a linear crossing edge iterator with the appropriate cutoff planes given the geometry.
@@ -980,7 +889,8 @@ class GeoComplexPolygon extends GeoBasePolygon {
private final double thePointY;
private final double thePointZ;
- private int crossingCount = 0;
+ private int aboveCrossingCount = 0;
+ private int belowCrossingCount = 0;
public SectorLinearCrossingEdgeIterator(final Plane plane, final Plane abovePlane, final Plane belowPlane, final double thePointX, final double thePointY, final double thePointZ) {
this.plane = plane;
@@ -996,7 +906,11 @@ class GeoComplexPolygon extends GeoBasePolygon {
@Override
public int getCrossingCount() {
- return crossingCount;
+ if (aboveCrossingCount < belowCrossingCount) {
+ return aboveCrossingCount;
+ } else {
+ return belowCrossingCount;
+ }
}
@Override
@@ -1005,139 +919,38 @@ class GeoComplexPolygon extends GeoBasePolygon {
if (edge.plane.evaluateIsZero(thePointX, thePointY, thePointZ) && edge.startPlane.isWithin(thePointX, thePointY, thePointZ) && edge.endPlane.isWithin(thePointX, thePointY, thePointZ)) {
return false;
}
- final GeoPoint[] crossingPoints = plane.findCrossings(planetModel, edge.plane, bound1, bound2, edge.startPlane, edge.endPlane);
- if (crossingPoints != null) {
- // We need to handle the endpoint case, which is quite tricky.
- for (final GeoPoint crossingPoint : crossingPoints) {
- countCrossingPoint(crossingPoint, edge);
- }
+
+ // This should precisely mirror what is in DualCrossingIterator, but without the dual crossings.
+ // Some edges are going to be given to us even when there's no real intersection, so do that as a sanity check, first.
+ final GeoPoint[] planeCrossings = plane.findIntersections(planetModel, edge.plane, bound1, bound2, edge.startPlane, edge.endPlane);
+ if (planeCrossings != null && planeCrossings.length == 0) {
+ // No actual crossing
+ return true;
+ }
+
+ // Determine crossings of this edge against all inside/outside planes. There's no further need to look at the actual travel plane itself.
+ final GeoPoint[] aboveCrossings = abovePlane.findCrossings(planetModel, edge.plane, bound1, bound2, edge.startPlane, edge.endPlane);
+ final GeoPoint[] belowCrossings = belowPlane.findCrossings(planetModel, edge.plane, bound1, bound2, edge.startPlane, edge.endPlane);
+
+ if (aboveCrossings != null) {
+ aboveCrossingCount += aboveCrossings.length;
+ }
+ if (belowCrossings != null) {
+ belowCrossingCount += belowCrossings.length;
}
+
return true;
}
- private void countCrossingPoint(final GeoPoint crossingPoint, final Edge edge) {
- if (crossingPoint.isNumericallyIdentical(edge.startPoint)) {
-
- // The crossing point is shared by two edges. If we are going to count it, this is the edge we'll count it on.
- // We have to figure out if this crossing should be counted.
-
- // We look at the above plane and the below plane and see if we cross either of them.
- // If we cross NEITHER of them: we're in the "zone" between the planes, and this edge doesn't count.
-
- // Does the crossing for this edge go up, or down? Or can't we tell?
- final GeoPoint[] aboveIntersections = abovePlane.findCrossings(planetModel, edge.plane, edge.startPlane, edge.endPlane);
- final GeoPoint[] belowIntersections = belowPlane.findCrossings(planetModel, edge.plane, edge.startPlane, edge.endPlane);
-
- if ((aboveIntersections == null || aboveIntersections.length == 0) && (belowIntersections == null || belowIntersections.length == 0)) {
- return;
- }
-
- // A null value means we have a situation where the edge is numerically identical. That's not counted as a "crossing".
-
- assert !(aboveIntersections != null && aboveIntersections.length > 0 && belowIntersections != null && belowIntersections.length > 0) : "edge that ends in a crossing can't be both up and down!";
-
- final boolean edgeCrossesAbove = aboveIntersections != null && aboveIntersections.length > 0;
-
- // This depends on the previous edge that first departs from identicalness.
- Edge assessEdge = edge;
- GeoPoint[] assessAboveIntersections;
- GeoPoint[] assessBelowIntersections;
- while (true) {
- assessEdge = assessEdge.previous;
- assessAboveIntersections = abovePlane.findCrossings(planetModel, assessEdge.plane, assessEdge.startPlane, assessEdge.endPlane);
- assessBelowIntersections = belowPlane.findCrossings(planetModel, assessEdge.plane, assessEdge.startPlane, assessEdge.endPlane);
-
- if ((assessAboveIntersections == null || assessAboveIntersections.length == 0) && (assessBelowIntersections == null || assessBelowIntersections.length == 0)) {
- continue;
- }
- break;
- }
-
- // Basically, we now want to assess whether both edges that come together at this endpoint leave the plane in opposite
- // directions. If they do, then we should count it as a crossing; if not, we should not. We also have to remember that
- // each edge we look at can also be looked at again if it, too, seems to cross the plane.
-
- // To handle the latter situation, we need to know if the other edge will be looked at also, and then we can make
- // a decision whether to count or not based on that.
-
- // Compute the crossing points of this other edge.
- final GeoPoint[] otherCrossingPoints = plane.findCrossings(planetModel, assessEdge.plane, bound1, bound2, assessEdge.startPlane, assessEdge.endPlane);
-
- // Look for a matching endpoint. If the other endpoint doesn't show up, it is either out of bounds (in which case the
- // transition won't be counted for that edge), or it is not a crossing for that edge (so, same conclusion).
- for (final GeoPoint otherCrossingPoint : otherCrossingPoints) {
- if (otherCrossingPoint.isNumericallyIdentical(assessEdge.endPoint)) {
- // Found it!
- // Both edges will try to contribute to the crossing count. By convention, we'll only include the earlier one.
- // Since we're the latter point, we exit here in that case.
- return;
- }
- }
-
- // Both edges will not count the same point, so we can proceed. We need to determine the direction of both edges at the
- // point where they hit the plane. This may be complicated by the 3D geometry; it may not be safe just to look at the endpoints of the edges
- // and make an assessment that way, since a single edge can intersect the plane at more than one point.
-
- final boolean assessEdgeAbove = assessAboveIntersections != null && assessAboveIntersections.length > 0;
- if (assessEdgeAbove != edgeCrossesAbove) {
- crossingCount++;
- }
-
- } else if (crossingPoint.isNumericallyIdentical(edge.endPoint)) {
- // Figure out if the crossing should be counted.
-
- // Does the crossing for this edge go up, or down? Or can't we tell?
- final GeoPoint[] aboveIntersections = abovePlane.findCrossings(planetModel, edge.plane, edge.startPlane, edge.endPlane);
- final GeoPoint[] belowIntersections = belowPlane.findCrossings(planetModel, edge.plane, edge.startPlane, edge.endPlane);
-
- if ((aboveIntersections == null || aboveIntersections.length == 0) && (belowIntersections == null || belowIntersections.length == 0)) {
- return;
- }
-
- final boolean edgeCrossesAbove = aboveIntersections != null && aboveIntersections.length > 0;
-
- // This depends on the previous edge that first departs from identicalness.
- Edge assessEdge = edge;
- GeoPoint[] assessAboveIntersections;
- GeoPoint[] assessBelowIntersections;
- while (true) {
- assessEdge = assessEdge.next;
- assessAboveIntersections = abovePlane.findCrossings(planetModel, assessEdge.plane, assessEdge.startPlane, assessEdge.endPlane);
- assessBelowIntersections = belowPlane.findCrossings(planetModel, assessEdge.plane, assessEdge.startPlane, assessEdge.endPlane);
-
- if (assessAboveIntersections != null && assessAboveIntersections.length == 0 && assessBelowIntersections != null && assessBelowIntersections.length == 0) {
- continue;
- }
- break;
- }
-
- // Basically, we now want to assess whether both edges that come together at this endpoint leave the plane in opposite
- // directions. If they do, then we should count it as a crossing; if not, we should not. We also have to remember that
- // each edge we look at can also be looked at again if it, too, seems to cross the plane.
-
- // By definition, we're the earlier plane in this case, so any crossing we detect we must count, by convention. It is unnecessary
- // to consider what the other edge does, because when we get to it, it will look back and figure out what we did for this one.
-
- // We need to determine the direction of both edges at the
- // point where they hit the plane. This may be complicated by the 3D geometry; it may not be safe just to look at the endpoints of the edges
- // and make an assessment that way, since a single edge can intersect the plane at more than one point.
-
- final boolean assessEdgeAbove = assessAboveIntersections != null && assessAboveIntersections.length > 0;
- if (assessEdgeAbove != edgeCrossesAbove) {
- crossingCount++;
- }
-
- } else {
- crossingCount++;
- }
- }
}
/** Count the number of verifiable edge crossings for a dual-leg journey.
*/
private class DualCrossingEdgeIterator implements EdgeIterator {
- private boolean isSecondLeg = false;
+ // This is a hash of which edges we've already looked at and tallied, so we don't repeat ourselves.
+ // It is lazily initialized since most transitions cross no edges at all.
+ private Set<Edge> seenEdges = null;
private final Plane testPointPlane;
private final Plane testPointAbovePlane;
@@ -1163,10 +976,12 @@ class GeoComplexPolygon extends GeoBasePolygon {
private Plane travelOutsidePlane;
private SidedPlane insideTestPointCutoffPlane;
private SidedPlane insideTravelCutoffPlane;
+ private SidedPlane outsideTestPointCutoffPlane;
+ private SidedPlane outsideTravelCutoffPlane;
- // The counter
-
- public int crossingCount = 0;
+ // The counters
+ public int innerCrossingCount = 0;
+ public int outerCrossingCount = 0;
public DualCrossingEdgeIterator(final Plane testPointPlane, final Plane testPointAbovePlane, final Plane testPointBelowPlane,
final Plane travelPlane, final double thePointX, final double thePointY, final double thePointZ, final GeoPoint intersectionPoint) {
@@ -1179,7 +994,7 @@ class GeoComplexPolygon extends GeoBasePolygon {
this.thePointZ = thePointZ;
this.intersectionPoint = intersectionPoint;
- //System.err.println("Intersection point = "+intersectionPoint);
+ //System.out.println("Intersection point = "+intersectionPoint);
assert travelPlane.evaluateIsZero(intersectionPoint) : "intersection point must be on travel plane";
assert testPointPlane.evaluateIsZero(intersectionPoint) : "intersection point must be on test point plane";
@@ -1216,6 +1031,9 @@ class GeoComplexPolygon extends GeoBasePolygon {
final Plane travelAbovePlane = new Plane(travelPlane, true);
final Plane travelBelowPlane = new Plane(travelPlane, false);
+ // Each of these can generate two solutions. We need to refine them to generate only one somehow -- the one in the same area of the world as intersectionPoint.
+ // Since the travel/testpoint planes have one fixed coordinate, and that is represented by the plane's D value, it should be possible to choose based on the
+ // point's coordinates.
final GeoPoint[] aboveAbove = travelAbovePlane.findIntersections(planetModel, testPointAbovePlane, intersectionBound1, intersectionBound2);
assert aboveAbove != null : "Above + above should not be coplanar";
final GeoPoint[] aboveBelow = travelAbovePlane.findIntersections(planetModel, testPointBelowPlane, intersectionBound1, intersectionBound2);
@@ -1227,265 +1045,197 @@ class GeoComplexPolygon extends GeoBasePolygon {
assert ((aboveAbove.length > 0)?1:0) + ((aboveBelow.length > 0)?1:0) + ((belowBelow.length > 0)?1:0) + ((belowAbove.length > 0)?1:0) == 1 : "Can be exactly one inside point, instead was: aa="+aboveAbove.length+" ab=" + aboveBelow.length+" bb="+ belowBelow.length+" ba=" + belowAbove.length;
+ final GeoPoint[] insideInsidePoints;
if (aboveAbove.length > 0) {
travelInsidePlane = travelAbovePlane;
testPointInsidePlane = testPointAbovePlane;
travelOutsidePlane = travelBelowPlane;
testPointOutsidePlane = testPointBelowPlane;
+ insideInsidePoints = aboveAbove;
} else if (aboveBelow.length > 0) {
travelInsidePlane = travelAbovePlane;
testPointInsidePlane = testPointBelowPlane;
travelOutsidePlane = travelBelowPlane;
testPointOutsidePlane = testPointAbovePlane;
+ insideInsidePoints = aboveBelow;
} else if (belowBelow.length > 0) {
travelInsidePlane = travelBelowPlane;
testPointInsidePlane = testPointBelowPlane;
travelOutsidePlane = travelAbovePlane;
testPointOutsidePlane = testPointAbovePlane;
+ insideInsidePoints = belowBelow;
} else {
travelInsidePlane = travelBelowPlane;
testPointInsidePlane = testPointAbovePlane;
travelOutsidePlane = travelAbovePlane;
testPointOutsidePlane = testPointBelowPlane;
+ insideInsidePoints = belowAbove;
}
- insideTravelCutoffPlane = new SidedPlane(thePointX, thePointY, thePointZ, testPointInsidePlane, testPointInsidePlane.D);
- insideTestPointCutoffPlane = new SidedPlane(testPoint, travelInsidePlane, travelInsidePlane.D);
+ // Get the inside-inside intersection point
+ // Picking which point, out of two, that corresponds to the already-selected intersectionPoint, is tricky, but it must be done.
+ // We expect the choice to be within a small delta of the intersection point in 2 of the dimensions, but not the third
+ final GeoPoint insideInsidePoint = pickProximate(insideInsidePoints);
+
+ // Get the outside-outside intersection point
+ final GeoPoint[] outsideOutsidePoints = testPointOutsidePlane.findIntersections(planetModel, travelOutsidePlane); //these don't add anything: , checkPointCutoffPlane, testPointCutoffPlane);
+ final GeoPoint outsideOutsidePoint = pickProximate(outsideOutsidePoints);
+
+ insideTravelCutoffPlane = new SidedPlane(thePointX, thePointY, thePointZ, travelInsidePlane, insideInsidePoint);
+ outsideTravelCutoffPlane = new SidedPlane(thePointX, thePointY, thePointZ, travelInsidePlane, outsideOutsidePoint);
+ insideTestPointCutoffPlane = new SidedPlane(testPoint, testPointInsidePlane, insideInsidePoint);
+ outsideTestPointCutoffPlane = new SidedPlane(testPoint, testPointOutsidePlane, outsideOutsidePoint);
+
+ /*
+ System.out.println("insideTravelCutoffPlane = "+insideTravelCutoffPlane);
+ System.out.println("outsideTravelCutoffPlane = "+outsideTravelCutoffPlane);
+ System.out.println("insideTestPointCutoffPlane = "+insideTestPointCutoffPlane);
+ System.out.println("outsideTestPointCutoffPlane = "+outsideTestPointCutoffPlane);
+ */
+
computedInsideOutside = true;
}
}
- public void setSecondLeg() {
- isSecondLeg = true;
+ private GeoPoint pickProximate(final GeoPoint[] points) {
+ if (points.length == 0) {
+ throw new IllegalArgumentException("No off-plane intersection points were found; can't compute traversal");
+ } else if (points.length == 1) {
+ return points[0];
+ } else {
+ final double p1dist = computeSquaredDistance(points[0], intersectionPoint);
+ final double p2dist = computeSquaredDistance(points[1], intersectionPoint);
+ if (p1dist < p2dist) {
+ return points[0];
+ } else if (p2dist < p1dist) {
+ return points[1];
+ } else {
+ throw new IllegalArgumentException("Neither off-plane intersection point matched intersection point; intersection = "+intersectionPoint+"; offplane choice 0: "+points[0]+"; offplane choice 1: "+points[1]);
+ }
+ }
+ }
+
+ public int getCrossingCount() {
+ // Doesn't return the actual crossing count -- just gets the even/odd part right
+ if (innerCrossingCount < outerCrossingCount) {
+ return innerCrossingCount;
+ } else {
+ return outerCrossingCount;
+ }
}
@Override
public boolean matches(final Edge edge) {
- //System.err.println("Processing edge "+edge+", startpoint="+edge.startPoint+" endpoint="+edge.endPoint);
- // Early exit if the point is on the edge.
+ // Early exit if the point is on the edge, in which case we accidentally discovered the answer.
if (edge.plane.evaluateIsZero(thePointX, thePointY, thePointZ) && edge.startPlane.isWithin(thePointX, thePointY, thePointZ) && edge.endPlane.isWithin(thePointX, thePointY, thePointZ)) {
- //System.err.println(" Check point is on edge: isWithin = true");
return false;
}
- // If the intersection point lies on this edge, we should still be able to consider crossing points only.
- // Even if an intersection point is eliminated because it's not a crossing of one plane, it will have to be a crossing
- // for at least one of the two planes in order to be a legitimate crossing of the combined path.
- final GeoPoint[] crossingPoints;
- if (isSecondLeg) {
- //System.err.println(" check point plane = "+travelPlane);
- crossingPoints = travelPlane.findCrossings(planetModel, edge.plane, checkPointCutoffPlane, checkPointOtherCutoffPlane, edge.startPlane, edge.endPlane);
- } else {
- //System.err.println(" test point plane = "+testPointPlane);
- crossingPoints = testPointPlane.findCrossings(planetModel, edge.plane, testPointCutoffPlane, testPointOtherCutoffPlane, edge.startPlane, edge.endPlane);
+
+ // All edges that touch the travel planes get assessed the same. So, for each intersecting edge on both legs:
+ // (1) If the edge contains the intersection point, we analyze it on only one leg. For the other leg, we do nothing.
+ // (2) We compute the crossings of the edge with ALL FOUR inner and outer bounding planes.
+ // (3) We add the numbers of each kind of crossing to the total for that class of crossing (innerTotal and outerTotal).
+ // (4) When done all edges tallied in this way, we take min(innerTotal, outerTotal) and assume that is the number of crossings.
+ //
+ // Q: What if we see the same edge in both traversals?
+ // A: We should really evaluate it only in one. Keep a hash of the edges we've looked at already and don't process edges twice.
+
+ // Every edge should be looked at only once.
+ if (seenEdges != null && seenEdges.contains(edge)) {
+ return true;
}
- if (crossingPoints != null) {
- // We need to handle the endpoint case, which is quite tricky.
- for (final GeoPoint crossingPoint : crossingPoints) {
- countCrossingPoint(crossingPoint, edge);
- }
- //System.err.println(" All crossing points processed");
- } else {
- //System.err.println(" No crossing points!");
+ if (seenEdges == null) {
+ seenEdges = new HashSet<>();
}
- return true;
- }
-
- private void countCrossingPoint(final GeoPoint crossingPoint, final Edge edge) {
- //System.err.println(" Crossing point "+crossingPoint);
- // We consider crossing points only in this method.
- // Unlike the linear case, there are additional cases when:
- // (1) The crossing point and the intersection point are the same, but are not the endpoint of an edge;
- // (2) The crossing point and the intersection point are the same, and they *are* the endpoint of an edge.
- // The other logical difference is that crossings of all kinds have to be considered so that:
- // (a) both inside edges are considered together at all times;
- // (b) both outside edges are considered together at all times;
- // (c) inside edge crossings that are between the other leg's inside and outside edge are ignored.
+ seenEdges.add(edge);
- // Intersection point crossings are either simple, or a crossing on an endpoint.
- // In either case, we have to be sure to count each edge only once, since it might appear in both the
- // first leg and the second. If the first leg can process it, it should, and the second should skip it.
- if (crossingPoint.isNumericallyIdentical(intersectionPoint)) {
- //System.err.println(" Crosses intersection point.");
- if (isSecondLeg) {
- // See whether this edge would have been processed in the first leg; if so, we skip it.
- final GeoPoint[] firstLegCrossings = testPointPlane.findCrossings(planetModel, edge.plane, testPointCutoffPlane, testPointOtherCutoffPlane, edge.startPlane, edge.endPlane);
- for (final GeoPoint firstLegCrossing : firstLegCrossings) {
- if (firstLegCrossing.isNumericallyIdentical(intersectionPoint)) {
- // We already processed it, so we're done here.
- //System.err.println(" Already processed on previous leg: exit");
- return;
- }
- }
- }
- }
-
- // Plane crossing, either first leg or second leg
+ //System.out.println("Considering edge "+(edge.startPoint)+" -> "+(edge.endPoint));
- if (crossingPoint.isNumericallyIdentical(edge.startPoint)) {
- //System.err.println(" Crossing point = edge.startPoint");
- // We have to figure out if this crossing should be counted.
- computeInsideOutside();
-
- // Does the crossing for this edge go up, or down? Or can't we tell?
- final GeoPoint[] insideTestPointPlaneIntersections = testPointInsidePlane.findCrossings(planetModel, edge.plane, edge.startPlane, edge.endPlane, insideTestPointCutoffPlane);
- final GeoPoint[] insideTravelPlaneIntersections = travelInsidePlane.findCrossings(planetModel, edge.plane, edge.startPlane, edge.endPlane, insideTravelCutoffPlane);
- final GeoPoint[] outsideTestPointPlaneIntersections = testPointOutsidePlane.findCrossings(planetModel, edge.plane, edge.startPlane, edge.endPlane);
- final GeoPoint[] outsideTravelPlaneIntersections = travelOutsidePlane.findCrossings(planetModel, edge.plane, edge.startPlane, edge.endPlane);
-
- if ((insideTestPointPlaneIntersections == null || insideTestPointPlaneIntersections.length == 0) &&
- (insideTravelPlaneIntersections == null || insideTravelPlaneIntersections.length == 0) &&
- (outsideTestPointPlaneIntersections == null || outsideTestPointPlaneIntersections.length == 0) &&
- (outsideTravelPlaneIntersections == null || outsideTravelPlaneIntersections.length == 0)) {
- //System.err.println(" No inside or outside crossings found");
- return;
- }
-
- final boolean edgeCrossesInside = insideTestPointPlaneIntersections.length + insideTravelPlaneIntersections.length > 0;
+ // We've never seen this edge before. Evaluate it in the context of inner and outer planes.
+ computeInsideOutside();
- // This depends on the previous edge that first departs from identicalness.
- Edge assessEdge = edge;
- GeoPoint[] assessInsideTestPointIntersections;
- GeoPoint[] assessInsideTravelIntersections;
- GeoPoint[] assessOutsideTestPointIntersections;
- GeoPoint[] assessOutsideTravelIntersections;
- while (true) {
- assessEdge = assessEdge.previous;
- assessInsideTestPointIntersections = testPointInsidePlane.findIntersections(planetModel, assessEdge.plane, assessEdge.startPlane, assessEdge.endPlane, insideTestPointCutoffPlane);
- assessInsideTravelIntersections = travelInsidePlane.findIntersections(planetModel, assessEdge.plane, assessEdge.startPlane, assessEdge.endPlane, insideTravelCutoffPlane);
- assessOutsideTestPointIntersections = testPointOutsidePlane.findIntersections(planetModel, assessEdge.plane, assessEdge.startPlane, assessEdge.endPlane);
- assessOutsideTravelIntersections = travelOutsidePlane.findIntersections(planetModel, assessEdge.plane, assessEdge.startPlane, assessEdge.endPlane);
-
- // If the assess edge is numerically identical to the edge we're trying to find the intersections with, there's not really a crossing, so count it as zero.
-
- if ((assessInsideTestPointIntersections == null || assessInsideTestPointIntersections.length == 0) &&
- (assessInsideTravelIntersections == null || assessInsideTravelIntersections.length == 0) &&
- (assessOutsideTestPointIntersections == null || assessOutsideTestPointIntersections.length == 0) &&
- (assessOutsideTravelIntersections == null || assessOutsideTravelIntersections.length == 0)) {
- continue;
- }
- break;
+ /*
+ System.out.println("\nThe following edges should intersect the travel/testpoint planes:");
+ Edge thisEdge = edge;
+ while (true) {
+ final GeoPoint[] travelCrossings = travelPlane.findIntersections(planetModel, thisEdge.plane, checkPointCutoffPlane, checkPointOtherCutoffPlane, thisEdge.startPlane, thisEdge.endPlane);
+ if (travelCrossings == null || travelCrossings.length > 0) {
+ System.out.println("Travel plane: "+thisEdge.startPoint+" -> "+thisEdge.endPoint);
}
-
- // Basically, we now want to assess whether both edges that come together at this endpoint leave the plane in opposite
- // directions. If they do, then we should count it as a crossing; if not, we should not. We also have to remember that
- // each edge we look at can also be looked at again if it, too, seems to cross the plane.
-
- // To handle the latter situation, we need to know if the other edge will be looked at also, and then we can make
- // a decision whether to count or not based on that.
-
- // Compute the crossing points of this other edge.
- final GeoPoint[] otherCrossingPoints;
- if (isSecondLeg) {
- otherCrossingPoints = travelPlane.findCrossings(planetModel, assessEdge.plane, checkPointCutoffPlane, checkPointOtherCutoffPlane, assessEdge.startPlane, assessEdge.endPlane);
- } else {
- otherCrossingPoints = testPointPlane.findCrossings(planetModel, assessEdge.plane, testPointCutoffPlane, testPointOtherCutoffPlane, assessEdge.startPlane, assessEdge.endPlane);
- }
-
- if (otherCrossingPoints == null) {
- // The assessEdge plane is the same as the travel plane. We consider this the same as "no crossing".
- return;
+ final GeoPoint[] testPointCrossings = testPointPlane.findIntersections(planetModel, thisEdge.plane, testPointCutoffPlane, testPointOtherCutoffPlane, thisEdge.startPlane, thisEdge.endPlane);
+ if (testPointCrossings == null || testPointCrossings.length > 0) {
+ System.out.println("Test point plane: "+thisEdge.startPoint+" -> "+thisEdge.endPoint);
}
-
- // Look for a matching endpoint. If the other endpoint doesn't show up, it is either out of bounds (in which case the
- // transition won't be counted for that edge), or it is not a crossing for that edge (so, same conclusion).
- for (final GeoPoint otherCrossingPoint : otherCrossingPoints) {
- if (otherCrossingPoint.isNumericallyIdentical(assessEdge.endPoint)) {
- // Found it!
- // Both edges will try to contribute to the crossing count. By convention, we'll only include the earlier one.
- // Since we're the latter point, we exit here in that case.
- //System.err.println(" Earlier point fired, so this one shouldn't");
- return;
- }
+ thisEdge = thisEdge.next;
+ if (thisEdge == edge) {
+ break;
}
-
- // Both edges will not count the same point, so we can proceed. We need to determine the direction of both edges at the
- // point where they hit the plane. This may be complicated by the 3D geometry; it may not be safe just to look at the endpoints of the edges
- // and make an assessment that way, since a single edge can intersect the plane at more than one point.
-
- final boolean assessEdgeInside = (assessInsideTestPointIntersections != null && assessInsideTestPointIntersections.length > 0) ||
- (assessInsideTravelIntersections != null && assessInsideTravelIntersections.length > 0);
- if (assessEdgeInside != edgeCrossesInside) {
- //System.err.println(" Incrementing crossing count");
- crossingCount++;
- } else {
- //System.err.println(" Entered and exited on same side");
+ }
+ System.out.println("");
+ */
+
+ // Some edges are going to be given to us even when there's no real intersection, so do that as a sanity check, first.
+ final GeoPoint[] travelCrossings = travelPlane.findIntersections(planetModel, edge.plane, checkPointCutoffPlane, checkPointOtherCutoffPlane, edge.startPlane, edge.endPlane);
+ if (travelCrossings != null && travelCrossings.length == 0) {
+ final GeoPoint[] testPointCrossings = testPointPlane.findIntersections(planetModel, edge.plane, testPointCutoffPlane, testPointOtherCutoffPlane, edge.startPlane, edge.endPlane);
+ if (testPointCrossings != null && testPointCrossings.length == 0) {
+ return true;
}
-
- } else if (crossingPoint.isNumericallyIdentical(edge.endPoint)) {
- //System.err.println(" Crossing point = edge.endPoint");
- // Figure out if the crossing should be counted.
- computeInsideOutside();
-
- // If the assess edge is numerically identical to the edge we're trying to find the intersections with, there's not really a crossing, so count it as zero.
-
- // Does the crossing for this edge go up, or down? Or can't we tell?
- final GeoPoint[] insideTestPointPlaneIntersections = testPointInsidePlane.findIntersections(planetModel, edge.plane, edge.startPlane, edge.endPlane, insideTestPointCutoffPlane);
- final GeoPoint[] insideTravelPlaneIntersections = travelInsidePlane.findIntersections(planetModel, edge.plane, edge.startPlane, edge.endPlane, insideTravelCutoffPlane);
- final GeoPoint[] outsideTestPointPlaneIntersections = testPointOutsidePlane.findIntersections(planetModel, edge.plane, edge.startPlane, edge.endPlane);
- final GeoPoint[] outsideTravelPlaneIntersections = travelOutsidePlane.findIntersections(planetModel, edge.plane, edge.startPlane, edge.endPlane);
-
- // An edge can cross both outside and inside, because of the corner. But it can be considered to cross the inside ONLY if it crosses either of the inside edges.
-
- if ((insideTestPointPlaneIntersections == null || insideTestPointPlaneIntersections.length == 0) &&
- (insideTravelPlaneIntersections == null || insideTravelPlaneIntersections.length == 0) &&
- (outsideTestPointPlaneIntersections == null || outsideTestPointPlaneIntersections.length == 0) &&
- (outsideTravelPlaneIntersections == null || outsideTravelPlaneIntersections.length == 0)) {
- //System.err.println(" No inside or outside crossings found");
- return;
+ }
+
+ // Determine crossings of this edge against all inside/outside planes. There's no further need to look at the actual travel plane itself.
+ final GeoPoint[] travelInnerCrossings = travelInsidePlane.findCrossings(planetModel, edge.plane, checkPointCutoffPlane, insideTravelCutoffPlane, edge.startPlane, edge.endPlane);
+ final GeoPoint[] travelOuterCrossings = travelOutsidePlane.findCrossings(planetModel, edge.plane, checkPointCutoffPlane, outsideTravelCutoffPlane, edge.startPlane, edge.endPlane);
+ final GeoPoint[] testPointInnerCrossings = testPointInsidePlane.findCrossings(planetModel, edge.plane, testPointCutoffPlane, insideTestPointCutoffPlane, edge.startPlane, edge.endPlane);
+ final GeoPoint[] testPointOuterCrossings = testPointOutsidePlane.findCrossings(planetModel, edge.plane, testPointCutoffPlane, outsideTestPointCutoffPlane, edge.startPlane, edge.endPlane);
+
+ // If the edge goes through the inner-inner intersection point, or the outer-outer intersection point, we need to be sure we count that only once.
+ // It may appear in both lists. Use a hash for this right now.
+ final Set<GeoPoint> countingHash = new HashSet<>(2);
+
+ if (travelInnerCrossings != null) {
+ for (final GeoPoint crossing : travelInnerCrossings) {
+ //System.out.println(" Travel inner point "+crossing);
+ countingHash.add(crossing);
}
-
- final boolean edgeCrossesInside = (insideTestPointPlaneIntersections !=null && insideTestPointPlaneIntersections.length > 0) ||
- (insideTravelPlaneIntersections != null && insideTravelPlaneIntersections.length > 0);
-
- // This depends on the previous edge that first departs from identicalness.
- Edge assessEdge = edge;
- GeoPoint[] assessInsideTestPointIntersections;
- GeoPoint[] assessInsideTravelIntersections;
- GeoPoint[] assessOutsideTestPointIntersections;
- GeoPoint[] assessOutsideTravelIntersections;
- while (true) {
- assessEdge = assessEdge.next;
- assessInsideTestPointIntersections = testPointInsidePlane.findIntersections(planetModel, assessEdge.plane, assessEdge.startPlane, assessEdge.endPlane, insideTestPointCutoffPlane);
- assessInsideTravelIntersections = travelInsidePlane.findIntersections(planetModel, assessEdge.plane, assessEdge.startPlane, assessEdge.endPlane, insideTravelCutoffPlane);
- assessOutsideTestPointIntersections = testPointOutsidePlane.findIntersections(planetModel, assessEdge.plane, assessEdge.startPlane, assessEdge.endPlane);
- assessOutsideTravelIntersections = travelOutsidePlane.findIntersections(planetModel, assessEdge.plane, assessEdge.startPlane, assessEdge.endPlane);
-
- if ((assessInsideTestPointIntersections == null || assessInsideTestPointIntersections.length == 0) &&
- (assessInsideTravelIntersections == null || assessInsideTravelIntersections.length == 0) &&
- (assessOutsideTestPointIntersections == null || assessOutsideTestPointIntersections.length == 0) &&
- (assessOutsideTravelIntersections == null || assessOutsideTravelIntersections.length == 0)) {
- continue;
- }
- break;
+ }
+ if (testPointInnerCrossings != null) {
+ for (final GeoPoint crossing : testPointInnerCrossings) {
+ //System.out.println(" Test point inner point "+crossing);
+ countingHash.add(crossing);
}
-
- // Basically, we now want to assess whether both edges that come together at this endpoint leave the plane in opposite
- // directions. If they do, then we should count it as a crossing; if not, we should not. We also have to remember that
- // each edge we look at can also be looked at again if it, too, seems to cross the plane.
-
- // By definition, we're the earlier plane in this case, so any crossing we detect we must count, by convention. It is unnecessary
- // to consider what the other edge does, because when we get to it, it will look back and figure out what we did for this one.
-
- // We need to determine the direction of both edges at the
- // point where they hit the plane. This may be complicated by the 3D geometry; it may not be safe just to look at the endpoints of the edges
- // and make an assessment that way, since a single edge can intersect the plane at more than one point.
-
- final boolean assessEdgeInside = (assessInsideTestPointIntersections !=null && assessInsideTestPointIntersections.length > 0) ||
- (assessInsideTravelIntersections != null && assessInsideTravelIntersections.length > 0);
- if (assessEdgeInside != edgeCrossesInside) {
- //System.err.println(" Incrementing crossing count");
- crossingCount++;
- } else {
- //System.err.println(" Entered and exited on same side");
+ }
+ //System.out.println(" Edge added "+countingHash.size()+" to innerCrossingCount");
+ innerCrossingCount += countingHash.size();
+
+ countingHash.clear();
+ if (travelOuterCrossings != null) {
+ for (final GeoPoint crossing : travelOuterCrossings) {
+ //System.out.println(" Travel outer point "+crossing);
+ countingHash.add(crossing);
+ }
+ }
+ if (testPointOuterCrossings != null) {
+ for (final GeoPoint crossing : testPointOuterCrossings) {
+ //System.out.println(" Test point outer point "+crossing);
+ countingHash.add(crossing);
}
- } else {
- //System.err.println(" Not a special case: incrementing crossing count");
- // Not a special case, so we can safely count a crossing.
- crossingCount++;
}
+ //System.out.println(" Edge added "+countingHash.size()+" to outerCrossingCount");
+ outerCrossingCount += countingHash.size();
+
+ return true;
}
- }
+ }
+
+ private static double computeSquaredDistance(final GeoPoint checkPoint, final GeoPoint intersectionPoint) {
+ final double distanceX = checkPoint.x - intersectionPoint.x;
+ final double distanceY = checkPoint.y - intersectionPoint.y;
+ final double distanceZ = checkPoint.z - intersectionPoint.z;
+ return distanceX * distanceX + distanceY * distanceY + distanceZ * distanceZ;
+ }
+
@Override
public boolean equals(Object o) {
if (!(o instanceof GeoComplexPolygon))
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bc40f6c7/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java
index 38b4114..5d58d5e 100644
--- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java
+++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java
@@ -81,8 +81,6 @@ import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.TestUtil;
-import org.junit.Ignore;
-
import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
public class TestGeo3DPoint extends LuceneTestCase {
@@ -190,8 +188,7 @@ public class TestGeo3DPoint extends LuceneTestCase {
}
/** Tests consistency of GeoArea.getRelationship vs GeoShape.isWithin */
- //@AwaitsFix("https://issues.apache.org/jira/browse/LUCENE-8227")
- @Ignore
+ @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/LUCENE-8227")
public void testGeo3DRelations() throws Exception {
int numDocs = atLeast(1000);
@@ -471,22 +468,16 @@ public class TestGeo3DPoint extends LuceneTestCase {
}
}
- //@AwaitsFix("https://issues.apache.org/jira/browse/LUCENE-8227")
- @Ignore
public void testRandomTiny() throws Exception {
// Make sure single-leaf-node case is OK:
doTestRandom(10);
}
- //@AwaitsFix("https://issues.apache.org/jira/browse/LUCENE-8227")
- @Ignore
public void testRandomMedium() throws Exception {
doTestRandom(10000);
}
@Nightly
- //@AwaitsFix("https://issues.apache.org/jira/browse/LUCENE-8227")
- @Ignore
public void testRandomBig() throws Exception {
doTestRandom(50000);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bc40f6c7/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
index ebfb0f4..65659b3 100755
--- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
+++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
@@ -253,6 +253,7 @@ public class GeoPolygonTest {
shapes.add(pd);
c = GeoPolygonFactory.makeLargeGeoPolygon(PlanetModel.SPHERE, shapes);
+ //System.out.println("Large polygon = "+c);
// Sample some points within
gp = new GeoPoint(PlanetModel.SPHERE, 0.0, -0.45);
@@ -1217,7 +1218,6 @@ shape:
[junit4] 1> quantized=[X=-0.9951793580415914, Y=-0.10888987641797832, Z=-2.3309121299774915E-10]
*/
@Test
- @Ignore
public void testLUCENE8227() throws Exception {
List<GeoPoint> points = new ArrayList<>();
points.add(new GeoPoint(PlanetModel.WGS84, -0.63542308910253, 0.9853722928232957));
@@ -1227,9 +1227,11 @@ shape:
points.add(new GeoPoint(PlanetModel.WGS84, -1.2205765069413237, 3.141592653589793));
GeoPolygonFactory.PolygonDescription pd = new GeoPolygonFactory.PolygonDescription(points);
+ /*
for (int i = 0; i < points.size(); i++) {
System.out.println("Point "+i+": "+points.get(i));
}
+ */
final GeoPoint unquantized = new GeoPoint(PlanetModel.WGS84, -3.1780051348770987E-74, -3.032608859187692);
final GeoPoint quantized = new GeoPoint(-0.9951793580415914, -0.10888987641797832, -2.3309121299774915E-10);
@@ -1237,31 +1239,22 @@ shape:
final GeoPoint negativeX = new GeoPoint(PlanetModel.WGS84, 0.0, Math.PI);
final GeoPoint negativeY = new GeoPoint(PlanetModel.WGS84, 0.0, -Math.PI * 0.5);
- // Construct a standard polygon first to see what that does
+ // Construct a standard polygon first to see what that does. This winds up being a large polygon under the covers.
GeoPolygon standard = GeoPolygonFactory.makeGeoPolygon(PlanetModel.WGS84, pd);
- System.out.println("Standard polygon: "+standard);
-
// This shows y < 0 hemisphere is all in-set
//assertTrue(standard.isWithin(negativeY));
// This should be in-set too, but isn't!!
assertTrue(standard.isWithin(negativeX));
-/*
final XYZBounds standardBounds = new XYZBounds();
standard.getBounds(standardBounds);
final XYZSolid standardSolid = XYZSolidFactory.makeXYZSolid(PlanetModel.WGS84, standardBounds);
- System.out.println("Standard bounds: "+standardBounds);
+ // If within shape, should be within bounds
+ assertTrue(standard.isWithin(quantized)?standardSolid.isWithin(quantized):true);
+ assertTrue(standard.isWithin(unquantized)?standardSolid.isWithin(unquantized):true);
- assertFalse(standardSolid.isWithin(quantized));
- assertFalse(standardSolid.isWithin(unquantized));
-*/
- // Now, both points should also not be in the poly
- assertFalse(standard.isWithin(unquantized));
- assertFalse(standard.isWithin(quantized));
-
-
}
}
[04/34] lucene-solr:jira/solr-12095: LUCENE-8106: add missing import
Posted by sh...@apache.org.
LUCENE-8106: add missing import
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/1ce72537
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/1ce72537
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/1ce72537
Branch: refs/heads/jira/solr-12095
Commit: 1ce72537b8b7577657c275dd7a6bfbb081392575
Parents: 701af06
Author: Steve Rowe <sa...@apache.org>
Authored: Thu Mar 29 13:51:39 2018 -0400
Committer: Steve Rowe <sa...@apache.org>
Committed: Thu Mar 29 13:51:39 2018 -0400
----------------------------------------------------------------------
dev-tools/scripts/reproduceJenkinsFailures.py | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1ce72537/dev-tools/scripts/reproduceJenkinsFailures.py
----------------------------------------------------------------------
diff --git a/dev-tools/scripts/reproduceJenkinsFailures.py b/dev-tools/scripts/reproduceJenkinsFailures.py
index 55194a0..f36513b 100644
--- a/dev-tools/scripts/reproduceJenkinsFailures.py
+++ b/dev-tools/scripts/reproduceJenkinsFailures.py
@@ -14,6 +14,7 @@
# limitations under the License.
import argparse
+import http.client
import os
import re
import subprocess
[16/34] lucene-solr:jira/solr-12095: LUCENE-8227: Remove unused
import to make precommit happy
Posted by sh...@apache.org.
LUCENE-8227: Remove unused import to make precommit happy
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b78d9800
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b78d9800
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b78d9800
Branch: refs/heads/jira/solr-12095
Commit: b78d980097d41e48acbecbe305794103df119190
Parents: 83cca5c
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Fri Mar 30 16:54:44 2018 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Fri Mar 30 16:54:44 2018 +0530
----------------------------------------------------------------------
.../src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b78d9800/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
index 65659b3..45f297f 100755
--- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
+++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
@@ -22,7 +22,6 @@ import java.util.BitSet;
import java.util.Collections;
import org.junit.Test;
-import org.junit.Ignore;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
[24/34] lucene-solr:jira/solr-12095: SOLR-12066: Move test to
DeleteInactiveReplicaTest
Posted by sh...@apache.org.
SOLR-12066: Move test to DeleteInactiveReplicaTest
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ab092942
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ab092942
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ab092942
Branch: refs/heads/jira/solr-12095
Commit: ab092942cf621b39afaae0d8b370deb3e084388a
Parents: bd85fd3
Author: Cao Manh Dat <da...@apache.org>
Authored: Sat Mar 31 14:50:28 2018 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Sat Mar 31 14:50:28 2018 +0700
----------------------------------------------------------------------
.../solr/cloud/DeleteInactiveReplicaTest.java | 22 ++++++++-----
.../apache/solr/cloud/DeleteReplicaTest.java | 33 --------------------
2 files changed, 14 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ab092942/solr/core/src/test/org/apache/solr/cloud/DeleteInactiveReplicaTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/DeleteInactiveReplicaTest.java b/solr/core/src/test/org/apache/solr/cloud/DeleteInactiveReplicaTest.java
index 0f4ff48..33a1a55 100644
--- a/solr/core/src/test/org/apache/solr/cloud/DeleteInactiveReplicaTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/DeleteInactiveReplicaTest.java
@@ -17,6 +17,8 @@
package org.apache.solr.cloud;
import java.lang.invoke.MethodHandles;
+import java.nio.file.Files;
+import java.util.concurrent.TimeUnit;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
@@ -26,7 +28,11 @@ import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.core.CoreContainer;
+import org.apache.solr.common.util.TimeSource;
+import org.apache.solr.core.CoreDescriptor;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.util.FileUtils;
+import org.apache.solr.util.TimeOut;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
@@ -64,6 +70,10 @@ public class DeleteInactiveReplicaTest extends SolrCloudTestCase {
Slice shard = getRandomShard(collectionState);
Replica replica = getRandomReplica(shard);
JettySolrRunner jetty = cluster.getReplicaJetty(replica);
+ CoreDescriptor replicaCd;
+ try (SolrCore core = jetty.getCoreContainer().getCore(replica.getCoreName())) {
+ replicaCd = core.getCoreDescriptor();
+ }
cluster.stopJettySolrRunner(jetty);
waitForState("Expected replica " + replica.getName() + " on down node to be removed from cluster state", collectionName, (n, c) -> {
@@ -80,13 +90,9 @@ public class DeleteInactiveReplicaTest extends SolrCloudTestCase {
cluster.startJettySolrRunner(jetty);
log.info("restarted jetty");
-
- CoreContainer cc = jetty.getCoreContainer();
- CoreContainer.CoreLoadFailure loadFailure = cc.getCoreInitFailures().get(replica.getCoreName());
- assertNotNull("Deleted core was still loaded!", loadFailure);
- assertNotNull(loadFailure.exception.getCause());
- assertTrue("Unexpected load failure message: " + loadFailure.exception.getCause().getMessage(),
- loadFailure.exception.getCause().getMessage().contains("does not exist in shard"));
+ TimeOut timeOut = new TimeOut(60, TimeUnit.SECONDS, TimeSource.NANO_TIME);
+ timeOut.waitFor("Expected data dir and instance dir of " + replica.getName() + " is deleted", ()
+ -> !Files.exists(replicaCd.getInstanceDir()) && !FileUtils.fileExists(replicaCd.getDataDir()));
// Check that we can't create a core with no coreNodeName
try (SolrClient queryClient = getHttpSolrClient(jetty.getBaseUrl().toString())) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ab092942/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java b/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java
index 1a021d7..3208ebd 100644
--- a/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java
@@ -41,10 +41,7 @@ import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.TimeSource;
import org.apache.solr.common.util.Utils;
-import org.apache.solr.core.CoreDescriptor;
-import org.apache.solr.core.SolrCore;
import org.apache.solr.core.ZkContainer;
-import org.apache.solr.util.FileUtils;
import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.KeeperException;
import org.junit.BeforeClass;
@@ -156,36 +153,6 @@ public class DeleteReplicaTest extends SolrCloudTestCase {
}
@Test
- public void deleteReplicaOnDownNode() throws Exception {
- final String collectionName = "deleteReplicaOnDownNode";
- CollectionAdminRequest.createCollection(collectionName, "conf", 1, 2).process(cluster.getSolrClient());
- waitForState("Expected one shards with two replicas", collectionName, clusterShape(1, 2));
-
- Slice shard = getCollectionState(collectionName).getSlice("shard1");
- Replica replica = shard.getReplicas(rep -> !rep.getName().equals(shard.getLeader().getName())).get(0);
- JettySolrRunner replicaJetty = getJettyForReplica(replica);
- CoreDescriptor replicaCd;
- try (SolrCore core = replicaJetty.getCoreContainer().getCore(replica.getCoreName())) {
- replicaCd = core.getCoreDescriptor();
- }
- assertNotNull("Expected core descriptor of "+ replica.getName() + " is not null",replicaCd);
- String replicaJettyNodeName = replicaJetty.getNodeName();
-
- // shutdown node of a replica
- replicaJetty.stop();
- waitForNodeLeave(replicaJettyNodeName);
- waitForState("Expected one shards with one replica", collectionName, clusterShape(1, 1));
- CollectionAdminRequest.deleteReplica(collectionName, shard.getName(), replica.getName()).process(cluster.getSolrClient());
- waitForState("Expected only one replica left", collectionName, (liveNodes, collectionState) -> collectionState.getReplicas().size() == 1);
-
- // restart the test and make sure the data get deleted
- replicaJetty.start();
- TimeOut timeOut = new TimeOut(60, TimeUnit.SECONDS, TimeSource.NANO_TIME);
- timeOut.waitFor("Expected data dir and instance dir of " + replica.getName() + " is deleted", ()
- -> !Files.exists(replicaCd.getInstanceDir()) && !FileUtils.fileExists(replicaCd.getDataDir()));
- }
-
- @Test
public void deleteReplicaByCountForAllShards() throws Exception {
final String collectionName = "deleteByCountNew";
[22/34] lucene-solr:jira/solr-12095: SOLR-12133: Fix race conditions
that caused TriggerIntegrationTest.testEventQueue to fail
Posted by sh...@apache.org.
SOLR-12133: Fix race conditions that caused TriggerIntegrationTest.testEventQueue to fail
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/85decabe
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/85decabe
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/85decabe
Branch: refs/heads/jira/solr-12095
Commit: 85decabe46966ec3a73e80294fe33cfa862975fd
Parents: e3c67b1
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Fri Mar 30 22:53:55 2018 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Fri Mar 30 22:53:55 2018 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/85decabe/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 12bc25a..6864e0b 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -135,6 +135,8 @@ Other Changes
* SOLR-12152: Split up TriggerIntegrationTest into multiple tests to isolate and increase reliability. (shalin)
+* SOLR-12133: Fix race conditions that caused TriggerIntegrationTest.testEventQueue to fail. (Mark Miller, shalin)
+
================== 7.3.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
[15/34] lucene-solr:jira/solr-12095: SOLR-12133: Fix failures in
TriggerIntegrationTest.testEventQueue due to race conditions
Posted by sh...@apache.org.
SOLR-12133: Fix failures in TriggerIntegrationTest.testEventQueue due to race conditions
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/83cca5cd
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/83cca5cd
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/83cca5cd
Branch: refs/heads/jira/solr-12095
Commit: 83cca5cd06aa0fdd9ce98415d6e9a0b1ecdfc7f5
Parents: ae6d29f
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Fri Mar 30 16:42:35 2018 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Fri Mar 30 16:42:35 2018 +0530
----------------------------------------------------------------------
.../cloud/autoscaling/ScheduledTriggers.java | 13 +++++++++
.../autoscaling/TriggerIntegrationTest.java | 28 +++++++++++---------
.../apache/solr/common/util/ExecutorUtil.java | 4 +++
3 files changed, 32 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/83cca5cd/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java
index 0e21b04..28efe92 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java
@@ -66,6 +66,7 @@ import static org.apache.solr.common.params.AutoScalingParams.ACTION_THROTTLE_PE
import static org.apache.solr.common.params.AutoScalingParams.TRIGGER_COOLDOWN_PERIOD_SECONDS;
import static org.apache.solr.common.params.AutoScalingParams.TRIGGER_CORE_POOL_SIZE;
import static org.apache.solr.common.params.AutoScalingParams.TRIGGER_SCHEDULE_DELAY_SECONDS;
+import static org.apache.solr.common.util.ExecutorUtil.awaitTermination;
/**
* Responsible for scheduling active triggers, starting and stopping them and
@@ -497,9 +498,21 @@ public class ScheduledTriggers implements Closeable {
}
// shutdown and interrupt all running tasks because there's no longer any
// guarantee about cluster state
+ log.debug("Shutting down scheduled thread pool executor now");
scheduledThreadPoolExecutor.shutdownNow();
+
+ log.debug("Shutting down action executor now");
actionExecutor.shutdownNow();
+
listeners.close();
+
+ log.debug("Awaiting termination for action executor");
+ awaitTermination(actionExecutor);
+
+ log.debug("Awaiting termination for scheduled thread pool executor");
+ awaitTermination(scheduledThreadPoolExecutor);
+
+ log.debug("ScheduledTriggers closed completely");
}
private class TriggerWrapper implements Runnable, Closeable {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/83cca5cd/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
index 5dfe34c..2902c48 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
@@ -64,15 +64,16 @@ import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_P
public class TriggerIntegrationTest extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private static CountDownLatch actionConstructorCalled;
- private static CountDownLatch actionInitCalled;
- private static CountDownLatch triggerFiredLatch;
- private static int waitForSeconds = 1;
- private static CountDownLatch actionStarted;
- private static CountDownLatch actionInterrupted;
- private static CountDownLatch actionCompleted;
+ private static volatile CountDownLatch actionConstructorCalled;
+ private static volatile CountDownLatch actionInitCalled;
+ private static volatile CountDownLatch triggerFiredLatch;
+ private static volatile int waitForSeconds = 1;
+ private static volatile CountDownLatch actionStarted;
+ private static volatile CountDownLatch actionInterrupted;
+ private static volatile CountDownLatch actionCompleted;
private static AtomicBoolean triggerFired;
private static Set<TriggerEvent> events = ConcurrentHashMap.newKeySet();
+ public static volatile long eventQueueActionWait = 5000;
private static SolrCloudManager cloudManager;
// use the same time source as triggers use
@@ -166,6 +167,7 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
events.clear();
listenerEvents.clear();
lastActionExecutedAt.set(0);
+ eventQueueActionWait = 5000;
while (cluster.getJettySolrRunners().size() < 2) {
// perhaps a test stopped a node but didn't start it back
// lets start a node
@@ -415,14 +417,17 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
public void process(TriggerEvent event, ActionContext actionContext) {
log.info("-- event: " + event);
events.add(event);
+ long eventQueueActionWaitCopy = eventQueueActionWait;
getActionStarted().countDown();
try {
- Thread.sleep(eventQueueActionWait);
+ log.info("-- Going to sleep for {} ms", eventQueueActionWaitCopy);
+ Thread.sleep(eventQueueActionWaitCopy);
+ log.info("-- Woke up after sleeping for {} ms", eventQueueActionWaitCopy);
triggerFired.compareAndSet(false, true);
getActionCompleted().countDown();
} catch (InterruptedException e) {
+ log.info("-- Interrupted");
getActionInterrupted().countDown();
- return;
}
}
@@ -434,10 +439,7 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
}
}
- public static long eventQueueActionWait = 5000;
-
@Test
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
public void testEventQueue() throws Exception {
waitForSeconds = 1;
CloudSolrClient solrClient = cluster.getSolrClient();
@@ -471,6 +473,7 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
JettySolrRunner newNode = cluster.startJettySolrRunner();
boolean await = actionStarted.await(60, TimeUnit.SECONDS);
assertTrue("action did not start", await);
+ eventQueueActionWait = 1;
// event should be there
NodeAddedTrigger.NodeAddedEvent nodeAddedEvent = (NodeAddedTrigger.NodeAddedEvent) events.iterator().next();
assertNotNull(nodeAddedEvent);
@@ -478,7 +481,6 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
assertFalse(triggerFired.get());
events.clear();
actionStarted = new CountDownLatch(1);
- eventQueueActionWait = 1;
// kill overseer leader
cluster.stopJettySolrRunner(overseerLeaderIndex);
Thread.sleep(5000);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/83cca5cd/solr/solrj/src/java/org/apache/solr/common/util/ExecutorUtil.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/ExecutorUtil.java b/solr/solrj/src/java/org/apache/solr/common/util/ExecutorUtil.java
index a045726..7458016 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/ExecutorUtil.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/ExecutorUtil.java
@@ -73,6 +73,10 @@ public class ExecutorUtil {
public static void shutdownAndAwaitTermination(ExecutorService pool) {
pool.shutdown(); // Disable new tasks from being submitted
+ awaitTermination(pool);
+ }
+
+ public static void awaitTermination(ExecutorService pool) {
boolean shutdown = false;
while (!shutdown) {
try {
[12/34] lucene-solr:jira/solr-12095: SOLR-12152: Extracted
TriggerIntegrationTest.testEventFromRestoredState into its own test class
Posted by sh...@apache.org.
SOLR-12152: Extracted TriggerIntegrationTest.testEventFromRestoredState into its own test class
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0e5374e9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0e5374e9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0e5374e9
Branch: refs/heads/jira/solr-12095
Commit: 0e5374e9631c639da11a7183a2947f4641a28c41
Parents: 1aafc90
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Fri Mar 30 12:41:18 2018 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Fri Mar 30 12:41:18 2018 +0530
----------------------------------------------------------------------
.../autoscaling/RestoreTriggerStateTest.java | 169 +++++++++++++++++++
.../autoscaling/TriggerIntegrationTest.java | 53 ------
2 files changed, 169 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0e5374e9/solr/core/src/test/org/apache/solr/cloud/autoscaling/RestoreTriggerStateTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/RestoreTriggerStateTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/RestoreTriggerStateTest.java
new file mode 100644
index 0000000..a3417cf
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/RestoreTriggerStateTest.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud.autoscaling;
+
+import java.lang.invoke.MethodHandles;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.util.LogLevel;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.createAutoScalingRequest;
+import static org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.WAIT_FOR_DELTA_NANOS;
+import static org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.timeSource;
+
+/**
+ * Integration test to ensure that triggers can restore state from ZooKeeper after overseer restart
+ * so that events detected before restart are not lost.
+ *
+ * Added in SOLR-10515
+ */
+@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG")
+public class RestoreTriggerStateTest extends SolrCloudTestCase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private static CountDownLatch actionInitCalled;
+ private static CountDownLatch triggerFiredLatch;
+ private static AtomicBoolean triggerFired;
+ private static CountDownLatch actionConstructorCalled;
+ private static Set<TriggerEvent> events = ConcurrentHashMap.newKeySet();
+ private static int waitForSeconds = 1;
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(2)
+ .addConfig("conf", configset("cloud-minimal"))
+ .configure();
+ // disable .scheduled_maintenance
+ String suspendTriggerCommand = "{" +
+ "'suspend-trigger' : {'name' : '.scheduled_maintenance'}" +
+ "}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
+ SolrClient solrClient = cluster.getSolrClient();
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ actionInitCalled = new CountDownLatch(1);
+ triggerFiredLatch = new CountDownLatch(1);
+ actionConstructorCalled = new CountDownLatch(1);
+ triggerFired = new AtomicBoolean();
+ }
+
+ @Test
+ public void testEventFromRestoredState() throws Exception {
+ CloudSolrClient solrClient = cluster.getSolrClient();
+ String setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'node_added_triggerEFRS'," +
+ "'event' : 'nodeAdded'," +
+ "'waitFor' : '10s'," +
+ "'enabled' : true," +
+ "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
+ "}}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ if (!actionInitCalled.await(10, TimeUnit.SECONDS)) {
+ fail("The TriggerAction should have been created by now");
+ }
+
+ NamedList<Object> overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
+ String overseerLeader = (String) overSeerStatus.get("leader");
+ int overseerLeaderIndex = 0;
+ for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
+ JettySolrRunner jetty = cluster.getJettySolrRunner(i);
+ if (jetty.getNodeName().equals(overseerLeader)) {
+ overseerLeaderIndex = i;
+ break;
+ }
+ }
+
+ events.clear();
+
+ JettySolrRunner newNode = cluster.startJettySolrRunner();
+ boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
+ assertTrue("The trigger did not fire at all", await);
+ assertTrue(triggerFired.get());
+ // reset
+ triggerFired.set(false);
+ triggerFiredLatch = new CountDownLatch(1);
+ NodeAddedTrigger.NodeAddedEvent nodeAddedEvent = (NodeAddedTrigger.NodeAddedEvent) events.iterator().next();
+ assertNotNull(nodeAddedEvent);
+ List<String> nodeNames = (List<String>) nodeAddedEvent.getProperty(TriggerEvent.NODE_NAMES);
+ assertTrue(nodeNames.contains(newNode.getNodeName()));
+ // add a second node - state of the trigger will change but it won't fire for waitFor sec.
+ JettySolrRunner newNode2 = cluster.startJettySolrRunner();
+ Thread.sleep(10000);
+ // kill overseer leader
+ cluster.stopJettySolrRunner(overseerLeaderIndex);
+ await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
+ assertTrue("The trigger did not fire at all", await);
+ assertTrue(triggerFired.get());
+ }
+
+ public static class TestTriggerAction extends TriggerActionBase {
+
+ public TestTriggerAction() {
+ actionConstructorCalled.countDown();
+ }
+
+ @Override
+ public void process(TriggerEvent event, ActionContext actionContext) {
+ try {
+ if (triggerFired.compareAndSet(false, true)) {
+ events.add(event);
+ long currentTimeNanos = timeSource.getTimeNs();
+ long eventTimeNanos = event.getEventTime();
+ long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS;
+ if (currentTimeNanos - eventTimeNanos <= waitForNanos) {
+ fail(event.source + " was fired before the configured waitFor period");
+ }
+ triggerFiredLatch.countDown();
+ } else {
+ fail(event.source + " was fired more than once!");
+ }
+ } catch (Throwable t) {
+ log.debug("--throwable", t);
+ throw t;
+ }
+ }
+
+ @Override
+ public void init(Map<String, String> args) {
+ log.info("TestTriggerAction init");
+ actionInitCalled.countDown();
+ super.init(args);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0e5374e9/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
index f536633..5dfe34c 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
@@ -496,59 +496,6 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
assertTrue(triggerFired.get());
}
- @Test
- public void testEventFromRestoredState() throws Exception {
- CloudSolrClient solrClient = cluster.getSolrClient();
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_added_triggerEFRS'," +
- "'event' : 'nodeAdded'," +
- "'waitFor' : '10s'," +
- "'enabled' : true," +
- "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
- "}}";
- SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- if (!actionInitCalled.await(10, TimeUnit.SECONDS)) {
- fail("The TriggerAction should have been created by now");
- }
-
- NamedList<Object> overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
- String overseerLeader = (String) overSeerStatus.get("leader");
- int overseerLeaderIndex = 0;
- for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
- JettySolrRunner jetty = cluster.getJettySolrRunner(i);
- if (jetty.getNodeName().equals(overseerLeader)) {
- overseerLeaderIndex = i;
- break;
- }
- }
-
- events.clear();
-
- JettySolrRunner newNode = cluster.startJettySolrRunner();
- boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
- assertTrue("The trigger did not fire at all", await);
- assertTrue(triggerFired.get());
- // reset
- triggerFired.set(false);
- triggerFiredLatch = new CountDownLatch(1);
- NodeAddedTrigger.NodeAddedEvent nodeAddedEvent = (NodeAddedTrigger.NodeAddedEvent) events.iterator().next();
- assertNotNull(nodeAddedEvent);
- List<String> nodeNames = (List<String>)nodeAddedEvent.getProperty(TriggerEvent.NODE_NAMES);
- assertTrue(nodeNames.contains(newNode.getNodeName()));
- // add a second node - state of the trigger will change but it won't fire for waitFor sec.
- JettySolrRunner newNode2 = cluster.startJettySolrRunner();
- Thread.sleep(10000);
- // kill overseer leader
- cluster.stopJettySolrRunner(overseerLeaderIndex);
- await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
- assertTrue("The trigger did not fire at all", await);
- assertTrue(triggerFired.get());
- }
-
static Map<String, List<CapturedEvent>> listenerEvents = new HashMap<>();
static CountDownLatch listenerCreated = new CountDownLatch(1);
static boolean failDummyAction = false;
[34/34] lucene-solr:jira/solr-12095: SOLR-12095: Fixing compilation
errors after merging master
Posted by sh...@apache.org.
SOLR-12095: Fixing compilation errors after merging master
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d83fcbd1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d83fcbd1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d83fcbd1
Branch: refs/heads/jira/solr-12095
Commit: d83fcbd1f97115e4639268c9d0192c226202b288
Parents: 0796226
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Mon Apr 2 13:07:26 2018 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Mon Apr 2 13:07:26 2018 +0530
----------------------------------------------------------------------
.../NodeAddedTriggerIntegrationTest.java | 5 ++---
.../NodeLostTriggerIntegrationTest.java | 5 ++---
.../autoscaling/NodeMarkersRegistrationTest.java | 5 ++---
.../autoscaling/RestoreTriggerStateTest.java | 5 ++---
.../TriggerCooldownIntegrationTest.java | 4 ++--
.../TriggerSetPropertiesIntegrationTest.java | 19 ++++++++++++-------
6 files changed, 22 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d83fcbd1/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeAddedTriggerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeAddedTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeAddedTriggerIntegrationTest.java
index ecf2437..ddc56ec 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeAddedTriggerIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeAddedTriggerIntegrationTest.java
@@ -19,7 +19,6 @@ package org.apache.solr.cloud.autoscaling;
import java.lang.invoke.MethodHandles;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
@@ -291,10 +290,10 @@ public class NodeAddedTriggerIntegrationTest extends SolrCloudTestCase {
}
@Override
- public void init(Map<String, String> args) {
+ public void init() throws Exception {
log.info("TestTriggerAction init");
actionInitCalled.countDown();
- super.init(args);
+ super.init();
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d83fcbd1/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerIntegrationTest.java
index 6b1af65..b756dcd 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerIntegrationTest.java
@@ -19,7 +19,6 @@ package org.apache.solr.cloud.autoscaling;
import java.lang.invoke.MethodHandles;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
@@ -313,10 +312,10 @@ public class NodeLostTriggerIntegrationTest extends SolrCloudTestCase {
}
@Override
- public void init(Map<String, String> args) {
+ public void init() throws Exception {
log.info("TestTriggerAction init");
actionInitCalled.countDown();
- super.init(args);
+ super.init();
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d83fcbd1/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeMarkersRegistrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeMarkersRegistrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeMarkersRegistrationTest.java
index 38c2165..fe1c94f 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeMarkersRegistrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeMarkersRegistrationTest.java
@@ -20,7 +20,6 @@ package org.apache.solr.cloud.autoscaling;
import java.lang.invoke.MethodHandles;
import java.util.HashSet;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.concurrent.ConcurrentHashMap;
@@ -260,10 +259,10 @@ public class NodeMarkersRegistrationTest extends SolrCloudTestCase {
}
@Override
- public void init(Map<String, String> args) {
+ public void init() throws Exception {
log.info("TestEventMarkerAction init");
actionInitCalled.countDown();
- super.init(args);
+ super.init();
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d83fcbd1/solr/core/src/test/org/apache/solr/cloud/autoscaling/RestoreTriggerStateTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/RestoreTriggerStateTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/RestoreTriggerStateTest.java
index a3417cf..4949e6f 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/RestoreTriggerStateTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/RestoreTriggerStateTest.java
@@ -19,7 +19,6 @@ package org.apache.solr.cloud.autoscaling;
import java.lang.invoke.MethodHandles;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
@@ -160,10 +159,10 @@ public class RestoreTriggerStateTest extends SolrCloudTestCase {
}
@Override
- public void init(Map<String, String> args) {
+ public void init() throws Exception {
log.info("TestTriggerAction init");
actionInitCalled.countDown();
- super.init(args);
+ super.init();
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d83fcbd1/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerCooldownIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerCooldownIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerCooldownIntegrationTest.java
index 8d69bad..e6e4116 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerCooldownIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerCooldownIntegrationTest.java
@@ -214,10 +214,10 @@ public class TriggerCooldownIntegrationTest extends SolrCloudTestCase {
}
@Override
- public void init(Map<String, String> args) {
+ public void init() throws Exception {
log.info("TestTriggerAction init");
actionInitCalled.countDown();
- super.init(args);
+ super.init();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d83fcbd1/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerSetPropertiesIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerSetPropertiesIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerSetPropertiesIntegrationTest.java
index 5f4243f..0ee0e1c 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerSetPropertiesIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerSetPropertiesIntegrationTest.java
@@ -80,7 +80,7 @@ public class TriggerSetPropertiesIntegrationTest extends SolrCloudTestCase {
try (ScheduledTriggers scheduledTriggers = new ScheduledTriggers(resourceLoader, solrCloudManager)) {
AutoScalingConfig config = new AutoScalingConfig(Collections.emptyMap());
scheduledTriggers.setAutoScalingConfig(config);
- scheduledTriggers.add(new TriggerBase(TriggerEventType.NODELOST, "x", Collections.emptyMap(), resourceLoader, solrCloudManager) {
+ AutoScaling.Trigger t = new TriggerBase(TriggerEventType.NODELOST, "x") {
@Override
protected Map<String, Object> getState() {
return Collections.singletonMap("x", "y");
@@ -103,7 +103,10 @@ public class TriggerSetPropertiesIntegrationTest extends SolrCloudTestCase {
diff.set(timeSource.getTimeNs() - l);
getTriggerFiredLatch().countDown();
}
- });
+ };
+ t.configure(runner.getCoreContainer().getResourceLoader(), runner.getCoreContainer().getZkController().getSolrCloudManager(), Collections.emptyMap());
+ scheduledTriggers.add(t);
+
assertTrue(getTriggerFiredLatch().await(4, TimeUnit.SECONDS));
assertTrue(diff.get() - TimeUnit.SECONDS.toNanos(ScheduledTriggers.DEFAULT_SCHEDULED_TRIGGER_DELAY_SECONDS) >= 0);
@@ -125,7 +128,7 @@ public class TriggerSetPropertiesIntegrationTest extends SolrCloudTestCase {
final Set<String> triggerNames = Collections.synchronizedSet(new HashSet<>());
triggerFiredLatch = new CountDownLatch(8);
for (int i = 0; i < 8; i++) {
- triggerList.add(new MockTrigger(TriggerEventType.NODELOST, "x" + i, Collections.emptyMap(), resourceLoader, solrCloudManager) {
+ AutoScaling.Trigger trigger = new MockTrigger(TriggerEventType.NODELOST, "x" + i) {
@Override
public void run() {
try {
@@ -140,8 +143,10 @@ public class TriggerSetPropertiesIntegrationTest extends SolrCloudTestCase {
threadNames.add(Thread.currentThread().getName());
}
}
- });
- scheduledTriggers.add(triggerList.get(i));
+ };
+ trigger.configure(resourceLoader, solrCloudManager, Collections.emptyMap());
+ triggerList.add(trigger);
+ scheduledTriggers.add(trigger);
}
assertTrue("Timed out waiting for latch to fire", getTriggerFiredLatch().await(20, TimeUnit.SECONDS));
assertEquals("Expected 8 triggers but found: " + triggerNames, 8, triggerNames.size());
@@ -168,8 +173,8 @@ public class TriggerSetPropertiesIntegrationTest extends SolrCloudTestCase {
public static class MockTrigger extends TriggerBase {
- public MockTrigger(TriggerEventType eventType, String name, Map<String, Object> properties, SolrResourceLoader loader, SolrCloudManager cloudManager) {
- super(eventType, name, properties, loader, cloudManager);
+ public MockTrigger(TriggerEventType eventType, String name) {
+ super(eventType, name);
}
@Override
[33/34] lucene-solr:jira/solr-12095: Merge branch 'master' into
jira/solr-12095
Posted by sh...@apache.org.
Merge branch 'master' into jira/solr-12095
# Conflicts:
# solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
# solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
# solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/07962265
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/07962265
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/07962265
Branch: refs/heads/jira/solr-12095
Commit: 0796226509e6df433a4648b2bf7d9d0edd629a6c
Parents: 059f495 a4789db
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Mon Apr 2 12:59:44 2018 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Mon Apr 2 12:59:44 2018 +0530
----------------------------------------------------------------------
build.xml | 2 +-
dev-tools/scripts/reproduceJenkinsFailures.py | 1 +
.../lucene/index/BufferedUpdatesStream.java | 4 +-
.../org/apache/lucene/index/IndexWriter.java | 30 +-
.../org/apache/lucene/index/PendingDeletes.java | 193 +++
.../apache/lucene/index/ReadersAndUpdates.java | 185 +--
.../src/java/org/apache/lucene/util/Bits.java | 2 +-
.../apache/lucene/index/TestPendingDeletes.java | 142 +++
.../spatial3d/geom/GeoComplexPolygon.java | 686 ++++------
.../spatial3d/geom/GeoConcavePolygon.java | 16 +-
.../lucene/spatial3d/geom/GeoConvexPolygon.java | 20 +-
.../spatial3d/geom/GeoPolygonFactory.java | 7 +-
.../org/apache/lucene/spatial3d/geom/Plane.java | 60 +-
.../lucene/spatial3d/geom/GeoPolygonTest.java | 262 +++-
solr/CHANGES.txt | 16 +
.../org/apache/solr/cloud/ZkController.java | 22 +-
.../cloud/autoscaling/NodeAddedTrigger.java | 2 +-
.../solr/cloud/autoscaling/NodeLostTrigger.java | 2 +-
.../autoscaling/OverseerTriggerThread.java | 4 +
.../cloud/autoscaling/ScheduledTriggers.java | 13 +
.../org/apache/solr/core/CoreContainer.java | 7 +-
.../apache/solr/core/CorePropertiesLocator.java | 2 +-
.../org/apache/solr/handler/IndexFetcher.java | 35 +-
.../org/apache/solr/util/TestInjection.java | 15 +
.../solr/cloud/DeleteInactiveReplicaTest.java | 22 +-
.../solr/cloud/DocValuesNotIndexedTest.java | 1 +
.../solr/cloud/LIROnShardRestartTest.java | 17 +-
.../solr/cloud/RestartWhileUpdatingTest.java | 1 +
.../apache/solr/cloud/TestCloudConsistency.java | 2 +
.../org/apache/solr/cloud/TestPullReplica.java | 5 +-
.../apache/solr/cloud/TestSegmentSorting.java | 2 +
.../CollectionsAPIDistributedZkTest.java | 1 +
.../autoscaling/ComputePlanActionTest.java | 25 +-
.../MetricTriggerIntegrationTest.java | 242 ++++
.../NodeAddedTriggerIntegrationTest.java | 300 +++++
.../NodeLostTriggerIntegrationTest.java | 322 +++++
.../NodeMarkersRegistrationTest.java | 269 ++++
.../autoscaling/RestoreTriggerStateTest.java | 169 +++
.../ScheduledTriggerIntegrationTest.java | 142 +++
.../SearchRateTriggerIntegrationTest.java | 217 ++++
.../TriggerCooldownIntegrationTest.java | 238 ++++
.../autoscaling/TriggerIntegrationTest.java | 1194 +-----------------
.../TriggerSetPropertiesIntegrationTest.java | 195 +++
.../autoscaling/sim/TestTriggerIntegration.java | 1 +
.../cloud/hdfs/HdfsBasicDistributedZk2Test.java | 2 +
.../solr/handler/TestReplicationHandler.java | 7 +-
.../admin/AutoscalingHistoryHandlerTest.java | 3 +-
solr/solr-ref-guide/src/highlighting.adoc | 23 +-
.../apache/solr/common/util/ExecutorUtil.java | 4 +
49 files changed, 3306 insertions(+), 1826 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07962265/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
index d83f8b9,6190a49..484fbe0
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
@@@ -53,11 -58,8 +53,11 @@@ public class NodeAddedTrigger extends T
}
@Override
- public void init() {
+ public void init() throws Exception {
super.init();
+ lastLiveNodes = new HashSet<>(cloudManager.getClusterStateProvider().getLiveNodes());
- log.debug("Initial livenodes: {}", lastLiveNodes);
++ log.debug("NodeAddedTrigger {} - Initial livenodes: {}", name, lastLiveNodes);
+ log.debug("NodeAddedTrigger {} instantiated with properties: {}", name, properties);
// pick up added nodes for which marker paths were created
try {
List<String> added = stateManager.listData(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07962265/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
index 5bf243f,2981a48..6dfccb2
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
@@@ -52,10 -56,8 +52,10 @@@ public class NodeLostTrigger extends Tr
}
@Override
- public void init() {
+ public void init() throws Exception {
super.init();
+ lastLiveNodes = new HashSet<>(cloudManager.getClusterStateProvider().getLiveNodes());
- log.debug("Initial livenodes: {}", lastLiveNodes);
++ log.debug("NodeLostTrigger {} - Initial livenodes: {}", name, lastLiveNodes);
// pick up lost nodes for which marker paths were created
try {
List<String> lost = stateManager.listData(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07962265/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07962265/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07962265/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
----------------------------------------------------------------------
diff --cc solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
index 5952e40,67b5fa0..4e48f59
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
@@@ -366,19 -376,9 +377,19 @@@ public class ComputePlanActionTest exte
}
public static class AssertingTriggerAction implements TriggerAction {
- static String expectedNode;
+ static volatile String expectedNode;
@Override
+ public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map<String, Object> properties) throws TriggerValidationException {
+
+ }
+
+ @Override
+ public void init() {
+
+ }
+
+ @Override
public String getName() {
return null;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07962265/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07962265/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestTriggerIntegration.java
----------------------------------------------------------------------
[10/34] lucene-solr:jira/solr-12095: SOLR-12152: Fix node count to 2
for TriggerSetPropertiesIntegrationTest
Posted by sh...@apache.org.
SOLR-12152: Fix node count to 2 for TriggerSetPropertiesIntegrationTest
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ac8cbaac
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ac8cbaac
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ac8cbaac
Branch: refs/heads/jira/solr-12095
Commit: ac8cbaac06aa1b6436518bc74a4a4f7e045da5d2
Parents: ed9e5eb
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Fri Mar 30 12:19:13 2018 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Fri Mar 30 12:19:13 2018 +0530
----------------------------------------------------------------------
.../cloud/autoscaling/TriggerSetPropertiesIntegrationTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ac8cbaac/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerSetPropertiesIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerSetPropertiesIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerSetPropertiesIntegrationTest.java
index 47ac227..5f4243f 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerSetPropertiesIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerSetPropertiesIntegrationTest.java
@@ -54,7 +54,7 @@ public class TriggerSetPropertiesIntegrationTest extends SolrCloudTestCase {
@BeforeClass
public static void setupCluster() throws Exception {
- configureCluster(5)
+ configureCluster(2)
.addConfig("conf", configset("cloud-minimal"))
.configure();
// disable .scheduled_maintenance
[26/34] lucene-solr:jira/solr-12095: LUCENE-8232: Write and
Checkpoint DV updates seperately if we drop a reader
Posted by sh...@apache.org.
LUCENE-8232: Write and Checkpoint DV updates seperately if we drop a reader
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ca02e637
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ca02e637
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ca02e637
Branch: refs/heads/jira/solr-12095
Commit: ca02e637ffa117084dc8c59ff32ad487599aae77
Parents: acb3c37
Author: Simon Willnauer <si...@apache.org>
Authored: Sat Mar 31 14:24:36 2018 +0200
Committer: Simon Willnauer <si...@apache.org>
Committed: Sat Mar 31 14:24:36 2018 +0200
----------------------------------------------------------------------
.../core/src/java/org/apache/lucene/index/IndexWriter.java | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ca02e637/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index 3791e19..2e14166 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -604,10 +604,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
if (!poolReaders && rld.refCount() == 1 && readerMap.containsKey(rld.info)) {
// This is the last ref to this RLD, and we're not
// pooling, so remove it:
- boolean changed = rld.writeLiveDocs(directory);
- changed |= rld.writeFieldUpdates(directory, globalFieldNumberMap, bufferedUpdatesStream.getCompletedDelGen(), infoStream);
-
- if (changed) {
+ if (rld.writeLiveDocs(directory)) {
// Make sure we only write del docs for a live segment:
assert assertInfoLive == false || assertInfoIsLive(rld.info);
// Must checkpoint because we just
@@ -619,6 +616,9 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
// did was move the state to disk:
checkpointNoSIS();
}
+ if (rld.writeFieldUpdates(directory, globalFieldNumberMap, bufferedUpdatesStream.getCompletedDelGen(), infoStream)) {
+ checkpointNoSIS();
+ }
if (rld.getNumDVUpdates() == 0) {
rld.dropReaders();
readerMap.remove(rld.info);
[28/34] lucene-solr:jira/solr-12095: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/lucene-solr
Posted by sh...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/lucene-solr
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/dc9c6032
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/dc9c6032
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/dc9c6032
Branch: refs/heads/jira/solr-12095
Commit: dc9c60322a0fb026553d5fe359c8e5c41b0d07ef
Parents: 590e671 ca02e63
Author: Karl Wright <Da...@gmail.com>
Authored: Sat Mar 31 10:04:01 2018 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Sat Mar 31 10:04:01 2018 -0400
----------------------------------------------------------------------
.../lucene/index/BufferedUpdatesStream.java | 4 +-
.../org/apache/lucene/index/IndexWriter.java | 30 +--
.../org/apache/lucene/index/PendingDeletes.java | 193 +++++++++++++++++++
.../apache/lucene/index/ReadersAndUpdates.java | 185 +++++-------------
.../src/java/org/apache/lucene/util/Bits.java | 2 +-
.../apache/lucene/index/TestPendingDeletes.java | 142 ++++++++++++++
solr/CHANGES.txt | 7 +
.../org/apache/solr/cloud/ZkController.java | 22 ++-
.../cloud/autoscaling/NodeAddedTrigger.java | 2 +-
.../solr/cloud/autoscaling/NodeLostTrigger.java | 2 +-
.../org/apache/solr/core/CoreContainer.java | 7 +-
.../solr/cloud/DeleteInactiveReplicaTest.java | 22 ++-
.../solr/cloud/LIROnShardRestartTest.java | 17 +-
.../autoscaling/ComputePlanActionTest.java | 25 ++-
14 files changed, 484 insertions(+), 176 deletions(-)
----------------------------------------------------------------------
[09/34] lucene-solr:jira/solr-12095: SOLR-12152: Split up
TriggerIntegrationTest into multiple tests to isolate and increase
reliability
Posted by sh...@apache.org.
SOLR-12152: Split up TriggerIntegrationTest into multiple tests to isolate and increase reliability
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ed9e5eb7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ed9e5eb7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ed9e5eb7
Branch: refs/heads/jira/solr-12095
Commit: ed9e5eb75b38fb24c1d32e885941d065d284ffa0
Parents: 2370731
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Fri Mar 30 11:08:56 2018 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Fri Mar 30 11:08:56 2018 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../MetricTriggerIntegrationTest.java | 242 +++++
.../NodeAddedTriggerIntegrationTest.java | 300 ++++++
.../NodeLostTriggerIntegrationTest.java | 322 ++++++
.../NodeMarkersRegistrationTest.java | 269 +++++
.../ScheduledTriggerIntegrationTest.java | 141 +++
.../SearchRateTriggerIntegrationTest.java | 217 ++++
.../TriggerCooldownIntegrationTest.java | 238 +++++
.../autoscaling/TriggerIntegrationTest.java | 1006 +-----------------
.../TriggerSetPropertiesIntegrationTest.java | 195 ++++
10 files changed, 1929 insertions(+), 1003 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ed9e5eb7/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index aa7150f..5854e0f 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -131,6 +131,8 @@ Other Changes
* SOLR-12118: Solr Ref-Guide can now use some ivy version props directly as attributes in content (hossman)
+* SOLR-12152: Split up TriggerIntegrationTest into multiple tests to isolate and increase reliability. (shalin)
+
================== 7.3.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ed9e5eb7/solr/core/src/test/org/apache/solr/cloud/autoscaling/MetricTriggerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/MetricTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/MetricTriggerIntegrationTest.java
new file mode 100644
index 0000000..7b6da5a
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/MetricTriggerIntegrationTest.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud.autoscaling;
+
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.cloud.SolrCloudManager;
+import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
+import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.metrics.SolrCoreMetricManager;
+import org.apache.solr.util.LogLevel;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.createAutoScalingRequest;
+import static org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.WAIT_FOR_DELTA_NANOS;
+import static org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.timeSource;
+
+/**
+ * Integration test for {@link MetricTrigger}
+ */
+@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG")
+public class MetricTriggerIntegrationTest extends SolrCloudTestCase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ static Map<String, List<CapturedEvent>> listenerEvents = new HashMap<>();
+ static CountDownLatch listenerCreated = new CountDownLatch(1);
+ private static CountDownLatch triggerFiredLatch;
+ private static int waitForSeconds = 1;
+ private static Set<TriggerEvent> events = ConcurrentHashMap.newKeySet();
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(2)
+ .addConfig("conf", configset("cloud-minimal"))
+ .configure();
+ // disable .scheduled_maintenance
+ String suspendTriggerCommand = "{" +
+ "'suspend-trigger' : {'name' : '.scheduled_maintenance'}" +
+ "}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
+ SolrClient solrClient = cluster.getSolrClient();
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ triggerFiredLatch = new CountDownLatch(1);
+ }
+
+ @Test
+ public void testMetricTrigger() throws Exception {
+ cluster.waitForAllNodes(5);
+
+ String collectionName = "testMetricTrigger";
+ CloudSolrClient solrClient = cluster.getSolrClient();
+ CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,
+ "conf", 2, 2).setMaxShardsPerNode(2);
+ create.process(solrClient);
+ solrClient.setDefaultCollection(collectionName);
+
+ waitForState("Timed out waiting for collection:" + collectionName + " to become active", collectionName, clusterShape(2, 2));
+
+ DocCollection docCollection = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
+ String shardId = "shard1";
+ Replica replica = docCollection.getSlice(shardId).getReplicas().iterator().next();
+ String coreName = replica.getCoreName();
+ String replicaName = Utils.parseMetricsReplicaName(collectionName, coreName);
+ long waitForSeconds = 2 + random().nextInt(5);
+ String registry = SolrCoreMetricManager.createRegistryName(true, collectionName, shardId, replicaName, null);
+ String tag = "metrics:" + registry + ":INDEX.sizeInBytes";
+
+ String setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'metric_trigger'," +
+ "'event' : 'metric'," +
+ "'waitFor' : '" + waitForSeconds + "s'," +
+ "'enabled' : true," +
+ "'metric': '" + tag + "'" +
+ "'above' : 100.0," +
+ "'collection': '" + collectionName + "'" +
+ "'shard':'" + shardId + "'" +
+ "'actions' : [" +
+ "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," +
+ "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}," +
+ "{'name':'test','class':'" + MetricAction.class.getName() + "'}" +
+ "]" +
+ "}}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ String setListenerCommand1 = "{" +
+ "'set-listener' : " +
+ "{" +
+ "'name' : 'srt'," +
+ "'trigger' : 'metric_trigger'," +
+ "'stage' : ['FAILED','SUCCEEDED']," +
+ "'afterAction': ['compute', 'execute', 'test']," +
+ "'class' : '" + TestTriggerListener.class.getName() + "'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setListenerCommand1);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ // start more nodes so that we have at least 4
+ for (int i = cluster.getJettySolrRunners().size(); i < 4; i++) {
+ cluster.startJettySolrRunner();
+ }
+ cluster.waitForAllNodes(10);
+
+ List<SolrInputDocument> docs = new ArrayList<>(500);
+ for (int i = 0; i < 500; i++) {
+ docs.add(new SolrInputDocument("id", String.valueOf(i), "x_s", "x" + i));
+ }
+ solrClient.add(docs);
+ solrClient.commit();
+
+ boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
+ assertTrue("The trigger did not fire at all", await);
+ // wait for listener to capture the SUCCEEDED stage
+ Thread.sleep(2000);
+ assertEquals(listenerEvents.toString(), 4, listenerEvents.get("srt").size());
+ CapturedEvent ev = listenerEvents.get("srt").get(0);
+ long now = timeSource.getTimeNs();
+ // verify waitFor
+ assertTrue(TimeUnit.SECONDS.convert(waitForSeconds, TimeUnit.NANOSECONDS) - WAIT_FOR_DELTA_NANOS <= now - ev.event.getEventTime());
+ assertEquals(collectionName, ev.event.getProperties().get("collection"));
+
+ // find a new replica and create its metric name
+ docCollection = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
+ replica = docCollection.getSlice(shardId).getReplicas().iterator().next();
+ coreName = replica.getCoreName();
+ replicaName = Utils.parseMetricsReplicaName(collectionName, coreName);
+ registry = SolrCoreMetricManager.createRegistryName(true, collectionName, shardId, replicaName, null);
+ tag = "metrics:" + registry + ":INDEX.sizeInBytes";
+
+ triggerFiredLatch = new CountDownLatch(1);
+ listenerEvents.clear();
+
+ setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'metric_trigger'," +
+ "'event' : 'metric'," +
+ "'waitFor' : '" + waitForSeconds + "s'," +
+ "'enabled' : true," +
+ "'metric': '" + tag + "'" +
+ "'above' : 100.0," +
+ "'collection': '" + collectionName + "'" +
+ "'shard':'" + shardId + "'" +
+ "'preferredOperation':'addreplica'" +
+ "'actions' : [" +
+ "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," +
+ "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}," +
+ "{'name':'test','class':'" + MetricAction.class.getName() + "'}" +
+ "]" +
+ "}}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
+ assertTrue("The trigger did not fire at all", await);
+ // wait for listener to capture the SUCCEEDED stage
+ Thread.sleep(2000);
+ assertEquals(listenerEvents.toString(), 4, listenerEvents.get("srt").size());
+ ev = listenerEvents.get("srt").get(0);
+ now = timeSource.getTimeNs();
+ // verify waitFor
+ assertTrue(TimeUnit.SECONDS.convert(waitForSeconds, TimeUnit.NANOSECONDS) - WAIT_FOR_DELTA_NANOS <= now - ev.event.getEventTime());
+ assertEquals(collectionName, ev.event.getProperties().get("collection"));
+ docCollection = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
+ assertEquals(5, docCollection.getReplicas().size());
+ }
+
+ public static class MetricAction extends TriggerActionBase {
+
+ @Override
+ public void process(TriggerEvent event, ActionContext context) throws Exception {
+ try {
+ events.add(event);
+ long currentTimeNanos = timeSource.getTimeNs();
+ long eventTimeNanos = event.getEventTime();
+ long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS;
+ if (currentTimeNanos - eventTimeNanos <= waitForNanos) {
+ fail(event.source + " was fired before the configured waitFor period");
+ }
+ triggerFiredLatch.countDown();
+ } catch (Throwable t) {
+ log.debug("--throwable", t);
+ throw t;
+ }
+ }
+ }
+
+ public static class TestTriggerListener extends TriggerListenerBase {
+ @Override
+ public void init(SolrCloudManager cloudManager, AutoScalingConfig.TriggerListenerConfig config) {
+ super.init(cloudManager, config);
+ listenerCreated.countDown();
+ }
+
+ @Override
+ public synchronized void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName,
+ ActionContext context, Throwable error, String message) {
+ List<CapturedEvent> lst = listenerEvents.computeIfAbsent(config.name, s -> new ArrayList<>());
+ lst.add(new CapturedEvent(timeSource.getTimeNs(), context, config, stage, actionName, event, message));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ed9e5eb7/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeAddedTriggerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeAddedTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeAddedTriggerIntegrationTest.java
new file mode 100644
index 0000000..ecf2437
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeAddedTriggerIntegrationTest.java
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud.autoscaling;
+
+import java.lang.invoke.MethodHandles;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.cloud.SolrCloudManager;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.cloud.Overseer;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.util.LogLevel;
+import org.apache.solr.util.TimeOut;
+import org.apache.zookeeper.data.Stat;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.createAutoScalingRequest;
+import static org.apache.solr.cloud.autoscaling.ScheduledTriggers.DEFAULT_SCHEDULED_TRIGGER_DELAY_SECONDS;
+import static org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.WAIT_FOR_DELTA_NANOS;
+import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH;
+
+@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG")
+public class NodeAddedTriggerIntegrationTest extends SolrCloudTestCase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private static CountDownLatch actionConstructorCalled;
+ private static CountDownLatch actionInitCalled;
+ private static CountDownLatch triggerFiredLatch;
+ private static int waitForSeconds = 1;
+ private static AtomicBoolean triggerFired;
+ private static Set<TriggerEvent> events = ConcurrentHashMap.newKeySet();
+ private static SolrCloudManager cloudManager;
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(2)
+ .addConfig("conf", configset("cloud-minimal"))
+ .configure();
+ // disable .scheduled_maintenance
+ String suspendTriggerCommand = "{" +
+ "'suspend-trigger' : {'name' : '.scheduled_maintenance'}" +
+ "}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
+ SolrClient solrClient = cluster.getSolrClient();
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ }
+
+ private static CountDownLatch getTriggerFiredLatch() {
+ return triggerFiredLatch;
+ }
+
+ @Before
+ public void setupTest() throws Exception {
+ // ensure that exactly 2 jetty nodes are running
+ int numJetties = cluster.getJettySolrRunners().size();
+ log.info("Found {} jetty instances running", numJetties);
+ for (int i = 2; i < numJetties; i++) {
+ int r = random().nextInt(cluster.getJettySolrRunners().size());
+ log.info("Shutdown extra jetty instance at port {}", cluster.getJettySolrRunner(r).getLocalPort());
+ cluster.stopJettySolrRunner(r);
+ }
+ for (int i = cluster.getJettySolrRunners().size(); i < 2; i++) {
+ // start jetty instances
+ cluster.startJettySolrRunner();
+ }
+ cluster.waitForAllNodes(5);
+
+ NamedList<Object> overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
+ String overseerLeader = (String) overSeerStatus.get("leader");
+ int overseerLeaderIndex = 0;
+ for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
+ JettySolrRunner jetty = cluster.getJettySolrRunner(i);
+ if (jetty.getNodeName().equals(overseerLeader)) {
+ overseerLeaderIndex = i;
+ break;
+ }
+ }
+ Overseer overseer = cluster.getJettySolrRunner(overseerLeaderIndex).getCoreContainer().getZkController().getOverseer();
+ ScheduledTriggers scheduledTriggers = ((OverseerTriggerThread) overseer.getTriggerThread().getThread()).getScheduledTriggers();
+ // aggressively remove all active scheduled triggers
+ scheduledTriggers.removeAll();
+
+ // clear any persisted auto scaling configuration
+ Stat stat = zkClient().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), true);
+ log.info(SOLR_AUTOSCALING_CONF_PATH + " reset, new znode version {}", stat.getVersion());
+
+ cluster.deleteAllCollections();
+ cluster.getSolrClient().setDefaultCollection(null);
+
+ // restart Overseer. Even though we reset the autoscaling config some already running
+ // trigger threads may still continue to execute and produce spurious events
+ cluster.stopJettySolrRunner(overseerLeaderIndex);
+ Thread.sleep(5000);
+
+ waitForSeconds = 1 + random().nextInt(3);
+ actionConstructorCalled = new CountDownLatch(1);
+ actionInitCalled = new CountDownLatch(1);
+ triggerFiredLatch = new CountDownLatch(1);
+ triggerFired = new AtomicBoolean(false);
+ events.clear();
+
+ while (cluster.getJettySolrRunners().size() < 2) {
+ // perhaps a test stopped a node but didn't start it back
+ // lets start a node
+ cluster.startJettySolrRunner();
+ }
+
+ cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager();
+ // clear any events or markers
+ // todo: consider the impact of such cleanup on regular cluster restarts
+ deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH);
+ deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH);
+ deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH);
+ deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH);
+ }
+
+ private void deleteChildrenRecursively(String path) throws Exception {
+ cloudManager.getDistribStateManager().removeRecursively(path, true, false);
+ }
+
+ @Test
+ public void testNodeAddedTriggerRestoreState() throws Exception {
+ // for this test we want to update the trigger so we must assert that the actions were created twice
+ actionInitCalled = new CountDownLatch(2);
+
+ CloudSolrClient solrClient = cluster.getSolrClient();
+ waitForSeconds = 5;
+ String setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'node_added_restore_trigger'," +
+ "'event' : 'nodeAdded'," +
+ "'waitFor' : '5s'," + // should be enough for us to update the trigger
+ "'enabled' : true," +
+ "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
+ "}}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ TimeOut timeOut = new TimeOut(2, TimeUnit.SECONDS, cloudManager.getTimeSource());
+ while (actionInitCalled.getCount() == 0 && !timeOut.hasTimedOut()) {
+ Thread.sleep(200);
+ }
+ assertTrue("The action specified in node_added_restore_trigger was not instantiated even after 2 seconds", actionInitCalled.getCount() > 0);
+
+ // start a new node
+ JettySolrRunner newNode = cluster.startJettySolrRunner();
+
+ // ensure that the old trigger sees the new node, todo find a better way to do this
+ Thread.sleep(500 + TimeUnit.SECONDS.toMillis(DEFAULT_SCHEDULED_TRIGGER_DELAY_SECONDS));
+
+ waitForSeconds = 0;
+ setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'node_added_restore_trigger'," +
+ "'event' : 'nodeAdded'," +
+ "'waitFor' : '0s'," + // update a property so that it replaces the old trigger, also we want it to fire immediately
+ "'enabled' : true," +
+ "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
+ "}}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ // wait until the second instance of action is created
+ if (!actionInitCalled.await(3, TimeUnit.SECONDS)) {
+ fail("Two TriggerAction instances should have been created by now");
+ }
+
+ boolean await = triggerFiredLatch.await(5, TimeUnit.SECONDS);
+ assertTrue("The trigger did not fire at all", await);
+ assertTrue(triggerFired.get());
+ NodeAddedTrigger.NodeAddedEvent nodeAddedEvent = (NodeAddedTrigger.NodeAddedEvent) events.iterator().next();
+ assertNotNull(nodeAddedEvent);
+ List<String> nodeNames = (List<String>) nodeAddedEvent.getProperty(TriggerEvent.NODE_NAMES);
+ assertTrue(nodeNames.contains(newNode.getNodeName()));
+ }
+
+ @Test
+ public void testNodeAddedTrigger() throws Exception {
+ CloudSolrClient solrClient = cluster.getSolrClient();
+ String setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'node_added_trigger'," +
+ "'event' : 'nodeAdded'," +
+ "'waitFor' : '" + waitForSeconds + "s'," +
+ "'enabled' : true," +
+ "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
+ "}}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ if (!actionInitCalled.await(3, TimeUnit.SECONDS)) {
+ fail("The TriggerAction should have been created by now");
+ }
+
+ JettySolrRunner newNode = cluster.startJettySolrRunner();
+ boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
+ assertTrue("The trigger did not fire at all", await);
+ assertTrue(triggerFired.get());
+ NodeAddedTrigger.NodeAddedEvent nodeAddedEvent = (NodeAddedTrigger.NodeAddedEvent) events.iterator().next();
+ assertNotNull(nodeAddedEvent);
+ List<String> nodeNames = (List<String>) nodeAddedEvent.getProperty(TriggerEvent.NODE_NAMES);
+ assertTrue(nodeNames.contains(newNode.getNodeName()));
+
+ // reset
+ actionConstructorCalled = new CountDownLatch(1);
+ actionInitCalled = new CountDownLatch(1);
+
+ // update the trigger with exactly the same data
+ setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'node_added_trigger'," +
+ "'event' : 'nodeAdded'," +
+ "'waitFor' : '" + waitForSeconds + "s'," +
+ "'enabled' : true," +
+ "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
+ "}}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ // this should be a no-op so the action should have been created but init should not be called
+ if (!actionConstructorCalled.await(3, TimeUnit.SECONDS)) {
+ fail("The TriggerAction should have been created by now");
+ }
+
+ assertFalse(actionInitCalled.await(2, TimeUnit.SECONDS));
+ }
+
+ public static class TestTriggerAction extends TriggerActionBase {
+
+ public TestTriggerAction() {
+ actionConstructorCalled.countDown();
+ }
+
+ @Override
+ public void process(TriggerEvent event, ActionContext actionContext) {
+ try {
+ if (triggerFired.compareAndSet(false, true)) {
+ events.add(event);
+ long currentTimeNanos = TriggerIntegrationTest.timeSource.getTimeNs();
+ long eventTimeNanos = event.getEventTime();
+ long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS;
+ if (currentTimeNanos - eventTimeNanos <= waitForNanos) {
+ fail(event.source + " was fired before the configured waitFor period");
+ }
+ getTriggerFiredLatch().countDown();
+ } else {
+ fail(event.source + " was fired more than once!");
+ }
+ } catch (Throwable t) {
+ log.debug("--throwable", t);
+ throw t;
+ }
+ }
+
+ @Override
+ public void init(Map<String, String> args) {
+ log.info("TestTriggerAction init");
+ actionInitCalled.countDown();
+ super.init(args);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ed9e5eb7/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerIntegrationTest.java
new file mode 100644
index 0000000..6b1af65
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerIntegrationTest.java
@@ -0,0 +1,322 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud.autoscaling;
+
+import java.lang.invoke.MethodHandles;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.cloud.SolrCloudManager;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.cloud.Overseer;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.util.LogLevel;
+import org.apache.solr.util.TimeOut;
+import org.apache.zookeeper.data.Stat;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.createAutoScalingRequest;
+import static org.apache.solr.cloud.autoscaling.ScheduledTriggers.DEFAULT_SCHEDULED_TRIGGER_DELAY_SECONDS;
+import static org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.WAIT_FOR_DELTA_NANOS;
+import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH;
+
+@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG")
+public class NodeLostTriggerIntegrationTest extends SolrCloudTestCase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private static CountDownLatch actionConstructorCalled;
+ private static CountDownLatch actionInitCalled;
+ private static CountDownLatch triggerFiredLatch;
+ private static int waitForSeconds = 1;
+ private static AtomicBoolean triggerFired;
+ private static Set<TriggerEvent> events = ConcurrentHashMap.newKeySet();
+ private static SolrCloudManager cloudManager;
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(2)
+ .addConfig("conf", configset("cloud-minimal"))
+ .configure();
+ // disable .scheduled_maintenance
+ String suspendTriggerCommand = "{" +
+ "'suspend-trigger' : {'name' : '.scheduled_maintenance'}" +
+ "}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
+ SolrClient solrClient = cluster.getSolrClient();
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ }
+
+ private static CountDownLatch getTriggerFiredLatch() {
+ return triggerFiredLatch;
+ }
+
+ @Before
+ public void setupTest() throws Exception {
+ // ensure that exactly 2 jetty nodes are running
+ int numJetties = cluster.getJettySolrRunners().size();
+ log.info("Found {} jetty instances running", numJetties);
+ for (int i = 2; i < numJetties; i++) {
+ int r = random().nextInt(cluster.getJettySolrRunners().size());
+ log.info("Shutdown extra jetty instance at port {}", cluster.getJettySolrRunner(r).getLocalPort());
+ cluster.stopJettySolrRunner(r);
+ }
+ for (int i = cluster.getJettySolrRunners().size(); i < 2; i++) {
+ // start jetty instances
+ cluster.startJettySolrRunner();
+ }
+ cluster.waitForAllNodes(5);
+
+ NamedList<Object> overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
+ String overseerLeader = (String) overSeerStatus.get("leader");
+ int overseerLeaderIndex = 0;
+ for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
+ JettySolrRunner jetty = cluster.getJettySolrRunner(i);
+ if (jetty.getNodeName().equals(overseerLeader)) {
+ overseerLeaderIndex = i;
+ break;
+ }
+ }
+ Overseer overseer = cluster.getJettySolrRunner(overseerLeaderIndex).getCoreContainer().getZkController().getOverseer();
+ ScheduledTriggers scheduledTriggers = ((OverseerTriggerThread) overseer.getTriggerThread().getThread()).getScheduledTriggers();
+ // aggressively remove all active scheduled triggers
+ scheduledTriggers.removeAll();
+
+ // clear any persisted auto scaling configuration
+ Stat stat = zkClient().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), true);
+ log.info(SOLR_AUTOSCALING_CONF_PATH + " reset, new znode version {}", stat.getVersion());
+
+ cluster.deleteAllCollections();
+ cluster.getSolrClient().setDefaultCollection(null);
+
+ // restart Overseer. Even though we reset the autoscaling config some already running
+ // trigger threads may still continue to execute and produce spurious events
+ cluster.stopJettySolrRunner(overseerLeaderIndex);
+ Thread.sleep(5000);
+
+ waitForSeconds = 1 + random().nextInt(3);
+ actionConstructorCalled = new CountDownLatch(1);
+ actionInitCalled = new CountDownLatch(1);
+ triggerFiredLatch = new CountDownLatch(1);
+ triggerFired = new AtomicBoolean(false);
+ events.clear();
+
+ while (cluster.getJettySolrRunners().size() < 2) {
+ // perhaps a test stopped a node but didn't start it back
+ // lets start a node
+ cluster.startJettySolrRunner();
+ }
+
+ cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager();
+ // clear any events or markers
+ // todo: consider the impact of such cleanup on regular cluster restarts
+ deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH);
+ deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH);
+ deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH);
+ deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH);
+ }
+
+ private void deleteChildrenRecursively(String path) throws Exception {
+ cloudManager.getDistribStateManager().removeRecursively(path, true, false);
+ }
+
+ @Test
+ public void testNodeLostTriggerRestoreState() throws Exception {
+ // for this test we want to update the trigger so we must assert that the actions were created twice
+ actionInitCalled = new CountDownLatch(2);
+
+ // start a new node
+ JettySolrRunner newNode = cluster.startJettySolrRunner();
+ String nodeName = newNode.getNodeName();
+
+ CloudSolrClient solrClient = cluster.getSolrClient();
+ waitForSeconds = 5;
+ String setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'node_lost_restore_trigger'," +
+ "'event' : 'nodeLost'," +
+ "'waitFor' : '5s'," + // should be enough for us to update the trigger
+ "'enabled' : true," +
+ "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
+ "}}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ TimeOut timeOut = new TimeOut(2, TimeUnit.SECONDS, cloudManager.getTimeSource());
+ while (actionInitCalled.getCount() == 0 && !timeOut.hasTimedOut()) {
+ Thread.sleep(200);
+ }
+ assertTrue("The action specified in node_lost_restore_trigger was not instantiated even after 2 seconds", actionInitCalled.getCount() > 0);
+
+ List<JettySolrRunner> jettySolrRunners = cluster.getJettySolrRunners();
+ int index = -1;
+ for (int i = 0; i < jettySolrRunners.size(); i++) {
+ JettySolrRunner runner = jettySolrRunners.get(i);
+ if (runner == newNode) index = i;
+ }
+ assertFalse(index == -1);
+ cluster.stopJettySolrRunner(index);
+
+ // ensure that the old trigger sees the stopped node, todo find a better way to do this
+ Thread.sleep(500 + TimeUnit.SECONDS.toMillis(DEFAULT_SCHEDULED_TRIGGER_DELAY_SECONDS));
+
+ waitForSeconds = 0;
+ setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'node_lost_restore_trigger'," +
+ "'event' : 'nodeLost'," +
+ "'waitFor' : '0s'," + // update a property so that it replaces the old trigger, also we want it to fire immediately
+ "'enabled' : true," +
+ "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
+ "}}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ // wait until the second instance of action is created
+ if (!actionInitCalled.await(3, TimeUnit.SECONDS)) {
+ fail("Two TriggerAction instances should have been created by now");
+ }
+
+ boolean await = triggerFiredLatch.await(5, TimeUnit.SECONDS);
+ assertTrue("The trigger did not fire at all", await);
+ assertTrue(triggerFired.get());
+ NodeLostTrigger.NodeLostEvent nodeLostEvent = (NodeLostTrigger.NodeLostEvent) events.iterator().next();
+ assertNotNull(nodeLostEvent);
+ List<String> nodeNames = (List<String>) nodeLostEvent.getProperty(TriggerEvent.NODE_NAMES);
+ assertTrue(nodeNames.contains(nodeName));
+ }
+
+ @Test
+ public void testNodeLostTrigger() throws Exception {
+ CloudSolrClient solrClient = cluster.getSolrClient();
+ String setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'node_lost_trigger'," +
+ "'event' : 'nodeLost'," +
+ "'waitFor' : '" + waitForSeconds + "s'," +
+ "'enabled' : true," +
+ "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
+ "}}";
+ NamedList<Object> overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
+ String overseerLeader = (String) overSeerStatus.get("leader");
+ int nonOverseerLeaderIndex = 0;
+ for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
+ JettySolrRunner jetty = cluster.getJettySolrRunner(i);
+ if (!jetty.getNodeName().equals(overseerLeader)) {
+ nonOverseerLeaderIndex = i;
+ }
+ }
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ if (!actionInitCalled.await(3, TimeUnit.SECONDS)) {
+ fail("The TriggerAction should have been created by now");
+ }
+
+ triggerFired.set(false);
+ triggerFiredLatch = new CountDownLatch(1);
+ String lostNodeName = cluster.getJettySolrRunner(nonOverseerLeaderIndex).getNodeName();
+ cluster.stopJettySolrRunner(nonOverseerLeaderIndex);
+ boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
+ assertTrue("The trigger did not fire at all", await);
+ assertTrue(triggerFired.get());
+ NodeLostTrigger.NodeLostEvent nodeLostEvent = (NodeLostTrigger.NodeLostEvent) events.iterator().next();
+ assertNotNull(nodeLostEvent);
+ List<String> nodeNames = (List<String>) nodeLostEvent.getProperty(TriggerEvent.NODE_NAMES);
+ assertTrue(nodeNames.contains(lostNodeName));
+
+ // reset
+ actionConstructorCalled = new CountDownLatch(1);
+ actionInitCalled = new CountDownLatch(1);
+
+ // update the trigger with exactly the same data
+ setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'node_lost_trigger'," +
+ "'event' : 'nodeLost'," +
+ "'waitFor' : '" + waitForSeconds + "s'," +
+ "'enabled' : true," +
+ "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
+ "}}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ // this should be a no-op so the action should have been created but init should not be called
+ if (!actionConstructorCalled.await(3, TimeUnit.SECONDS)) {
+ fail("The TriggerAction should have been created by now");
+ }
+
+ assertFalse(actionInitCalled.await(2, TimeUnit.SECONDS));
+ }
+
+ public static class TestTriggerAction extends TriggerActionBase {
+
+ public TestTriggerAction() {
+ actionConstructorCalled.countDown();
+ }
+
+ @Override
+ public void process(TriggerEvent event, ActionContext actionContext) {
+ try {
+ if (triggerFired.compareAndSet(false, true)) {
+ events.add(event);
+ long currentTimeNanos = TriggerIntegrationTest.timeSource.getTimeNs();
+ long eventTimeNanos = event.getEventTime();
+ long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS;
+ if (currentTimeNanos - eventTimeNanos <= waitForNanos) {
+ fail(event.source + " was fired before the configured waitFor period");
+ }
+ getTriggerFiredLatch().countDown();
+ } else {
+ fail(event.source + " was fired more than once!");
+ }
+ } catch (Throwable t) {
+ log.debug("--throwable", t);
+ throw t;
+ }
+ }
+
+ @Override
+ public void init(Map<String, String> args) {
+ log.info("TestTriggerAction init");
+ actionInitCalled.countDown();
+ super.init(args);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ed9e5eb7/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeMarkersRegistrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeMarkersRegistrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeMarkersRegistrationTest.java
new file mode 100644
index 0000000..38c2165
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeMarkersRegistrationTest.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud.autoscaling;
+
+import java.lang.invoke.MethodHandles;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.cloud.LiveNodesListener;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.util.LogLevel;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.createAutoScalingRequest;
+
+@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG")
+@LuceneTestCase.BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-12028")
+public class NodeMarkersRegistrationTest extends SolrCloudTestCase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private static CountDownLatch actionInitCalled;
+ private static CountDownLatch triggerFiredLatch;
+ private static CountDownLatch actionConstructorCalled;
+ private static Set<TriggerEvent> events = ConcurrentHashMap.newKeySet();
+ private static ZkStateReader zkStateReader;
+ private static ReentrantLock lock = new ReentrantLock();
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(2)
+ .addConfig("conf", configset("cloud-minimal"))
+ .configure();
+ zkStateReader = cluster.getSolrClient().getZkStateReader();
+ // disable .scheduled_maintenance
+ String suspendTriggerCommand = "{" +
+ "'suspend-trigger' : {'name' : '.scheduled_maintenance'}" +
+ "}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
+ SolrClient solrClient = cluster.getSolrClient();
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ }
+
+ private static CountDownLatch getTriggerFiredLatch() {
+ return triggerFiredLatch;
+ }
+
+ @Test
+ public void testNodeMarkersRegistration() throws Exception {
+ // for this test we want to create two triggers so we must assert that the actions were created twice
+ actionInitCalled = new CountDownLatch(2);
+ // similarly we want both triggers to fire
+ triggerFiredLatch = new CountDownLatch(2);
+ actionConstructorCalled = new CountDownLatch(1);
+ TestLiveNodesListener listener = registerLiveNodesListener();
+
+ NamedList<Object> overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
+ String overseerLeader = (String) overSeerStatus.get("leader");
+ int overseerLeaderIndex = 0;
+ for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
+ JettySolrRunner jetty = cluster.getJettySolrRunner(i);
+ if (jetty.getNodeName().equals(overseerLeader)) {
+ overseerLeaderIndex = i;
+ break;
+ }
+ }
+ // add a node
+ JettySolrRunner node = cluster.startJettySolrRunner();
+ if (!listener.onChangeLatch.await(10, TimeUnit.SECONDS)) {
+ fail("onChange listener didn't execute on cluster change");
+ }
+ assertEquals(1, listener.addedNodes.size());
+ assertEquals(node.getNodeName(), listener.addedNodes.iterator().next());
+ // verify that a znode doesn't exist (no trigger)
+ String pathAdded = ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH + "/" + node.getNodeName();
+ assertFalse("Path " + pathAdded + " was created but there are no nodeAdded triggers", zkClient().exists(pathAdded, true));
+ listener.reset();
+ // stop overseer
+ log.info("====== KILL OVERSEER 1");
+ cluster.stopJettySolrRunner(overseerLeaderIndex);
+ if (!listener.onChangeLatch.await(10, TimeUnit.SECONDS)) {
+ fail("onChange listener didn't execute on cluster change");
+ }
+ assertEquals(1, listener.lostNodes.size());
+ assertEquals(overseerLeader, listener.lostNodes.iterator().next());
+ assertEquals(0, listener.addedNodes.size());
+ // wait until the new overseer is up
+ Thread.sleep(5000);
+ // verify that a znode does NOT exist - there's no nodeLost trigger,
+ // so the new overseer cleaned up existing nodeLost markers
+ String pathLost = ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH + "/" + overseerLeader;
+ assertFalse("Path " + pathLost + " exists", zkClient().exists(pathLost, true));
+
+ listener.reset();
+
+ // set up triggers
+ CloudSolrClient solrClient = cluster.getSolrClient();
+
+ log.info("====== ADD TRIGGERS");
+ String setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'node_added_triggerMR'," +
+ "'event' : 'nodeAdded'," +
+ "'waitFor' : '1s'," +
+ "'enabled' : true," +
+ "'actions' : [{'name':'test','class':'" + TestEventMarkerAction.class.getName() + "'}]" +
+ "}}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'node_lost_triggerMR'," +
+ "'event' : 'nodeLost'," +
+ "'waitFor' : '1s'," +
+ "'enabled' : true," +
+ "'actions' : [{'name':'test','class':'" + TestEventMarkerAction.class.getName() + "'}]" +
+ "}}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
+ overseerLeader = (String) overSeerStatus.get("leader");
+ overseerLeaderIndex = 0;
+ for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
+ JettySolrRunner jetty = cluster.getJettySolrRunner(i);
+ if (jetty.getNodeName().equals(overseerLeader)) {
+ overseerLeaderIndex = i;
+ break;
+ }
+ }
+
+ // create another node
+ log.info("====== ADD NODE 1");
+ JettySolrRunner node1 = cluster.startJettySolrRunner();
+ if (!listener.onChangeLatch.await(10, TimeUnit.SECONDS)) {
+ fail("onChange listener didn't execute on cluster change");
+ }
+ assertEquals(1, listener.addedNodes.size());
+ assertEquals(node1.getNodeName(), listener.addedNodes.iterator().next());
+ // verify that a znode exists
+ pathAdded = ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH + "/" + node1.getNodeName();
+ assertTrue("Path " + pathAdded + " wasn't created", zkClient().exists(pathAdded, true));
+
+ Thread.sleep(5000);
+ // nodeAdded marker should be consumed now by nodeAdded trigger
+ assertFalse("Path " + pathAdded + " should have been deleted", zkClient().exists(pathAdded, true));
+
+ listener.reset();
+ events.clear();
+ triggerFiredLatch = new CountDownLatch(1);
+ // kill overseer again
+ log.info("====== KILL OVERSEER 2");
+ cluster.stopJettySolrRunner(overseerLeaderIndex);
+ if (!listener.onChangeLatch.await(10, TimeUnit.SECONDS)) {
+ fail("onChange listener didn't execute on cluster change");
+ }
+
+
+ if (!triggerFiredLatch.await(20, TimeUnit.SECONDS)) {
+ fail("Trigger should have fired by now");
+ }
+ assertEquals(1, events.size());
+ TriggerEvent ev = events.iterator().next();
+ List<String> nodeNames = (List<String>) ev.getProperty(TriggerEvent.NODE_NAMES);
+ assertTrue(nodeNames.contains(overseerLeader));
+ assertEquals(TriggerEventType.NODELOST, ev.getEventType());
+ }
+
+ private TestLiveNodesListener registerLiveNodesListener() {
+ TestLiveNodesListener listener = new TestLiveNodesListener();
+ zkStateReader.registerLiveNodesListener(listener);
+ return listener;
+ }
+
+ private static class TestLiveNodesListener implements LiveNodesListener {
+ Set<String> lostNodes = new HashSet<>();
+ Set<String> addedNodes = new HashSet<>();
+ CountDownLatch onChangeLatch = new CountDownLatch(1);
+
+ public void reset() {
+ lostNodes.clear();
+ addedNodes.clear();
+ onChangeLatch = new CountDownLatch(1);
+ }
+
+ @Override
+ public void onChange(SortedSet<String> oldLiveNodes, SortedSet<String> newLiveNodes) {
+ onChangeLatch.countDown();
+ Set<String> old = new HashSet<>(oldLiveNodes);
+ old.removeAll(newLiveNodes);
+ if (!old.isEmpty()) {
+ lostNodes.addAll(old);
+ }
+ newLiveNodes.removeAll(oldLiveNodes);
+ if (!newLiveNodes.isEmpty()) {
+ addedNodes.addAll(newLiveNodes);
+ }
+ }
+ }
+
+ public static class TestEventMarkerAction extends TriggerActionBase {
+
+ public TestEventMarkerAction() {
+ actionConstructorCalled.countDown();
+ }
+
+ @Override
+ public void process(TriggerEvent event, ActionContext actionContext) {
+ boolean locked = lock.tryLock();
+ if (!locked) {
+ log.info("We should never have a tryLock fail because actions are never supposed to be executed concurrently");
+ return;
+ }
+ try {
+ events.add(event);
+ getTriggerFiredLatch().countDown();
+ } catch (Throwable t) {
+ log.debug("--throwable", t);
+ throw t;
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ @Override
+ public void init(Map<String, String> args) {
+ log.info("TestEventMarkerAction init");
+ actionInitCalled.countDown();
+ super.init(args);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ed9e5eb7/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledTriggerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledTriggerIntegrationTest.java
new file mode 100644
index 0000000..24e7420
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledTriggerIntegrationTest.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud.autoscaling;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.util.LogLevel;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.createAutoScalingRequest;
+
+/**
+ * Integration test for {@link ScheduledTrigger}
+ */
+@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG")
+@BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 26-Mar-2018
+public class ScheduledTriggerIntegrationTest extends SolrCloudTestCase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private static CountDownLatch triggerFiredLatch;
+ private static Set<TriggerEvent> events = ConcurrentHashMap.newKeySet();
+ private static AtomicReference<Map<String, Object>> actionContextPropertiesRef = new AtomicReference<>();
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(2)
+ .addConfig("conf", configset("cloud-minimal"))
+ .configure();
+ // disable .scheduled_maintenance
+ String suspendTriggerCommand = "{" +
+ "'suspend-trigger' : {'name' : '.scheduled_maintenance'}" +
+ "}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
+ SolrClient solrClient = cluster.getSolrClient();
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ triggerFiredLatch = new CountDownLatch(1);
+ }
+
+ @Test
+ public void testScheduledTrigger() throws Exception {
+ CloudSolrClient solrClient = cluster.getSolrClient();
+
+ // this collection will place 2 cores on 1st node and 1 core on 2nd node
+ String collectionName = "testScheduledTrigger";
+ CollectionAdminRequest.createCollection(collectionName, 1, 3)
+ .setMaxShardsPerNode(5).process(solrClient);
+ waitForState("", collectionName, clusterShape(1, 3));
+
+ // create a policy which allows only 1 core per node thereby creating a violation for the above collection
+ String setClusterPolicy = "{\n" +
+ " \"set-cluster-policy\" : [\n" +
+ " {\"cores\" : \"<2\", \"node\" : \"#EACH\"}\n" +
+ " ]\n" +
+ "}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setClusterPolicy);
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ // start a new node which can be used to balance the cluster as per policy
+ JettySolrRunner newNode = cluster.startJettySolrRunner();
+ cluster.waitForAllNodes(10);
+
+ String setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'sched_trigger_integration1'," +
+ "'event' : 'scheduled'," +
+ "'startTime' : '" + new Date().toInstant().toString() + "'" +
+ "'every' : '+3SECONDS'" +
+ "'actions' : [" +
+ "{'name' : 'compute','class':'" + ComputePlanAction.class.getName() + "'}," +
+ "{'name' : 'execute','class':'" + ExecutePlanAction.class.getName() + "'}," +
+ "{'name' : 'recorder', 'class': '" + ContextPropertiesRecorderAction.class.getName() + "'}" +
+ "]}}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ assertTrue("ScheduledTrigger did not fire within 20 seconds", triggerFiredLatch.await(20, TimeUnit.SECONDS));
+ assertEquals(1, events.size());
+ Map<String, Object> actionContextProps = actionContextPropertiesRef.get();
+ assertNotNull(actionContextProps);
+ TriggerEvent event = events.iterator().next();
+ List<SolrRequest> operations = (List<SolrRequest>) actionContextProps.get("operations");
+ assertNotNull(operations);
+ assertEquals(1, operations.size());
+ for (SolrRequest operation : operations) {
+ SolrParams params = operation.getParams();
+ assertEquals(newNode.getNodeName(), params.get("targetNode"));
+ }
+ }
+
+ public static class ContextPropertiesRecorderAction extends TriggerActionBase {
+ @Override
+ public void process(TriggerEvent event, ActionContext actionContext) {
+ actionContextPropertiesRef.set(actionContext.getProperties());
+ try {
+ events.add(event);
+ triggerFiredLatch.countDown();
+ } catch (Throwable t) {
+ log.debug("--throwable", t);
+ throw t;
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ed9e5eb7/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
new file mode 100644
index 0000000..547be5c
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud.autoscaling;
+
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.util.concurrent.AtomicDouble;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.cloud.SolrCloudManager;
+import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
+import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
+import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.util.LogLevel;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.createAutoScalingRequest;
+import static org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.WAIT_FOR_DELTA_NANOS;
+import static org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.timeSource;
+
+/**
+ * Integration test for {@link SearchRateTrigger}
+ */
+@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG")
+@LuceneTestCase.BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-12028")
+public class SearchRateTriggerIntegrationTest extends SolrCloudTestCase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private static CountDownLatch triggerFiredLatch = new CountDownLatch(1);
+ private static CountDownLatch listenerCreated = new CountDownLatch(1);
+ private static int waitForSeconds = 1;
+ private static Set<TriggerEvent> events = ConcurrentHashMap.newKeySet();
+ private static Map<String, List<CapturedEvent>> listenerEvents = new HashMap<>();
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(5)
+ .addConfig("conf", configset("cloud-minimal"))
+ .configure();
+ // disable .scheduled_maintenance
+ String suspendTriggerCommand = "{" +
+ "'suspend-trigger' : {'name' : '.scheduled_maintenance'}" +
+ "}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
+ SolrClient solrClient = cluster.getSolrClient();
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ }
+
+ @Test
+ public void testSearchRate() throws Exception {
+ CloudSolrClient solrClient = cluster.getSolrClient();
+ String COLL1 = "collection1";
+ CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(COLL1,
+ "conf", 1, 2);
+ create.process(solrClient);
+ String setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'search_rate_trigger'," +
+ "'event' : 'searchRate'," +
+ "'waitFor' : '" + waitForSeconds + "s'," +
+ "'enabled' : true," +
+ "'rate' : 1.0," +
+ "'actions' : [" +
+ "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," +
+ "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}," +
+ "{'name':'test','class':'" + TestSearchRateAction.class.getName() + "'}" +
+ "]" +
+ "}}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ String setListenerCommand1 = "{" +
+ "'set-listener' : " +
+ "{" +
+ "'name' : 'srt'," +
+ "'trigger' : 'search_rate_trigger'," +
+ "'stage' : ['FAILED','SUCCEEDED']," +
+ "'afterAction': ['compute', 'execute', 'test']," +
+ "'class' : '" + TestTriggerListener.class.getName() + "'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setListenerCommand1);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ SolrParams query = params(CommonParams.Q, "*:*");
+ for (int i = 0; i < 500; i++) {
+ solrClient.query(COLL1, query);
+ }
+ boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
+ assertTrue("The trigger did not fire at all", await);
+ // wait for listener to capture the SUCCEEDED stage
+ Thread.sleep(5000);
+ List<CapturedEvent> events = listenerEvents.get("srt");
+ assertEquals(listenerEvents.toString(), 4, events.size());
+ assertEquals("AFTER_ACTION", events.get(0).stage.toString());
+ assertEquals("compute", events.get(0).actionName);
+ assertEquals("AFTER_ACTION", events.get(1).stage.toString());
+ assertEquals("execute", events.get(1).actionName);
+ assertEquals("AFTER_ACTION", events.get(2).stage.toString());
+ assertEquals("test", events.get(2).actionName);
+ assertEquals("SUCCEEDED", events.get(3).stage.toString());
+ assertNull(events.get(3).actionName);
+
+ CapturedEvent ev = events.get(0);
+ long now = timeSource.getTimeNs();
+ // verify waitFor
+ assertTrue(TimeUnit.SECONDS.convert(waitForSeconds, TimeUnit.NANOSECONDS) - WAIT_FOR_DELTA_NANOS <= now - ev.event.getEventTime());
+ Map<String, Double> nodeRates = (Map<String, Double>) ev.event.getProperties().get("node");
+ assertNotNull("nodeRates", nodeRates);
+ assertTrue(nodeRates.toString(), nodeRates.size() > 0);
+ AtomicDouble totalNodeRate = new AtomicDouble();
+ nodeRates.forEach((n, r) -> totalNodeRate.addAndGet(r));
+ List<ReplicaInfo> replicaRates = (List<ReplicaInfo>) ev.event.getProperties().get("replica");
+ assertNotNull("replicaRates", replicaRates);
+ assertTrue(replicaRates.toString(), replicaRates.size() > 0);
+ AtomicDouble totalReplicaRate = new AtomicDouble();
+ replicaRates.forEach(r -> {
+ assertTrue(r.toString(), r.getVariable("rate") != null);
+ totalReplicaRate.addAndGet((Double) r.getVariable("rate"));
+ });
+ Map<String, Object> shardRates = (Map<String, Object>) ev.event.getProperties().get("shard");
+ assertNotNull("shardRates", shardRates);
+ assertEquals(shardRates.toString(), 1, shardRates.size());
+ shardRates = (Map<String, Object>) shardRates.get(COLL1);
+ assertNotNull("shardRates", shardRates);
+ assertEquals(shardRates.toString(), 1, shardRates.size());
+ AtomicDouble totalShardRate = new AtomicDouble();
+ shardRates.forEach((s, r) -> totalShardRate.addAndGet((Double) r));
+ Map<String, Double> collectionRates = (Map<String, Double>) ev.event.getProperties().get("collection");
+ assertNotNull("collectionRates", collectionRates);
+ assertEquals(collectionRates.toString(), 1, collectionRates.size());
+ Double collectionRate = collectionRates.get(COLL1);
+ assertNotNull(collectionRate);
+ assertTrue(collectionRate > 5.0);
+ assertEquals(collectionRate, totalNodeRate.get(), 5.0);
+ assertEquals(collectionRate, totalShardRate.get(), 5.0);
+ assertEquals(collectionRate, totalReplicaRate.get(), 5.0);
+
+ // check operations
+ List<Map<String, Object>> ops = (List<Map<String, Object>>) ev.context.get("properties.operations");
+ assertNotNull(ops);
+ assertTrue(ops.size() > 1);
+ for (Map<String, Object> m : ops) {
+ assertEquals("ADDREPLICA", m.get("params.action"));
+ }
+ }
+
+ public static class TestSearchRateAction extends TriggerActionBase {
+
+ @Override
+ public void process(TriggerEvent event, ActionContext context) throws Exception {
+ try {
+ events.add(event);
+ long currentTimeNanos = timeSource.getTimeNs();
+ long eventTimeNanos = event.getEventTime();
+ long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS;
+ if (currentTimeNanos - eventTimeNanos <= waitForNanos) {
+ fail(event.source + " was fired before the configured waitFor period");
+ }
+ triggerFiredLatch.countDown();
+ } catch (Throwable t) {
+ log.debug("--throwable", t);
+ throw t;
+ }
+ }
+ }
+
+ public static class TestTriggerListener extends TriggerListenerBase {
+ @Override
+ public void init(SolrCloudManager cloudManager, AutoScalingConfig.TriggerListenerConfig config) {
+ super.init(cloudManager, config);
+ listenerCreated.countDown();
+ }
+
+ @Override
+ public synchronized void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName,
+ ActionContext context, Throwable error, String message) {
+ List<CapturedEvent> lst = listenerEvents.computeIfAbsent(config.name, s -> new ArrayList<>());
+ lst.add(new CapturedEvent(timeSource.getTimeNs(), context, config, stage, actionName, event, message));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ed9e5eb7/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerCooldownIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerCooldownIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerCooldownIntegrationTest.java
new file mode 100644
index 0000000..8d69bad
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerCooldownIntegrationTest.java
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud.autoscaling;
+
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.cloud.SolrCloudManager;
+import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
+import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.params.AutoScalingParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.util.LogLevel;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.createAutoScalingRequest;
+import static org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.WAIT_FOR_DELTA_NANOS;
+import static org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.timeSource;
+
+@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG")
+public class TriggerCooldownIntegrationTest extends SolrCloudTestCase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ static Map<String, List<CapturedEvent>> listenerEvents = new HashMap<>();
+ static CountDownLatch listenerCreated = new CountDownLatch(1);
+ static boolean failDummyAction = false;
+ private static CountDownLatch actionConstructorCalled = new CountDownLatch(1);
+ private static CountDownLatch actionInitCalled = new CountDownLatch(1);
+ private static CountDownLatch triggerFiredLatch = new CountDownLatch(1);
+ private static int waitForSeconds = 1;
+ private static AtomicBoolean triggerFired = new AtomicBoolean();
+ private static Set<TriggerEvent> events = ConcurrentHashMap.newKeySet();
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(2)
+ .addConfig("conf", configset("cloud-minimal"))
+ .configure();
+ // disable .scheduled_maintenance
+ String suspendTriggerCommand = "{" +
+ "'suspend-trigger' : {'name' : '.scheduled_maintenance'}" +
+ "}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
+ SolrClient solrClient = cluster.getSolrClient();
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ }
+
+ @Test
+ public void testCooldown() throws Exception {
+ CloudSolrClient solrClient = cluster.getSolrClient();
+ failDummyAction = false;
+ waitForSeconds = 1;
+ String setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'node_added_cooldown_trigger'," +
+ "'event' : 'nodeAdded'," +
+ "'waitFor' : '" + waitForSeconds + "s'," +
+ "'enabled' : true," +
+ "'actions' : [" +
+ "{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}" +
+ "]" +
+ "}}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ String setListenerCommand1 = "{" +
+ "'set-listener' : " +
+ "{" +
+ "'name' : 'bar'," +
+ "'trigger' : 'node_added_cooldown_trigger'," +
+ "'stage' : ['FAILED','SUCCEEDED', 'IGNORED']," +
+ "'class' : '" + TestTriggerListener.class.getName() + "'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setListenerCommand1);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ listenerCreated = new CountDownLatch(1);
+ listenerEvents.clear();
+
+ JettySolrRunner newNode = cluster.startJettySolrRunner();
+ boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
+ assertTrue("The trigger did not fire at all", await);
+ assertTrue(triggerFired.get());
+ // wait for listener to capture the SUCCEEDED stage
+ Thread.sleep(1000);
+
+ List<CapturedEvent> capturedEvents = listenerEvents.get("bar");
+ // we may get a few IGNORED events if other tests caused events within cooldown period
+ assertTrue(capturedEvents.toString(), capturedEvents.size() > 0);
+ long prevTimestamp = capturedEvents.get(capturedEvents.size() - 1).timestamp;
+
+ // reset the trigger and captured events
+ listenerEvents.clear();
+ triggerFiredLatch = new CountDownLatch(1);
+ triggerFired.compareAndSet(true, false);
+
+ JettySolrRunner newNode2 = cluster.startJettySolrRunner();
+ await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
+ assertTrue("The trigger did not fire at all", await);
+ // wait for listener to capture the SUCCEEDED stage
+ Thread.sleep(2000);
+
+ // there must be at least one IGNORED event due to cooldown, and one SUCCEEDED event
+ capturedEvents = listenerEvents.get("bar");
+ assertEquals(capturedEvents.toString(), 1, capturedEvents.size());
+ CapturedEvent ev = capturedEvents.get(0);
+ assertEquals(ev.toString(), TriggerEventProcessorStage.SUCCEEDED, ev.stage);
+ // the difference between timestamps of the first SUCCEEDED and the last SUCCEEDED
+ // must be larger than cooldown period
+ assertTrue("timestamp delta is less than default cooldown period", ev.timestamp - prevTimestamp > TimeUnit.SECONDS.toNanos(ScheduledTriggers.DEFAULT_COOLDOWN_PERIOD_SECONDS));
+ prevTimestamp = ev.timestamp;
+
+ // this also resets the cooldown period
+ long modifiedCooldownPeriodSeconds = 7;
+ String setPropertiesCommand = "{\n" +
+ "\t\"set-properties\" : {\n" +
+ "\t\t\"" + AutoScalingParams.TRIGGER_COOLDOWN_PERIOD_SECONDS + "\" : " + modifiedCooldownPeriodSeconds + "\n" +
+ "\t}\n" +
+ "}";
+ solrClient.request(createAutoScalingRequest(SolrRequest.METHOD.POST, setPropertiesCommand));
+ req = createAutoScalingRequest(SolrRequest.METHOD.GET, null);
+ response = solrClient.request(req);
+
+ // reset the trigger and captured events
+ listenerEvents.clear();
+ triggerFiredLatch = new CountDownLatch(1);
+ triggerFired.compareAndSet(true, false);
+
+ JettySolrRunner newNode3 = cluster.startJettySolrRunner();
+ await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
+ assertTrue("The trigger did not fire at all", await);
+ triggerFiredLatch = new CountDownLatch(1);
+ triggerFired.compareAndSet(true, false);
+ // add another node
+ JettySolrRunner newNode4 = cluster.startJettySolrRunner();
+ await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
+ assertTrue("The trigger did not fire at all", await);
+ // wait for listener to capture the SUCCEEDED stage
+ Thread.sleep(2000);
+
+ // there must be two SUCCEEDED (due to newNode3 and newNode4) and maybe some ignored events
+ capturedEvents = listenerEvents.get("bar");
+ assertTrue(capturedEvents.toString(), capturedEvents.size() >= 2);
+ // first event should be SUCCEEDED
+ ev = capturedEvents.get(0);
+ assertEquals(ev.toString(), TriggerEventProcessorStage.SUCCEEDED, ev.stage);
+
+ ev = capturedEvents.get(capturedEvents.size() - 1);
+ assertEquals(ev.toString(), TriggerEventProcessorStage.SUCCEEDED, ev.stage);
+ // the difference between timestamps of the first SUCCEEDED and the last SUCCEEDED
+ // must be larger than the modified cooldown period
+ assertTrue("timestamp delta is less than default cooldown period", ev.timestamp - prevTimestamp > TimeUnit.SECONDS.toNanos(modifiedCooldownPeriodSeconds));
+ }
+
+ public static class TestTriggerAction extends TriggerActionBase {
+
+ public TestTriggerAction() {
+ actionConstructorCalled.countDown();
+ }
+
+ @Override
+ public void process(TriggerEvent event, ActionContext actionContext) {
+ try {
+ if (triggerFired.compareAndSet(false, true)) {
+ events.add(event);
+ long currentTimeNanos = timeSource.getTimeNs();
+ long eventTimeNanos = event.getEventTime();
+ long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS;
+ if (currentTimeNanos - eventTimeNanos <= waitForNanos) {
+ fail(event.source + " was fired before the configured waitFor period");
+ }
+ triggerFiredLatch.countDown();
+ } else {
+ fail(event.source + " was fired more than once!");
+ }
+ } catch (Throwable t) {
+ log.debug("--throwable", t);
+ throw t;
+ }
+ }
+
+ @Override
+ public void init(Map<String, String> args) {
+ log.info("TestTriggerAction init");
+ actionInitCalled.countDown();
+ super.init(args);
+ }
+ }
+
+ public static class TestTriggerListener extends TriggerListenerBase {
+ @Override
+ public void init(SolrCloudManager cloudManager, AutoScalingConfig.TriggerListenerConfig config) {
+ super.init(cloudManager, config);
+ listenerCreated.countDown();
+ }
+
+ @Override
+ public synchronized void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName,
+ ActionContext context, Throwable error, String message) {
+ List<CapturedEvent> lst = listenerEvents.computeIfAbsent(config.name, s -> new ArrayList<>());
+ lst.add(new CapturedEvent(timeSource.getTimeNs(), context, config, stage, actionName, event, message));
+ }
+ }
+}
[14/34] lucene-solr:jira/solr-12095: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/lucene-solr
Posted by sh...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/lucene-solr
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ae6d29f0
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ae6d29f0
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ae6d29f0
Branch: refs/heads/jira/solr-12095
Commit: ae6d29f0aeb8e4774e9eec5696f81302359cb0b4
Parents: bc40f6c 0e5374e
Author: Karl Wright <Da...@gmail.com>
Authored: Fri Mar 30 06:43:57 2018 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Fri Mar 30 06:43:57 2018 -0400
----------------------------------------------------------------------
dev-tools/scripts/reproduceJenkinsFailures.py | 1 +
solr/CHANGES.txt | 2 +
.../apache/solr/core/CorePropertiesLocator.java | 2 +-
.../solr/cloud/DocValuesNotIndexedTest.java | 1 +
.../solr/cloud/RestartWhileUpdatingTest.java | 1 +
.../apache/solr/cloud/TestCloudConsistency.java | 2 +
.../org/apache/solr/cloud/TestPullReplica.java | 5 +-
.../apache/solr/cloud/TestSegmentSorting.java | 2 +
.../CollectionsAPIDistributedZkTest.java | 1 +
.../MetricTriggerIntegrationTest.java | 242 ++++
.../NodeAddedTriggerIntegrationTest.java | 300 +++++
.../NodeLostTriggerIntegrationTest.java | 322 +++++
.../NodeMarkersRegistrationTest.java | 269 ++++
.../autoscaling/RestoreTriggerStateTest.java | 169 +++
.../ScheduledTriggerIntegrationTest.java | 142 +++
.../SearchRateTriggerIntegrationTest.java | 217 ++++
.../TriggerCooldownIntegrationTest.java | 238 ++++
.../autoscaling/TriggerIntegrationTest.java | 1161 +-----------------
.../TriggerSetPropertiesIntegrationTest.java | 195 +++
.../autoscaling/sim/TestTriggerIntegration.java | 1 +
.../cloud/hdfs/HdfsBasicDistributedZk2Test.java | 2 +
.../admin/AutoscalingHistoryHandlerTest.java | 3 +-
solr/solr-ref-guide/src/highlighting.adoc | 23 +-
23 files changed, 2183 insertions(+), 1118 deletions(-)
----------------------------------------------------------------------