You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/06/18 17:46:00 UTC
svn commit: r1603492 [2/2] - in /lucene/dev/branches/lucene5752: ./ lucene/
lucene/codecs/ lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/
lucene/codecs/src/java/org/apache/lucene/codecs/bloom/
lucene/codecs/src/java/org/apache/lucene/codec...
Modified: lucene/dev/branches/lucene5752/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java (original)
+++ lucene/dev/branches/lucene5752/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java Wed Jun 18 15:45:58 2014
@@ -54,7 +54,7 @@ import org.apache.lucene.spatial.query.U
* <li>Only indexes Rectangles; just one per field value.</li>
* <li>Can query only by a Rectangle.</li>
* <li>Supports all {@link SpatialOperation}s.</li>
- * <li>Uses the FieldCache for any sorting / relevancy.</li>
+ * <li>Uses the DocValues API for any sorting / relevancy.</li>
* </ul>
*
* <h4>Implementation:</h4>
@@ -116,7 +116,7 @@ public class BBoxStrategy extends Spatia
public Field[] createIndexableFields(Shape shape) {
if (shape instanceof Rectangle)
return createIndexableFields((Rectangle)shape);
- throw new UnsupportedOperationException("Can only index Rectangle, not " + shape);
+ throw new UnsupportedOperationException("Can only index a Rectangle, not " + shape);
}
public Field[] createIndexableFields(Rectangle bbox) {
@@ -174,7 +174,7 @@ public class BBoxStrategy extends Spatia
throw new UnsupportedOperationException("Can only query by Rectangle, not " + shape);
Rectangle bbox = (Rectangle) shape;
- Query spatial = null;
+ Query spatial;
// Useful for understanding Relations:
// http://edndoc.esri.com/arcsde/9.1/general_topics/understand_spatial_relations.htm
@@ -212,10 +212,10 @@ public class BBoxStrategy extends Spatia
// docMinY <= queryExtent.getMinY() AND docMaxY >= queryExtent.getMaxY()
Query qMinY = NumericRangeQuery.newDoubleRange(field_minY, precisionStep, null, bbox.getMinY(), false, true);
Query qMaxY = NumericRangeQuery.newDoubleRange(field_maxY, precisionStep, bbox.getMaxY(), null, true, false);
- Query yConditions = this.makeQuery(new Query[]{qMinY, qMaxY}, BooleanClause.Occur.MUST);
+ Query yConditions = this.makeQuery(BooleanClause.Occur.MUST, qMinY, qMaxY);
// X conditions
- Query xConditions = null;
+ Query xConditions;
// queries that do not cross the date line
if (!bbox.getCrossesDateLine()) {
@@ -225,7 +225,7 @@ public class BBoxStrategy extends Spatia
// docMinX <= queryExtent.getMinX() AND docMaxX >= queryExtent.getMaxX()
Query qMinX = NumericRangeQuery.newDoubleRange(field_minX, precisionStep, null, bbox.getMinX(), false, true);
Query qMaxX = NumericRangeQuery.newDoubleRange(field_maxX, precisionStep, bbox.getMaxX(), null, true, false);
- Query qMinMax = this.makeQuery(new Query[]{qMinX, qMaxX}, BooleanClause.Occur.MUST);
+ Query qMinMax = this.makeQuery(BooleanClause.Occur.MUST, qMinX, qMaxX);
Query qNonXDL = this.makeXDL(false, qMinMax);
// X Conditions for documents that cross the date line,
@@ -234,11 +234,11 @@ public class BBoxStrategy extends Spatia
// docMinXLeft <= queryExtent.getMinX() OR docMaxXRight >= queryExtent.getMaxX()
Query qXDLLeft = NumericRangeQuery.newDoubleRange(field_minX, precisionStep, null, bbox.getMinX(), false, true);
Query qXDLRight = NumericRangeQuery.newDoubleRange(field_maxX, precisionStep, bbox.getMaxX(), null, true, false);
- Query qXDLLeftRight = this.makeQuery(new Query[]{qXDLLeft, qXDLRight}, BooleanClause.Occur.SHOULD);
+ Query qXDLLeftRight = this.makeQuery(BooleanClause.Occur.SHOULD, qXDLLeft, qXDLRight);
Query qXDL = this.makeXDL(true, qXDLLeftRight);
// apply the non-XDL and XDL conditions
- xConditions = this.makeQuery(new Query[]{qNonXDL, qXDL}, BooleanClause.Occur.SHOULD);
+ xConditions = this.makeQuery(BooleanClause.Occur.SHOULD, qNonXDL, qXDL);
// queries that cross the date line
} else {
@@ -251,13 +251,13 @@ public class BBoxStrategy extends Spatia
// docMinXLeft <= queryExtent.getMinX() AND docMaxXRight >= queryExtent.getMaxX()
Query qXDLLeft = NumericRangeQuery.newDoubleRange(field_minX, precisionStep, null, bbox.getMinX(), false, true);
Query qXDLRight = NumericRangeQuery.newDoubleRange(field_maxX, precisionStep, bbox.getMaxX(), null, true, false);
- Query qXDLLeftRight = this.makeQuery(new Query[]{qXDLLeft, qXDLRight}, BooleanClause.Occur.MUST);
+ Query qXDLLeftRight = this.makeQuery(BooleanClause.Occur.MUST, qXDLLeft, qXDLRight);
xConditions = this.makeXDL(true, qXDLLeftRight);
}
// both X and Y conditions must occur
- return this.makeQuery(new Query[]{xConditions, yConditions}, BooleanClause.Occur.MUST);
+ return this.makeQuery(BooleanClause.Occur.MUST, xConditions, yConditions);
}
/**
@@ -274,10 +274,10 @@ public class BBoxStrategy extends Spatia
// docMinY > queryExtent.getMaxY() OR docMaxY < queryExtent.getMinY()
Query qMinY = NumericRangeQuery.newDoubleRange(field_minY, precisionStep, bbox.getMaxY(), null, false, false);
Query qMaxY = NumericRangeQuery.newDoubleRange(field_maxY, precisionStep, null, bbox.getMinY(), false, false);
- Query yConditions = this.makeQuery(new Query[]{qMinY, qMaxY}, BooleanClause.Occur.SHOULD);
+ Query yConditions = this.makeQuery(BooleanClause.Occur.SHOULD, qMinY, qMaxY);
// X conditions
- Query xConditions = null;
+ Query xConditions;
// queries that do not cross the date line
if (!bbox.getCrossesDateLine()) {
@@ -286,7 +286,7 @@ public class BBoxStrategy extends Spatia
// docMinX > queryExtent.getMaxX() OR docMaxX < queryExtent.getMinX()
Query qMinX = NumericRangeQuery.newDoubleRange(field_minX, precisionStep, bbox.getMaxX(), null, false, false);
Query qMaxX = NumericRangeQuery.newDoubleRange(field_maxX, precisionStep, null, bbox.getMinX(), false, false);
- Query qMinMax = this.makeQuery(new Query[]{qMinX, qMaxX}, BooleanClause.Occur.SHOULD);
+ Query qMinMax = this.makeQuery(BooleanClause.Occur.SHOULD, qMinX, qMaxX);
Query qNonXDL = this.makeXDL(false, qMinMax);
// X Conditions for documents that cross the date line,
@@ -298,11 +298,11 @@ public class BBoxStrategy extends Spatia
// (docMinXRight > queryExtent.getMaxX()) equates to (-180.0 > queryExtent.getMaxX()) and is ignored
Query qMinXLeft = NumericRangeQuery.newDoubleRange(field_minX, precisionStep, bbox.getMaxX(), null, false, false);
Query qMaxXRight = NumericRangeQuery.newDoubleRange(field_maxX, precisionStep, null, bbox.getMinX(), false, false);
- Query qLeftRight = this.makeQuery(new Query[]{qMinXLeft, qMaxXRight}, BooleanClause.Occur.MUST);
+ Query qLeftRight = this.makeQuery(BooleanClause.Occur.MUST, qMinXLeft, qMaxXRight);
Query qXDL = this.makeXDL(true, qLeftRight);
// apply the non-XDL and XDL conditions
- xConditions = this.makeQuery(new Query[]{qNonXDL, qXDL}, BooleanClause.Occur.SHOULD);
+ xConditions = this.makeQuery(BooleanClause.Occur.SHOULD, qNonXDL, qXDL);
// queries that cross the date line
} else {
@@ -315,9 +315,9 @@ public class BBoxStrategy extends Spatia
Query qMaxXLeft = NumericRangeQuery.newDoubleRange(field_maxX, precisionStep, null, bbox.getMinX(), false, false);
Query qMinXRight = NumericRangeQuery.newDoubleRange(field_minX, precisionStep, bbox.getMaxX(), null, false, false);
Query qMaxXRight = NumericRangeQuery.newDoubleRange(field_maxX, precisionStep, null, -180.0, false, false);
- Query qLeft = this.makeQuery(new Query[]{qMinXLeft, qMaxXLeft}, BooleanClause.Occur.SHOULD);
- Query qRight = this.makeQuery(new Query[]{qMinXRight, qMaxXRight}, BooleanClause.Occur.SHOULD);
- Query qLeftRight = this.makeQuery(new Query[]{qLeft, qRight}, BooleanClause.Occur.MUST);
+ Query qLeft = this.makeQuery(BooleanClause.Occur.SHOULD, qMinXLeft, qMaxXLeft);
+ Query qRight = this.makeQuery(BooleanClause.Occur.SHOULD, qMinXRight, qMaxXRight);
+ Query qLeftRight = this.makeQuery(BooleanClause.Occur.MUST, qLeft, qRight);
// No need to search for documents that do not cross the date line
@@ -325,7 +325,7 @@ public class BBoxStrategy extends Spatia
}
// either X or Y conditions should occur
- return this.makeQuery(new Query[]{xConditions, yConditions}, BooleanClause.Occur.SHOULD);
+ return this.makeQuery(BooleanClause.Occur.SHOULD, xConditions, yConditions);
}
/**
@@ -359,13 +359,13 @@ public class BBoxStrategy extends Spatia
// switch to a NOT Disjoint query
// MUST_NOT causes a problem when it's the only clause type within a BooleanQuery,
- // to get round it we add all documents as a SHOULD
+ // to get around it we add all documents as a SHOULD
// there must be an envelope, it must not be disjoint
Query qDisjoint = makeDisjoint(bbox);
Query qIsNonXDL = this.makeXDL(false);
Query qIsXDL = this.makeXDL(true);
- Query qHasEnv = this.makeQuery(new Query[]{qIsNonXDL, qIsXDL}, BooleanClause.Occur.SHOULD);
+ Query qHasEnv = this.makeQuery(BooleanClause.Occur.SHOULD, qIsNonXDL, qIsXDL);
BooleanQuery qNotDisjoint = new BooleanQuery();
qNotDisjoint.add(qHasEnv, BooleanClause.Occur.MUST);
qNotDisjoint.add(qDisjoint, BooleanClause.Occur.MUST_NOT);
@@ -380,11 +380,11 @@ public class BBoxStrategy extends Spatia
/**
* Makes a boolean query based upon a collection of queries and a logical operator.
*
- * @param queries the query collection
* @param occur the logical operator
+ * @param queries the query collection
* @return the query
*/
- BooleanQuery makeQuery(Query[] queries, BooleanClause.Occur occur) {
+ BooleanQuery makeQuery(BooleanClause.Occur occur, Query... queries) {
BooleanQuery bq = new BooleanQuery();
for (Query query : queries) {
bq.add(query, occur);
@@ -406,10 +406,10 @@ public class BBoxStrategy extends Spatia
// docMinY >= queryExtent.getMinY() AND docMaxY <= queryExtent.getMaxY()
Query qMinY = NumericRangeQuery.newDoubleRange(field_minY, precisionStep, bbox.getMinY(), null, true, false);
Query qMaxY = NumericRangeQuery.newDoubleRange(field_maxY, precisionStep, null, bbox.getMaxY(), false, true);
- Query yConditions = this.makeQuery(new Query[]{qMinY, qMaxY}, BooleanClause.Occur.MUST);
+ Query yConditions = this.makeQuery(BooleanClause.Occur.MUST, qMinY, qMaxY);
// X conditions
- Query xConditions = null;
+ Query xConditions;
// X Conditions for documents that cross the date line,
// the left portion of the document must be within the left portion of the query,
@@ -418,7 +418,7 @@ public class BBoxStrategy extends Spatia
// AND docMinXRight >= -180.0 AND docMaxXRight <= queryExtent.getMaxX()
Query qXDLLeft = NumericRangeQuery.newDoubleRange(field_minX, precisionStep, bbox.getMinX(), null, true, false);
Query qXDLRight = NumericRangeQuery.newDoubleRange(field_maxX, precisionStep, null, bbox.getMaxX(), false, true);
- Query qXDLLeftRight = this.makeQuery(new Query[]{qXDLLeft, qXDLRight}, BooleanClause.Occur.MUST);
+ Query qXDLLeftRight = this.makeQuery(BooleanClause.Occur.MUST, qXDLLeft, qXDLRight);
Query qXDL = this.makeXDL(true, qXDLLeftRight);
// queries that do not cross the date line
@@ -428,12 +428,12 @@ public class BBoxStrategy extends Spatia
// docMinX >= queryExtent.getMinX() AND docMaxX <= queryExtent.getMaxX()
Query qMinX = NumericRangeQuery.newDoubleRange(field_minX, precisionStep, bbox.getMinX(), null, true, false);
Query qMaxX = NumericRangeQuery.newDoubleRange(field_maxX, precisionStep, null, bbox.getMaxX(), false, true);
- Query qMinMax = this.makeQuery(new Query[]{qMinX, qMaxX}, BooleanClause.Occur.MUST);
+ Query qMinMax = this.makeQuery(BooleanClause.Occur.MUST, qMinX, qMaxX);
Query qNonXDL = this.makeXDL(false, qMinMax);
// apply the non-XDL or XDL X conditions
if ((bbox.getMinX() <= -180.0) && bbox.getMaxX() >= 180.0) {
- xConditions = this.makeQuery(new Query[]{qNonXDL, qXDL}, BooleanClause.Occur.SHOULD);
+ xConditions = this.makeQuery(BooleanClause.Occur.SHOULD, qNonXDL, qXDL);
} else {
xConditions = qNonXDL;
}
@@ -447,25 +447,25 @@ public class BBoxStrategy extends Spatia
// docMinX >= queryExtent.getMinX() AND docMaxX <= 180.0
Query qMinXLeft = NumericRangeQuery.newDoubleRange(field_minX, precisionStep, bbox.getMinX(), null, true, false);
Query qMaxXLeft = NumericRangeQuery.newDoubleRange(field_maxX, precisionStep, null, 180.0, false, true);
- Query qLeft = this.makeQuery(new Query[]{qMinXLeft, qMaxXLeft}, BooleanClause.Occur.MUST);
+ Query qLeft = this.makeQuery(BooleanClause.Occur.MUST, qMinXLeft, qMaxXLeft);
// the document should be within the right portion of the query
// docMinX >= -180.0 AND docMaxX <= queryExtent.getMaxX()
Query qMinXRight = NumericRangeQuery.newDoubleRange(field_minX, precisionStep, -180.0, null, true, false);
Query qMaxXRight = NumericRangeQuery.newDoubleRange(field_maxX, precisionStep, null, bbox.getMaxX(), false, true);
- Query qRight = this.makeQuery(new Query[]{qMinXRight, qMaxXRight}, BooleanClause.Occur.MUST);
+ Query qRight = this.makeQuery(BooleanClause.Occur.MUST, qMinXRight, qMaxXRight);
// either left or right conditions should occur,
// apply the left and right conditions to documents that do not cross the date line
- Query qLeftRight = this.makeQuery(new Query[]{qLeft, qRight}, BooleanClause.Occur.SHOULD);
+ Query qLeftRight = this.makeQuery(BooleanClause.Occur.SHOULD, qLeft, qRight);
Query qNonXDL = this.makeXDL(false, qLeftRight);
// apply the non-XDL and XDL conditions
- xConditions = this.makeQuery(new Query[]{qNonXDL, qXDL}, BooleanClause.Occur.SHOULD);
+ xConditions = this.makeQuery(BooleanClause.Occur.SHOULD, qNonXDL, qXDL);
}
// both X and Y conditions must occur
- return this.makeQuery(new Query[]{xConditions, yConditions}, BooleanClause.Occur.MUST);
+ return this.makeQuery(BooleanClause.Occur.MUST, xConditions, yConditions);
}
/**
Modified: lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java (original)
+++ lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java Wed Jun 18 15:45:58 2014
@@ -18,24 +18,106 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+import java.lang.reflect.Field;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.IdentityHashMap;
import java.util.Map;
+import java.util.Set;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.CloseableThreadLocal;
+import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.RamUsageTester;
+import org.apache.lucene.util.packed.PackedInts;
/**
* Common tests to all index formats.
*/
abstract class BaseIndexFileFormatTestCase extends LuceneTestCase {
+ // metadata or Directory-level objects
+ private static final Set<Class<?>> EXCLUDED_CLASSES = Collections.newSetFromMap(new IdentityHashMap<Class<?>,Boolean>());
+
+ // Notorious singletons
+ private static final Set<Object> EXCLUDED_OBJECTS = Collections.newSetFromMap(new IdentityHashMap<Object,Boolean>());
+
+ static {
+ // Directory objects, don't take into account eg. the NIO buffers
+ EXCLUDED_CLASSES.add(Directory.class);
+ EXCLUDED_CLASSES.add(IndexInput.class);
+
+ // used for thread management, not by the index
+ EXCLUDED_CLASSES.add(CloseableThreadLocal.class);
+
+ // don't follow references to the top-level reader
+ EXCLUDED_CLASSES.add(IndexReader.class);
+ EXCLUDED_CLASSES.add(IndexReaderContext.class);
+
+ // usually small but can bump memory usage for
+ // memory-efficient things like stored fields
+ EXCLUDED_CLASSES.add(FieldInfos.class);
+ EXCLUDED_CLASSES.add(SegmentInfo.class);
+ EXCLUDED_CLASSES.add(SegmentCommitInfo.class);
+ EXCLUDED_CLASSES.add(FieldInfo.class);
+
+ // singletons
+ EXCLUDED_CLASSES.add(Codec.class);
+ EXCLUDED_CLASSES.add(Enum.class);
+ for (PackedInts.Format format : PackedInts.Format.values()) {
+ for (int i = 1; i <= 64; ++i) {
+ if (format.isSupported(i)) {
+ EXCLUDED_OBJECTS.add(PackedInts.getDecoder(format, PackedInts.VERSION_CURRENT, i));
+ }
+ }
+ }
+ }
+
+ static class Accumulator extends RamUsageTester.Accumulator {
+
+ private final Object root;
+
+ Accumulator(Object root) {
+ this.root = root;
+ }
+
+ public long accumulateObject(Object o, long shallowSize, java.util.Map<Field, Object> fieldValues, java.util.Collection<Object> queue) {
+ if (EXCLUDED_OBJECTS.contains(o)) {
+ return 0L;
+ }
+ for (Class<?> clazz = o.getClass(); clazz != null; clazz = clazz.getSuperclass()) {
+ if (EXCLUDED_CLASSES.contains(clazz) && o != root) {
+ return 0;
+ }
+ }
+ // we have no way to estimate the size of these things in codecs although
+ // something like a Collections.newSetFromMap(new HashMap<>()) uses quite
+ // some memory... So for now the test ignores the overhead of such
+ // collections but can we do better?
+ if (o instanceof Collection) {
+ Collection<?> coll = (Collection<?>) o;
+ queue.addAll((Collection<?>) o);
+ return (long) coll.size() * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+ } else if (o instanceof Map) {
+ final Map<?, ?> map = (Map<?,?>) o;
+ queue.addAll(map.keySet());
+ queue.addAll(map.values());
+ return 2L * map.size() * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+ }
+ return super.accumulateObject(o, shallowSize, fieldValues, queue);
+ }
+
+ };
+
/** Returns the codec to run tests against */
protected abstract Codec getCodec();
@@ -73,11 +155,11 @@ abstract class BaseIndexFileFormatTestCa
* comparing indices that store the same content.
*/
protected Collection<String> excludedExtensionsFromByteCounts() {
- return new HashSet<String>(Arrays.asList(new String[] {
+ return new HashSet<String>(Arrays.asList(new String[] {
// segment infos store various pieces of information that don't solely depend
// on the content of the index in the diagnostics (such as a timestamp) so we
// exclude this file from the bytes counts
- "si",
+ "si",
// lock files are 0 bytes (one directory in the test could be RAMDir, the other FSDir)
"lock" }));
}
@@ -118,4 +200,38 @@ abstract class BaseIndexFileFormatTestCa
dir2.close();
}
+ /** Test the accuracy of the ramBytesUsed estimations. */
+ public void testRamBytesUsed() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig cfg = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ IndexWriter w = new IndexWriter(dir, cfg);
+ // we need to index enough documents so that constant overhead doesn't dominate
+ final int numDocs = atLeast(10000);
+ for (int i = 0; i < numDocs; ++i) {
+ Document d = new Document();
+ addRandomFields(d);
+ w.addDocument(d);
+ }
+ w.forceMerge(1);
+ w.commit();
+ w.close();
+
+ IndexReader reader = DirectoryReader.open(dir);
+
+ for (AtomicReaderContext context : reader.leaves()) {
+ final AtomicReader r = context.reader();
+ // beware of lazy-loaded stuff
+ new SimpleMergedSegmentWarmer(InfoStream.NO_OUTPUT).warm(r);
+ final long actualBytes = RamUsageTester.sizeOf(r, new Accumulator(r));
+ final long expectedBytes = ((SegmentReader) r).ramBytesUsed();
+ final long absoluteError = actualBytes - expectedBytes;
+ final double relativeError = (double) absoluteError / actualBytes;
+ final String message = "Actual RAM usage " + actualBytes + ", but got " + expectedBytes + ", " + relativeError + "% error";
+ assertTrue(message, Math.abs(relativeError) < 0.20d || Math.abs(absoluteError) < 1000);
+ }
+
+ reader.close();
+ dir.close();
+ }
+
}
Modified: lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java (original)
+++ lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java Wed Jun 18 15:45:58 2014
@@ -238,7 +238,8 @@ public abstract class BaseNormsFormatTes
@Override
protected void addRandomFields(Document doc) {
// TODO: improve
- doc.add(new TextField("foobar", "boo", Field.Store.NO));
+ doc.add(new TextField("foobar", TestUtil.randomSimpleString(random()), Field.Store.NO));
+
}
@Override
Modified: lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java (original)
+++ lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java Wed Jun 18 15:45:58 2014
@@ -20,39 +20,39 @@ package org.apache.lucene.util;
import java.lang.reflect.Array;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
+import java.util.AbstractList;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.IdentityHashMap;
import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
import java.util.NoSuchElementException;
/** Crawls object graph to collect RAM usage for testing */
public final class RamUsageTester {
-
- /**
- * A {@link Filter} that accepts all fields.
- */
- private static final Filter DEFAULT_FILTER = new Filter() {
- @Override
- public boolean accept(Field field) {
- return true;
- }
+ /** An accumulator of object references. This class allows for customizing RAM usage estimation. */
+ public static class Accumulator {
- public boolean accept(Object o) {
- return true;
+ /** Accumulate transitive references for the provided fields of the given
+ * object into <code>queue</code> and return the shallow size of this object. */
+ public long accumulateObject(Object o, long shallowSize, Map<Field, Object> fieldValues, Collection<Object> queue) {
+ for (Object value : fieldValues.values()) {
+ queue.add(value);
+ }
+ return shallowSize;
}
- };
-
- /** A filter that allows to decide on what to take into account when measuring RAM usage. */
- public static interface Filter {
-
- /** Whether the provided field should be taken into account when measuring RAM usage. */
- boolean accept(Field field);
-
- /** Whether the provided field value should be taken into account when measuring RAM usage. */
- boolean accept(Object o);
+ /** Accumulate transitive references for the provided values of the given
+ * array into <code>queue</code> and return the shallow size of this array. */
+ public long accumulateArray(Object array, long shallowSize, List<Object> values, Collection<Object> queue) {
+ queue.addAll(values);
+ return shallowSize;
+ }
}
@@ -65,13 +65,13 @@ public final class RamUsageTester {
* (it isn't side-effect free). After the method exits, this memory
* should be GCed.</p>
*/
- public static long sizeOf(Object obj, Filter filter) {
- return measureObjectSize(obj, filter);
+ public static long sizeOf(Object obj, Accumulator accumulator) {
+ return measureObjectSize(obj, accumulator);
}
/** Same as calling <code>sizeOf(obj, DEFAULT_FILTER)</code>. */
public static long sizeOf(Object obj) {
- return sizeOf(obj, DEFAULT_FILTER);
+ return sizeOf(obj, new Accumulator());
}
/**
@@ -89,7 +89,7 @@ public final class RamUsageTester {
* or complex graphs (a max. recursion depth on my machine was ~5000 objects linked in a chain
* so not too much).
*/
- private static long measureObjectSize(Object root, Filter filter) {
+ private static long measureObjectSize(Object root, Accumulator accumulator) {
// Objects seen so far.
final IdentityHashSet<Object> seen = new IdentityHashSet<>();
// Class cache with reference Field and precalculated shallow size.
@@ -114,25 +114,28 @@ public final class RamUsageTester {
* Consider an array, possibly of primitive types. Push any of its references to
* the processing stack and accumulate this array's shallow size.
*/
- long size = RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
+ final long shallowSize = RamUsageEstimator.shallowSizeOf(ob);
final int len = Array.getLength(ob);
- if (len > 0) {
- Class<?> componentClazz = obClazz.getComponentType();
- if (componentClazz.isPrimitive()) {
- size += (long) len * RamUsageEstimator.shallowSizeOfInstance(componentClazz);
- } else {
- size += (long) RamUsageEstimator.NUM_BYTES_OBJECT_REF * len;
-
- // Push refs for traversal later.
- for (int i = len; --i >= 0 ;) {
- final Object o = Array.get(ob, i);
- if (o != null && !seen.contains(o) && filter.accept(o)) {
- stack.add(o);
+ final List<Object> values;
+ Class<?> componentClazz = obClazz.getComponentType();
+ if (componentClazz.isPrimitive()) {
+ values = Collections.emptyList();
+ } else {
+ values = new AbstractList<Object>() {
+
+ @Override
+ public Object get(int index) {
+ return Array.get(ob, index);
+ }
+
+ @Override
+ public int size() {
+ return len;
}
- }
+
+ };
}
- }
- totalSize += RamUsageEstimator.alignObjectSize(size);
+ totalSize += accumulator.accumulateArray(ob, shallowSize, values, stack);
} else {
/*
* Consider an object. Push any references it has to the processing stack
@@ -144,17 +147,12 @@ public final class RamUsageTester {
classCache.put(obClazz, cachedInfo = createCacheEntry(obClazz));
}
+ Map<Field, Object> fieldValues = new HashMap<>();
for (Field f : cachedInfo.referenceFields) {
- if (filter.accept(f)) {
- // Fast path to eliminate redundancies.
- final Object o = f.get(ob);
- if (o != null && !seen.contains(o) && filter.accept(o)) {
- stack.add(o);
- }
- }
+ fieldValues.put(f, f.get(ob));
}
- totalSize += cachedInfo.alignedShallowInstanceSize;
+ totalSize += accumulator.accumulateObject(ob, cachedInfo.alignedShallowInstanceSize, fieldValues, stack);
} catch (IllegalAccessException e) {
// this should never happen as we enabled setAccessible().
throw new RuntimeException("Reflective field access failed?", e);
Modified: lucene/dev/branches/lucene5752/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/solr/CHANGES.txt?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene5752/solr/CHANGES.txt Wed Jun 18 15:45:58 2014
@@ -86,6 +86,12 @@ Other Changes
* SOLR-6169: Properly remove CoreAdminHandler handleAlias action (Alan Woodward)
+================== 4.10.0 =================
+
+Bug fixes
+
+* SOLR-6095 : SolrCloud cluster can end up without an overseer with overseer roles (Noble Paul, Shalin Mangar)
+
================== 4.9.0 ==================
Versions of Major Components
@@ -99,7 +105,11 @@ Apache ZooKeeper 3.4.6
Upgrading from Solr 4.8
----------------------
-* placeholder
+* Support for DiskDocValuesFormat (ie: fieldTypes configured with docValuesFormat="Disk")
+ has been removed due to poor performance. If you have an existing fieldTypes using
+ DiskDocValuesFormat please modify your schema.xml to remove the 'docValuesFormat'
+ attribute, and optimize your index to rewrite it into the default codec, prior to
+ upgrading to 4.9. See LUCENE-5761 for more details.
Detailed Change List
----------------------
@@ -187,6 +197,15 @@ Bug Fixes
* SOLR-6175: DebugComponent throws NPE on shard exceptions when using shards.tolerant.
(Tomás Fernández Löbbe via shalin)
+* SOLR-6129: DateFormatTransformer doesn't resolve dateTimeFormat. (Aaron LaBella via shalin)
+
+* SOLR-6164: Copy Fields Schema additions are not distributed to other nodes.
+ (Gregory Chanan via Steve Rowe)
+
+* SOLR-6160: An error was sometimes possible if a distributed search included grouping
+ with group.facet, faceting on facet.field and either facet.range or facet.query.
+ (David Smiley)
+
Other Changes
---------------------
Modified: lucene/dev/branches/lucene5752/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatTransformer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatTransformer.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatTransformer.java (original)
+++ lucene/dev/branches/lucene5752/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatTransformer.java Wed Jun 18 15:45:58 2014
@@ -58,6 +58,8 @@ public class DateFormatTransformer exten
String fmt = map.get(DATE_TIME_FMT);
if (fmt == null)
continue;
+ VariableResolver resolver = context.getVariableResolver();
+ fmt = resolver.replaceTokens(fmt);
String column = map.get(DataImporter.COLUMN);
String srcCol = map.get(RegexTransformer.SRC_COL_NAME);
if (srcCol == null)
Modified: lucene/dev/branches/lucene5752/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java (original)
+++ lucene/dev/branches/lucene5752/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java Wed Jun 18 15:45:58 2014
@@ -38,7 +38,7 @@ public class TestDateFormatTransformer e
fields.add(createMap(DataImporter.COLUMN, "lastModified"));
fields.add(createMap(DataImporter.COLUMN,
"dateAdded", RegexTransformer.SRC_COL_NAME, "lastModified",
- DateFormatTransformer.DATE_TIME_FMT, "MM/dd/yyyy"));
+ DateFormatTransformer.DATE_TIME_FMT, "${xyz.myDateFormat}"));
SimpleDateFormat format = new SimpleDateFormat("MM/dd/yyyy", Locale.ROOT);
Date now = format.parse(format.format(new Date()));
@@ -47,6 +47,7 @@ public class TestDateFormatTransformer e
VariableResolver resolver = new VariableResolver();
resolver.addNamespace("e", row);
+ resolver.addNamespace("xyz", createMap("myDateFormat", "MM/dd/yyyy"));
Context context = getContext(null, resolver,
null, Context.FULL_DUMP, fields, null);
Modified: lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java (original)
+++ lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java Wed Jun 18 15:45:58 2014
@@ -53,7 +53,7 @@ public abstract class ElectionContext {
final String leaderPath;
String leaderSeqPath;
private SolrZkClient zkClient;
-
+
public ElectionContext(final String coreNodeName,
final String electionPath, final String leaderPath, final ZkNodeProps leaderProps, final SolrZkClient zkClient) {
this.id = coreNodeName;
@@ -71,7 +71,7 @@ public abstract class ElectionContext {
zkClient.delete(leaderSeqPath, -1, true);
} catch (NoNodeException e) {
// fine
- log.warn("cancelElection did not find election node to remove",e);
+ log.warn("cancelElection did not find election node to remove {}" ,leaderSeqPath);
}
}
@@ -80,6 +80,10 @@ public abstract class ElectionContext {
public void checkIfIamLeaderFired() {}
public void joinedElectionFired() {}
+
+ public ElectionContext copy(){
+ throw new UnsupportedOperationException("copy");
+ }
}
class ShardLeaderElectionContextBase extends ElectionContext {
@@ -529,6 +533,11 @@ final class OverseerElectionContext exte
super.cancelElection();
overseer.close();
}
+
+ @Override
+ public ElectionContext copy() {
+ return new OverseerElectionContext(zkClient, overseer ,id);
+ }
@Override
public void joinedElectionFired() {
Modified: lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java (original)
+++ lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java Wed Jun 18 15:45:58 2014
@@ -57,7 +57,7 @@ public class LeaderElector {
static final String ELECTION_NODE = "/election";
- private final static Pattern LEADER_SEQ = Pattern.compile(".*?/?.*?-n_(\\d+)");
+ public final static Pattern LEADER_SEQ = Pattern.compile(".*?/?.*?-n_(\\d+)");
private final static Pattern SESSION_ID = Pattern.compile(".*?/?(.*?-.*?)-n_\\d+");
private final static Pattern NODE_NAME = Pattern.compile(".*?/?(.*?-)(.*?)-n_\\d+");
@@ -77,7 +77,7 @@ public class LeaderElector {
public ElectionContext getContext() {
return context;
}
-
+
/**
* Check if the candidate with the given n_* sequence number is the leader.
* If it is, set the leaderId on the leader zk node. If it is not, start
@@ -100,14 +100,34 @@ public class LeaderElector {
return;
}
if (seq <= intSeqs.get(0)) {
+ if(seq == intSeqs.get(0) && !context.leaderSeqPath.equals(holdElectionPath+"/"+seqs.get(0)) ) {//somebody else already became the leader with the same sequence id , not me
+ log.info("was going be leader {} , seq(0) {}",context.leaderSeqPath,holdElectionPath+"/"+seqs.get(0));//but someone else jumped the line
+ retryElection(context,false);//join at the tail again
+ return;
+ }
+ // first we delete the node advertising the old leader in case the ephem is still there
// first we delete the node advertising the old leader in case the ephem is still there
try {
zkClient.delete(context.leaderPath, -1, true);
- } catch(Exception e) {
+ }catch (KeeperException.NoNodeException nne){
+ //no problem
+ }catch (InterruptedException e){
+ throw e;
+ } catch (Exception e) {
+ //failed to delete the leader node
+ log.error("leader elect delete error",e);
+ retryElection(context, false);
+ return;
// fine
}
- runIamLeaderProcess(context, replacement);
+ try {
+ runIamLeaderProcess(context, replacement);
+ } catch (KeeperException.NodeExistsException e) {
+ log.error("node exists",e);
+ retryElection(context, false);
+ return;
+ }
} else {
// I am not the leader - watch the node below me
int i = 1;
@@ -124,7 +144,8 @@ public class LeaderElector {
return;
}
try {
- zkClient.getData(holdElectionPath + "/" + seqs.get(index), watcher = new ElectionWatcher(context.leaderSeqPath , seq, context) , null, true);
+ String watchedNode = holdElectionPath + "/" + seqs.get(index);
+ zkClient.getData(watchedNode, watcher = new ElectionWatcher(context.leaderSeqPath , watchedNode,seq, context) , null, true);
} catch (KeeperException.SessionExpiredException e) {
throw e;
} catch (KeeperException e) {
@@ -196,17 +217,20 @@ public class LeaderElector {
}
return intSeqs;
}
-
- /**
- * Begin participating in the election process. Gets a new sequential number
- * and begins watching the node with the sequence number before it, unless it
- * is the lowest number, in which case, initiates the leader process. If the
- * node that is watched goes down, check if we are the new lowest node, else
- * watch the next lowest numbered node.
- *
- * @return sequential node number
- */
public int joinElection(ElectionContext context, boolean replacement) throws KeeperException, InterruptedException, IOException {
+ return joinElection(context,replacement, false);
+ }
+
+ /**
+ * Begin participating in the election process. Gets a new sequential number
+ * and begins watching the node with the sequence number before it, unless it
+ * is the lowest number, in which case, initiates the leader process. If the
+ * node that is watched goes down, check if we are the new lowest node, else
+ * watch the next lowest numbered node.
+ *
+ * @return sequential node number
+ */
+ public int joinElection(ElectionContext context, boolean replacement,boolean joinAtHead) throws KeeperException, InterruptedException, IOException {
context.joinedElectionFired();
final String shardsElectZkPath = context.electionPath + LeaderElector.ELECTION_NODE;
@@ -218,8 +242,30 @@ public class LeaderElector {
int tries = 0;
while (cont) {
try {
- leaderSeqPath = zkClient.create(shardsElectZkPath + "/" + id + "-n_", null,
- CreateMode.EPHEMERAL_SEQUENTIAL, false);
+ if(joinAtHead){
+ log.info("node {} Trying to join election at the head ", id);
+ List<String> nodes = OverseerCollectionProcessor.getSortedElectionNodes(zkClient);
+ if(nodes.size() <2){
+ leaderSeqPath = zkClient.create(shardsElectZkPath + "/" + id + "-n_", null,
+ CreateMode.EPHEMERAL_SEQUENTIAL, false);
+ } else {
+ String firstInLine = nodes.get(1);
+ log.info("The current head: {}", firstInLine);
+ Matcher m = LEADER_SEQ.matcher(firstInLine);
+ if (!m.matches()) {
+ throw new IllegalStateException("Could not find regex match in:"
+ + firstInLine);
+ }
+ leaderSeqPath = shardsElectZkPath + "/" + id + "-n_"+ m.group(1);
+ zkClient.create(leaderSeqPath, null, CreateMode.EPHEMERAL, false);
+ log.info("Joined at the head {}", leaderSeqPath );
+
+ }
+ } else {
+ leaderSeqPath = zkClient.create(shardsElectZkPath + "/" + id + "-n_", null,
+ CreateMode.EPHEMERAL_SEQUENTIAL, false);
+ }
+
context.leaderSeqPath = leaderSeqPath;
cont = false;
} catch (ConnectionLossException e) {
@@ -270,14 +316,15 @@ public class LeaderElector {
}
private class ElectionWatcher implements Watcher {
- final String leaderSeqPath;
+ final String myNode,watchedNode;
final int seq;
final ElectionContext context;
private boolean canceled = false;
- private ElectionWatcher(String leaderSeqPath, int seq, ElectionContext context) {
- this.leaderSeqPath = leaderSeqPath;
+ private ElectionWatcher(String myNode, String watchedNode, int seq, ElectionContext context) {
+ this.myNode = myNode;
+ this.watchedNode = watchedNode;
this.seq = seq;
this.context = context;
}
@@ -295,7 +342,14 @@ public class LeaderElector {
return;
}
if(canceled) {
- log.info("This watcher is not active anymore {}", leaderSeqPath);
+ log.info("This watcher is not active anymore {}", myNode);
+ try {
+ zkClient.delete(myNode,-1,true);
+ }catch (KeeperException.NoNodeException nne) {
+ //expected . don't do anything
+ } catch (Exception e) {
+ log.warn("My watched node still exists and can't remove "+myNode, e);
+ }
return;
}
try {
@@ -332,16 +386,19 @@ public class LeaderElector {
@Override
public int compare(String o1, String o2) {
- return Integer.valueOf(getSeq(o1)).compareTo(
+ int i = Integer.valueOf(getSeq(o1)).compareTo(
Integer.valueOf(getSeq(o2)));
+ return i == 0 ? o1.compareTo(o2) : i ;
}
});
}
-
- void retryElection() throws KeeperException, InterruptedException, IOException {
- context.cancelElection();
+
+ void retryElection(ElectionContext context, boolean joinAtHead) throws KeeperException, InterruptedException, IOException {
ElectionWatcher watcher = this.watcher;
- if(watcher!= null) watcher.cancel(context.leaderSeqPath);
- joinElection(context, true);
+ ElectionContext ctx = context.copy();
+ if(watcher!= null) watcher.cancel(this.context.leaderSeqPath);
+ this.context.cancelElection();
+ this.context = ctx;
+ joinElection(ctx, true, joinAtHead);
}
}
Modified: lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/Overseer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original)
+++ lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/Overseer.java Wed Jun 18 15:45:58 2014
@@ -308,29 +308,31 @@ public class Overseer {
log.error("could not read the data" ,e);
return;
}
- Map m = (Map) ZkStateReader.fromJSON(data);
- String id = (String) m.get("id");
- if(overseerCollectionProcessor.getId().equals(id)){
- try {
- log.info("I'm exiting , but I'm still the leader");
- zkClient.delete(path,stat.getVersion(),true);
- } catch (KeeperException.BadVersionException e) {
- //no problem ignore it some other Overseer has already taken over
- } catch (Exception e) {
- log.error("Could not delete my leader node ", e);
- } finally {
+ try {
+ Map m = (Map) ZkStateReader.fromJSON(data);
+ String id = (String) m.get("id");
+ if(overseerCollectionProcessor.getId().equals(id)){
try {
- if(zkController !=null && !zkController.getCoreContainer().isShutDown()){
- zkController.rejoinOverseerElection();
- }
-
+ log.info("I'm exiting , but I'm still the leader");
+ zkClient.delete(path,stat.getVersion(),true);
+ } catch (KeeperException.BadVersionException e) {
+ //no problem ignore it some other Overseer has already taken over
} catch (Exception e) {
- log.error("error canceling overseer election election ",e);
+ log.error("Could not delete my leader node ", e);
}
- }
- } else{
- log.info("somebody else has already taken up the overseer position");
+ } else{
+ log.info("somebody else has already taken up the overseer position");
+ }
+ } finally {
+ //if I am not shutting down, Then I need to rejoin election
+ try {
+ if (zkController != null && !zkController.getCoreContainer().isShutDown()) {
+ zkController.rejoinOverseerElection(null, false);
+ }
+ } catch (Exception e) {
+ log.warn("Unable to rejoinElection ",e);
+ }
}
}
@@ -377,9 +379,13 @@ public class Overseer {
} else if(CLUSTERPROP.isEqual(operation)){
handleProp(message);
} else if( QUIT.equals(operation)){
- log.info("Quit command received {}", LeaderElector.getNodeName(myId));
- overseerCollectionProcessor.close();
- close();
+ if(myId.equals( message.get("id"))){
+ log.info("Quit command received {}", LeaderElector.getNodeName(myId));
+ overseerCollectionProcessor.close();
+ close();
+ } else {
+ log.warn("Overseer received wrong QUIT message {}", message);
+ }
} else{
throw new RuntimeException("unknown operation:" + operation
+ " contents:" + message.getProperties());
Modified: lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java (original)
+++ lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java Wed Jun 18 15:45:58 2014
@@ -407,95 +407,42 @@ public class OverseerCollectionProcessor
}
}
- private void prioritizeOverseerNodes() throws KeeperException, InterruptedException {
+ private synchronized void prioritizeOverseerNodes() throws KeeperException, InterruptedException {
SolrZkClient zk = zkStateReader.getZkClient();
if(!zk.exists(ZkStateReader.ROLES,true))return;
Map m = (Map) ZkStateReader.fromJSON(zk.getData(ZkStateReader.ROLES, null, new Stat(), true));
List overseerDesignates = (List) m.get("overseer");
if(overseerDesignates==null || overseerDesignates.isEmpty()) return;
- if(overseerDesignates.size() == 1 && overseerDesignates.contains(getLeaderNode(zk))) return;
- log.info("prioritizing overseer nodes at {}", LeaderElector.getNodeName(myId));
- log.info("overseer designates {}", overseerDesignates);
-
- List<String> nodeNames = getSortedOverseerNodeNames(zk);
- if(nodeNames.size()<2) return;
- boolean designateIsInFront = overseerDesignates.contains( nodeNames.get(0));
-
- ArrayList<String> nodesTobePushedBack = new ArrayList<>();
- //ensure that the node right behind the leader , i.e at position 1 is a Overseer
- List<String> availableDesignates = new ArrayList<>();
-
- log.info("sorted nodes {}", nodeNames);//TODO to be removed
- for (int i = 1; i < nodeNames.size(); i++) {
- String s = nodeNames.get(i);
-
- if (overseerDesignates.contains(s)) {
- availableDesignates.add(s);
-
- for(int j=1;j<i;j++){
- String n = nodeNames.get(j);
- if(!overseerDesignates.contains(n)) {
- if(!nodesTobePushedBack.contains(n)) nodesTobePushedBack.add(n);
- }
- }
-
+ String ldr = getLeaderNode(zk);
+ if(overseerDesignates.contains(ldr)) return;
+ log.info("prioritizing overseer nodes at {} overseer designates are {}", myId, overseerDesignates);
+ List<String> electionNodes = getSortedElectionNodes(zk);
+ if(electionNodes.size()<2) return;
+ log.info("sorted nodes {}", electionNodes);
+
+ String designateNodeId = null;
+ for (String electionNode : electionNodes) {
+ if(overseerDesignates.contains( LeaderElector.getNodeName(electionNode))){
+ designateNodeId = electionNode;
+ break;
}
- if(availableDesignates.size()>1) break;//we don't need to line up more than 2 designates
}
- if(!availableDesignates.isEmpty()){
- for (String s : nodesTobePushedBack) {
- log.info("pushing back {} ", s);
- invokeOverseerOp(s, "rejoin");
- }
-
- //wait for a while to ensure the designate has indeed come in front
- boolean prioritizationComplete = false;
- long timeout = System.nanoTime() + TimeUnit.NANOSECONDS.convert(2500, TimeUnit.MILLISECONDS);
-
- while (System.nanoTime() < timeout) {
- List<String> currentNodeNames = getSortedOverseerNodeNames(zk);
-
- int totalLeaders = 0;
-
- for(int i=0;i<availableDesignates.size();i++) {
- if(overseerDesignates.contains(currentNodeNames.get(i))) totalLeaders++;
- }
- if(totalLeaders == availableDesignates.size()){
- prioritizationComplete = true;
- break;
- }
- try {
- Thread.sleep(50);
- } catch (InterruptedException e) {
- log.warn("Thread interrupted",e);
- break;
-
- }
- }
-
- if(!prioritizationComplete) {
- log.warn("available designates and current state {} {} ", availableDesignates, getSortedOverseerNodeNames(zk));
- }
-
- } else if(!designateIsInFront) {
- log.warn("No overseer designates are available, overseerDesignates: {}, live nodes : {}",overseerDesignates,nodeNames);
+ if(designateNodeId == null){
+ log.warn("No live overseer designate ");
return;
}
-
- String leaderNode = getLeaderNode(zkStateReader.getZkClient());
- if(leaderNode ==null) return;
- if(!overseerDesignates.contains(leaderNode) ){
- List<String> sortedNodes = getSortedOverseerNodeNames(zk);
-
- if(leaderNode.equals(sortedNodes.get(0)) || // I am leader and I am in front of the queue
- overseerDesignates.contains(sortedNodes.get(0))) {// I am leader but somebody else is in the front , Screwed up leader election
- //this means there are I am not a designate and the next guy is lined up to become the leader, kill myself
- log.info("I am not an overseer designate , forcing myself out {} ", leaderNode);
- Overseer.getInQueue(zkStateReader.getZkClient()).offer(ZkStateReader.toJSON(new ZkNodeProps(Overseer.QUEUE_OPERATION, Overseer.QUIT)));
- }
+ if(!designateNodeId.equals( electionNodes.get(1))) { //checking if it is already at no:1
+ log.info("asking node {} to come join election at head", designateNodeId);
+ invokeOverseerOp(designateNodeId, "rejoinAtHead"); //ask designate to come first
+ log.info("asking the old first in line {} to rejoin election ",electionNodes.get(1) );
+ invokeOverseerOp(electionNodes.get(1), "rejoin");//ask second inline to go behind
}
+ //now ask the current leader to QUIT , so that the designate can takeover
+ Overseer.getInQueue(zkStateReader.getZkClient()).offer(
+ ZkStateReader.toJSON(new ZkNodeProps(Overseer.QUEUE_OPERATION, Overseer.QUIT,
+ "id",getLeaderId(zkStateReader.getZkClient()))));
}
@@ -513,28 +460,46 @@ public class OverseerCollectionProcessor
return nodeNames;
}
+ public static List<String> getSortedElectionNodes(SolrZkClient zk) throws KeeperException, InterruptedException {
+ List<String> children = null;
+ try {
+ children = zk.getChildren(OverseerElectionContext.PATH + LeaderElector.ELECTION_NODE, null, true);
+ LeaderElector.sortSeqs(children);
+ return children;
+ } catch (Exception e) {
+ throw e;
+ }
+
+ }
+
public static String getLeaderNode(SolrZkClient zkClient) throws KeeperException, InterruptedException {
- byte[] data = new byte[0];
+ String id = getLeaderId(zkClient);
+ return id==null ?
+ null:
+ LeaderElector.getNodeName( id);
+ }
+
+ public static String getLeaderId(SolrZkClient zkClient) throws KeeperException,InterruptedException{
+ byte[] data = null;
try {
data = zkClient.getData("/overseer_elect/leader", null, new Stat(), true);
} catch (KeeperException.NoNodeException e) {
return null;
}
Map m = (Map) ZkStateReader.fromJSON(data);
- String s = (String) m.get("id");
- String nodeName = LeaderElector.getNodeName(s);
- return nodeName;
+ return (String) m.get("id");
}
- private void invokeOverseerOp(String nodeName, String op) {
+ private void invokeOverseerOp(String electionNode, String op) {
ModifiableSolrParams params = new ModifiableSolrParams();
ShardHandler shardHandler = shardHandlerFactory.getShardHandler();
params.set(CoreAdminParams.ACTION, CoreAdminAction.OVERSEEROP.toString());
params.set("op", op);
params.set("qt", adminPath);
+ params.set("electionNode", electionNode);
ShardRequest sreq = new ShardRequest();
sreq.purpose = 1;
- String replica = zkStateReader.getBaseUrlForNodeName(nodeName);
+ String replica = zkStateReader.getBaseUrlForNodeName(LeaderElector.getNodeName(electionNode));
sreq.shards = new String[]{replica};
sreq.actualShards = sreq.shards;
sreq.params = params;
Modified: lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/ZkController.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/ZkController.java (original)
+++ lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/ZkController.java Wed Jun 18 15:45:58 2014
@@ -1691,9 +1691,33 @@ public final class ZkController {
return out;
}
- public void rejoinOverseerElection() {
+ public void rejoinOverseerElection(String electionNode, boolean joinAtHead) {
try {
- overseerElector.retryElection();
+ if(electionNode !=null){
+ //this call is from inside the JVM . not from CoreAdminHandler
+ if(overseerElector.getContext() == null || overseerElector.getContext().leaderSeqPath == null){
+ overseerElector.retryElection(new OverseerElectionContext(zkClient,
+ overseer, getNodeName()), joinAtHead);
+ return;
+ }
+ if(!overseerElector.getContext().leaderSeqPath.endsWith(electionNode)){
+ log.warn("Asked to rejoin with wrong election node : {}, current node is {}",electionNode, overseerElector.getContext().leaderSeqPath);
+ //however delete it . This is possible when the last attempt at deleting the election node failed.
+ if(electionNode.startsWith(getNodeName())){
+ try {
+ zkClient.delete(OverseerElectionContext.PATH+LeaderElector.ELECTION_NODE+"/"+electionNode,-1,true);
+ } catch (NoNodeException e) {
+ //no problem
+ } catch (InterruptedException e){
+ Thread.currentThread().interrupt();
+ } catch(Exception e) {
+ log.warn("Old election node exists , could not be removed ",e);
+ }
+ }
+ }
+ }else {
+ overseerElector.retryElection(overseerElector.getContext(), joinAtHead);
+ }
} catch (Exception e) {
throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to rejoin election", e);
}
Modified: lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java (original)
+++ lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java Wed Jun 18 15:45:58 2014
@@ -272,7 +272,12 @@ public class CoreAdminHandler extends Re
ZkController zkController = coreContainer.getZkController();
if(zkController != null){
String op = req.getParams().get("op");
- if ("rejoin".equals(op)) zkController.rejoinOverseerElection();
+ String electionNode = req.getParams().get("electionNode");
+ if(electionNode != null) {
+ zkController.rejoinOverseerElection(electionNode, "rejoinAtHead".equals(op));
+ } else {
+ log.info("electionNode is required param");
+ }
}
break;
}
Modified: lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/request/SimpleFacets.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/request/SimpleFacets.java (original)
+++ lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/request/SimpleFacets.java Wed Jun 18 15:45:58 2014
@@ -17,33 +17,11 @@
package org.apache.solr.request;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
-import java.util.EnumSet;
-import java.util.IdentityHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executor;
-import java.util.concurrent.Future;
-import java.util.concurrent.FutureTask;
-import java.util.concurrent.RunnableFuture;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.MultiDocsEnum;
-import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
@@ -61,7 +39,6 @@ import org.apache.lucene.search.grouping
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.StringHelper;
-import org.apache.lucene.util.UnicodeUtil;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.CommonParams;
@@ -82,7 +59,6 @@ import org.apache.solr.schema.SchemaFiel
import org.apache.solr.schema.TrieDateField;
import org.apache.solr.schema.TrieField;
import org.apache.solr.search.BitDocSet;
-import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocSet;
import org.apache.solr.search.Grouping;
import org.apache.solr.search.HashDocSet;
@@ -96,7 +72,26 @@ import org.apache.solr.search.grouping.G
import org.apache.solr.util.BoundedTreeSet;
import org.apache.solr.util.DateMathParser;
import org.apache.solr.util.DefaultSolrThreadFactory;
-import org.apache.solr.util.LongPriorityQueue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.EnumSet;
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Future;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.RunnableFuture;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
/**
* A class that generates simple Facet information for a request.
@@ -315,8 +310,7 @@ public class SimpleFacets {
* @see FacetParams#FACET_QUERY
*/
public int getGroupedFacetQueryCount(Query facetQuery) throws IOException {
- GroupingSpecification groupingSpecification = rb.getGroupingSpec();
- String groupField = groupingSpecification != null ? groupingSpecification.getFields()[0] : null;
+ String groupField = params.get(GroupParams.GROUP_FIELD);
if (groupField == null) {
throw new SolrException (
SolrException.ErrorCode.BAD_REQUEST,
Modified: lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/schema/IndexSchema.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/schema/IndexSchema.java (original)
+++ lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/schema/IndexSchema.java Wed Jun 18 15:45:58 2014
@@ -591,44 +591,7 @@ public class IndexSchema {
// expression = "/schema/copyField";
dynamicCopyFields = new DynamicCopy[] {};
- expression = "//" + COPY_FIELD;
- nodes = (NodeList) xpath.evaluate(expression, document, XPathConstants.NODESET);
-
- for (int i=0; i<nodes.getLength(); i++) {
- node = nodes.item(i);
- NamedNodeMap attrs = node.getAttributes();
-
- String source = DOMUtil.getAttr(attrs, SOURCE, COPY_FIELD + " definition");
- String dest = DOMUtil.getAttr(attrs, DESTINATION, COPY_FIELD + " definition");
- String maxChars = DOMUtil.getAttr(attrs, MAX_CHARS);
- int maxCharsInt = CopyField.UNLIMITED;
- if (maxChars != null) {
- try {
- maxCharsInt = Integer.parseInt(maxChars);
- } catch (NumberFormatException e) {
- log.warn("Couldn't parse " + MAX_CHARS + " attribute for " + COPY_FIELD + " from "
- + source + " to " + dest + " as integer. The whole field will be copied.");
- }
- }
-
- if (dest.equals(uniqueKeyFieldName)) {
- String msg = UNIQUE_KEY + " field ("+uniqueKeyFieldName+
- ") can not be the " + DESTINATION + " of a " + COPY_FIELD + "(" + SOURCE + "=" +source+")";
- log.error(msg);
- throw new SolrException(ErrorCode.SERVER_ERROR, msg);
-
- }
-
- registerCopyField(source, dest, maxCharsInt);
- }
-
- for (Map.Entry<SchemaField, Integer> entry : copyFieldTargetCounts.entrySet()) {
- if (entry.getValue() > 1 && !entry.getKey().multiValued()) {
- log.warn("Field " + entry.getKey().name + " is not multivalued "+
- "and destination for multiple " + COPY_FIELDS + " ("+
- entry.getValue()+")");
- }
- }
+ loadCopyFields(document, xpath);
//Run the callbacks on SchemaAware now that everything else is done
for (SchemaAware aware : schemaAware) {
@@ -747,6 +710,50 @@ public class IndexSchema {
}
/**
+ * Loads the copy fields
+ */
+ protected synchronized void loadCopyFields(Document document, XPath xpath) throws XPathExpressionException {
+ String expression = "//" + COPY_FIELD;
+ NodeList nodes = (NodeList)xpath.evaluate(expression, document, XPathConstants.NODESET);
+
+ for (int i=0; i<nodes.getLength(); i++) {
+ Node node = nodes.item(i);
+ NamedNodeMap attrs = node.getAttributes();
+
+ String source = DOMUtil.getAttr(attrs, SOURCE, COPY_FIELD + " definition");
+ String dest = DOMUtil.getAttr(attrs, DESTINATION, COPY_FIELD + " definition");
+ String maxChars = DOMUtil.getAttr(attrs, MAX_CHARS);
+
+ int maxCharsInt = CopyField.UNLIMITED;
+ if (maxChars != null) {
+ try {
+ maxCharsInt = Integer.parseInt(maxChars);
+ } catch (NumberFormatException e) {
+ log.warn("Couldn't parse " + MAX_CHARS + " attribute for " + COPY_FIELD + " from "
+ + source + " to " + dest + " as integer. The whole field will be copied.");
+ }
+ }
+
+ if (dest.equals(uniqueKeyFieldName)) {
+ String msg = UNIQUE_KEY + " field ("+uniqueKeyFieldName+
+ ") can not be the " + DESTINATION + " of a " + COPY_FIELD + "(" + SOURCE + "=" +source+")";
+ log.error(msg);
+ throw new SolrException(ErrorCode.SERVER_ERROR, msg);
+ }
+
+ registerCopyField(source, dest, maxCharsInt);
+ }
+
+ for (Map.Entry<SchemaField, Integer> entry : copyFieldTargetCounts.entrySet()) {
+ if (entry.getValue() > 1 && !entry.getKey().multiValued()) {
+ log.warn("Field " + entry.getKey().name + " is not multivalued "+
+ "and destination for multiple " + COPY_FIELDS + " ("+
+ entry.getValue()+")");
+ }
+ }
+ }
+
+ /**
* Converts a sequence of path steps into a rooted path, by inserting slashes in front of each step.
* @param steps The steps to join with slashes to form a path
* @return a rooted path: a leading slash followed by the given steps joined with slashes
Modified: lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java (original)
+++ lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java Wed Jun 18 15:45:58 2014
@@ -43,6 +43,7 @@ import java.nio.charset.StandardCharsets
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.Map;
/** Solr-managed schema - non-user-editable, but can be mutable via internal and external REST API requests. */
@@ -338,6 +339,13 @@ public final class ManagedIndexSchema ex
Document document = schemaConf.getDocument();
final XPath xpath = schemaConf.getXPath();
newSchema.loadFields(document, xpath);
+ // let's completely rebuild the copy fields from the schema in ZK.
+ // create new copyField-related objects so we don't affect the
+ // old schema
+ newSchema.copyFieldsMap = new HashMap<>();
+ newSchema.dynamicCopyFields = null;
+ newSchema.copyFieldTargetCounts = new HashMap<>();
+ newSchema.loadCopyFields(document, xpath);
if (null != uniqueKeyField) {
newSchema.requiredFields.add(uniqueKeyField);
}
Modified: lucene/dev/branches/lucene5752/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java (original)
+++ lucene/dev/branches/lucene5752/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java Wed Jun 18 15:45:58 2014
@@ -17,6 +17,7 @@ package org.apache.solr.cloud;
* limitations under the License.
*/
+
import static org.apache.solr.cloud.OverseerCollectionProcessor.MAX_SHARDS_PER_NODE;
import static org.apache.solr.cloud.OverseerCollectionProcessor.NUM_SLICES;
import static org.apache.solr.cloud.OverseerCollectionProcessor.REPLICATION_FACTOR;
@@ -54,7 +55,7 @@ import org.junit.BeforeClass;
@SuppressSSL // Currently unknown why SSL does not work
public class OverseerRolesTest extends AbstractFullDistribZkTestBase{
private CloudSolrServer client;
-
+
@BeforeClass
public static void beforeThisClass2() throws Exception {
@@ -173,80 +174,45 @@ public class OverseerRolesTest extends
log.info("Adding another overseer designate {}", anotherOverseer);
setOverseerRole(CollectionAction.ADDROLE, anotherOverseer);
- timeout = System.currentTimeMillis()+10000;
- leaderchanged = false;
- for(;System.currentTimeMillis() < timeout;){
- List<String> sortedNodeNames = getSortedOverseerNodeNames(client.getZkStateReader().getZkClient());
- if(sortedNodeNames.get(1) .equals(anotherOverseer) || sortedNodeNames.get(0).equals(anotherOverseer)){
- leaderchanged =true;
- break;
- }
- Thread.sleep(100);
- }
-
- assertTrue("New overseer not the frontrunner : "+ getSortedOverseerNodeNames(client.getZkStateReader().getZkClient()) + " expected : "+ anotherOverseer, leaderchanged);
-
-
String currentOverseer = getLeaderNode(client.getZkStateReader().getZkClient());
- String killedOverseer = currentOverseer;
-
log.info("Current Overseer {}", currentOverseer);
- Pattern pattern = Pattern.compile("(.*):(\\d*)(.*)");
- Matcher m = pattern.matcher(currentOverseer);
- JettySolrRunner stoppedJetty =null;
-
- String hostPort = null;
- StringBuilder sb = new StringBuilder();
- if(m.matches()){
- hostPort = m.group(1)+":"+m.group(2);
- log.info("hostPort : {}", hostPort);
-
- for (JettySolrRunner jetty : jettys) {
- String s = jetty.getBaseUrl().toString();
- sb.append(s).append(" , ");
- if(s.contains(hostPort)){
- log.info("leader node {}",s);
- ChaosMonkey.stop(jetty);
- stoppedJetty = jetty;
- timeout = System.currentTimeMillis()+10000;
- leaderchanged = false;
- for(;System.currentTimeMillis() < timeout;){
- currentOverseer = getLeaderNode(client.getZkStateReader().getZkClient());
- if(anotherOverseer.equals(currentOverseer)){
- leaderchanged =true;
- break;
- }
- Thread.sleep(100);
- }
- assertTrue("New overseer designate has not become the overseer, expected : "+ anotherOverseer + "actual : "+ currentOverseer, leaderchanged);
- }
+ String hostPort = currentOverseer.substring(0,currentOverseer.indexOf('_'));
+ StringBuilder sb = new StringBuilder();
+//
+//
+ log.info("hostPort : {}", hostPort);
+
+ JettySolrRunner leaderJetty = null;
+
+ for (JettySolrRunner jetty : jettys) {
+ String s = jetty.getBaseUrl().toString();
+ log.info("jetTy {}",s);
+ sb.append(s).append(" , ");
+ if (s.contains(hostPort)) {
+ leaderJetty = jetty;
+ break;
}
-
- } else{
- fail("pattern didn't match for"+currentOverseer );
}
- if(stoppedJetty !=null) {
- ChaosMonkey.start(stoppedJetty);
+ assertNotNull("Could not find a jetty2 kill", leaderJetty);
- timeout = System.currentTimeMillis() + 10000;
- leaderchanged = false;
- for (; System.currentTimeMillis() < timeout; ) {
- List<String> sortedNodeNames = getSortedOverseerNodeNames(client.getZkStateReader().getZkClient());
- if (sortedNodeNames.get(1).equals(killedOverseer) || sortedNodeNames.get(0).equals(killedOverseer)) {
- leaderchanged = true;
- break;
- }
- Thread.sleep(100);
+ log.info("leader node {}", leaderJetty.getBaseUrl());
+ log.info ("current election Queue", OverseerCollectionProcessor.getSortedElectionNodes(client.getZkStateReader().getZkClient()));
+ ChaosMonkey.stop(leaderJetty);
+ timeout = System.currentTimeMillis() + 10000;
+ leaderchanged = false;
+ for (; System.currentTimeMillis() < timeout; ) {
+ currentOverseer = getLeaderNode(client.getZkStateReader().getZkClient());
+ if (anotherOverseer.equals(currentOverseer)) {
+ leaderchanged = true;
+ break;
}
-
- assertTrue("New overseer not the frontrunner : " + getSortedOverseerNodeNames(client.getZkStateReader().getZkClient()) + " expected : " + killedOverseer, leaderchanged);
- } else {
- log.warn("The jetty where the overseer {} is running could not be located in {}",hostPort,sb);
+ Thread.sleep(100);
}
+ assertTrue("New overseer designate has not become the overseer, expected : " + anotherOverseer + "actual : " + getLeaderNode(client.getZkStateReader().getZkClient()), leaderchanged);
}
private void setOverseerRole(CollectionAction action, String overseerDesignate) throws Exception, IOException {
@@ -261,6 +227,7 @@ public class OverseerRolesTest extends
client.request(request);
}
+
protected void createCollection(String COLL_NAME, CloudSolrServer client) throws Exception {
int replicationFactor = 2;
int numShards = 4;
Modified: lucene/dev/branches/lucene5752/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java (original)
+++ lucene/dev/branches/lucene5752/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java Wed Jun 18 15:45:58 2014
@@ -396,7 +396,7 @@ public class DistributedDebugComponentTe
assertEquals("Unexpected response size for shard", 1, badShardTrack.size());
Entry<String, String> exception = badShardTrack.iterator().next();
assertEquals("Expected key 'Exception' not found", "Exception", exception.getKey());
- assertTrue("Unexpected exception message", exception.getValue().contains("Server refused connection"));
+ assertNotNull("Exception message should not be null", exception.getValue());
unIgnoreException("Server refused connection");
}