You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2014/06/19 18:25:36 UTC
svn commit: r1603938 [4/5] - in /lucene/dev/branches/solr-5473: ./
dev-tools/ lucene/ lucene/analysis/ lucene/analysis/common/
lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/
lucene/analysis/common/src/test/org/apache/lucene/analys...
Modified: lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenFilter.java?rev=1603938&r1=1603937&r2=1603938&view=diff
==============================================================================
--- lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenFilter.java (original)
+++ lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenFilter.java Thu Jun 19 16:25:31 2014
@@ -17,15 +17,15 @@ package org.apache.lucene.analysis;
* limitations under the License.
*/
-import static org.apache.lucene.util.automaton.BasicAutomata.makeEmpty;
-import static org.apache.lucene.util.automaton.BasicAutomata.makeString;
+import static org.apache.lucene.util.automaton.Automata.makeEmpty;
+import static org.apache.lucene.util.automaton.Automata.makeString;
import java.io.IOException;
import java.util.Arrays;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.util.automaton.BasicOperations;
+import org.apache.lucene.util.automaton.Operations;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
/**
@@ -43,7 +43,7 @@ public final class MockTokenFilter exten
/** Set of common english stopwords */
public static final CharacterRunAutomaton ENGLISH_STOPSET =
- new CharacterRunAutomaton(BasicOperations.union(Arrays.asList(
+ new CharacterRunAutomaton(Operations.union(Arrays.asList(
makeString("a"), makeString("an"), makeString("and"), makeString("are"),
makeString("as"), makeString("at"), makeString("be"), makeString("but"),
makeString("by"), makeString("for"), makeString("if"), makeString("in"),
Modified: lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java?rev=1603938&r1=1603937&r2=1603938&view=diff
==============================================================================
--- lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java (original)
+++ lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java Thu Jun 19 16:25:31 2014
@@ -18,24 +18,105 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+import java.lang.reflect.Field;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.IdentityHashMap;
+import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.mockrandom.MockRandomPostingsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.CloseableThreadLocal;
+import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.RamUsageTester;
/**
* Common tests to all index formats.
*/
abstract class BaseIndexFileFormatTestCase extends LuceneTestCase {
+ // metadata or Directory-level objects
+ private static final Set<Class<?>> EXCLUDED_CLASSES = Collections.newSetFromMap(new IdentityHashMap<Class<?>,Boolean>());
+
+ static {
+ // Directory objects, don't take into account eg. the NIO buffers
+ EXCLUDED_CLASSES.add(Directory.class);
+ EXCLUDED_CLASSES.add(IndexInput.class);
+
+ // used for thread management, not by the index
+ EXCLUDED_CLASSES.add(CloseableThreadLocal.class);
+ EXCLUDED_CLASSES.add(ThreadLocal.class);
+
+ // don't follow references to the top-level reader
+ EXCLUDED_CLASSES.add(IndexReader.class);
+ EXCLUDED_CLASSES.add(IndexReaderContext.class);
+
+ // usually small but can bump memory usage for
+ // memory-efficient things like stored fields
+ EXCLUDED_CLASSES.add(FieldInfos.class);
+ EXCLUDED_CLASSES.add(SegmentInfo.class);
+ EXCLUDED_CLASSES.add(SegmentCommitInfo.class);
+ EXCLUDED_CLASSES.add(FieldInfo.class);
+
+ // constant overhead is typically due to strings
+ // TODO: can we remove this and still pass the test consistently
+ EXCLUDED_CLASSES.add(String.class);
+ }
+
+ static class Accumulator extends RamUsageTester.Accumulator {
+
+ private final Object root;
+
+ Accumulator(Object root) {
+ this.root = root;
+ }
+
+ public long accumulateObject(Object o, long shallowSize, java.util.Map<Field, Object> fieldValues, java.util.Collection<Object> queue) {
+ for (Class<?> clazz = o.getClass(); clazz != null; clazz = clazz.getSuperclass()) {
+ if (EXCLUDED_CLASSES.contains(clazz) && o != root) {
+ return 0;
+ }
+ }
+ // we have no way to estimate the size of these things in codecs although
+ // something like a Collections.newSetFromMap(new HashMap<>()) uses quite
+ // some memory... So for now the test ignores the overhead of such
+ // collections but can we do better?
+ if (o instanceof Collection) {
+ Collection<?> coll = (Collection<?>) o;
+ queue.addAll((Collection<?>) o);
+ return (long) coll.size() * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+ } else if (o instanceof Map) {
+ final Map<?, ?> map = (Map<?,?>) o;
+ queue.addAll(map.keySet());
+ queue.addAll(map.values());
+ return 2L * map.size() * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+ }
+ long v = super.accumulateObject(o, shallowSize, fieldValues, queue);
+ // System.out.println(o.getClass() + "=" + v);
+ return v;
+ }
+
+ @Override
+ public long accumulateArray(Object array, long shallowSize,
+ List<Object> values, Collection<Object> queue) {
+ long v = super.accumulateArray(array, shallowSize, values, queue);
+ // System.out.println(array.getClass() + "=" + v);
+ return v;
+ }
+
+ };
+
/** Returns the codec to run tests against */
protected abstract Codec getCodec();
@@ -73,11 +154,11 @@ abstract class BaseIndexFileFormatTestCa
* comparing indices that store the same content.
*/
protected Collection<String> excludedExtensionsFromByteCounts() {
- return new HashSet<String>(Arrays.asList(new String[] {
+ return new HashSet<String>(Arrays.asList(new String[] {
// segment infos store various pieces of information that don't solely depend
// on the content of the index in the diagnostics (such as a timestamp) so we
// exclude this file from the bytes counts
- "si",
+ "si",
// lock files are 0 bytes (one directory in the test could be RAMDir, the other FSDir)
"lock" }));
}
@@ -118,4 +199,51 @@ abstract class BaseIndexFileFormatTestCa
dir2.close();
}
+ /** Test the accuracy of the ramBytesUsed estimations. */
+ public void testRamBytesUsed() throws IOException {
+ if (Codec.getDefault() instanceof RandomCodec) {
+ // this test relies on the fact that two segments will be written with
+ // the same codec so we need to disable MockRandomPF
+ final Set<String> avoidCodecs = new HashSet<>(((RandomCodec) Codec.getDefault()).avoidCodecs);
+ avoidCodecs.add(new MockRandomPostingsFormat().getName());
+ Codec.setDefault(new RandomCodec(random(), avoidCodecs));
+ }
+ Directory dir = newDirectory();
+ IndexWriterConfig cfg = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ IndexWriter w = new IndexWriter(dir, cfg);
+ // we need to index enough documents so that constant overhead doesn't dominate
+ final int numDocs = atLeast(10000);
+ AtomicReader reader1 = null;
+ for (int i = 0; i < numDocs; ++i) {
+ Document d = new Document();
+ addRandomFields(d);
+ w.addDocument(d);
+ if (i == 100) {
+ w.forceMerge(1);
+ w.commit();
+ reader1 = getOnlySegmentReader(DirectoryReader.open(dir));
+ }
+ }
+ w.forceMerge(1);
+ w.commit();
+ w.close();
+
+ AtomicReader reader2 = getOnlySegmentReader(DirectoryReader.open(dir));
+
+ for (AtomicReader reader : Arrays.asList(reader1, reader2)) {
+ new SimpleMergedSegmentWarmer(InfoStream.NO_OUTPUT).warm(reader);
+ }
+
+ final long actualBytes = RamUsageTester.sizeOf(reader2, new Accumulator(reader2)) - RamUsageTester.sizeOf(reader1, new Accumulator(reader1));
+ final long expectedBytes = ((SegmentReader) reader2).ramBytesUsed() - ((SegmentReader) reader1).ramBytesUsed();
+ final long absoluteError = actualBytes - expectedBytes;
+ final double relativeError = (double) absoluteError / actualBytes;
+ final String message = "Actual RAM usage " + actualBytes + ", but got " + expectedBytes + ", " + 100*relativeError + "% error";
+ assertTrue(message, Math.abs(relativeError) < 0.20d || Math.abs(absoluteError) < 1000);
+
+ reader1.close();
+ reader2.close();
+ dir.close();
+ }
+
}
Modified: lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java?rev=1603938&r1=1603937&r2=1603938&view=diff
==============================================================================
--- lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java (original)
+++ lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java Thu Jun 19 16:25:31 2014
@@ -238,7 +238,8 @@ public abstract class BaseNormsFormatTes
@Override
protected void addRandomFields(Document doc) {
// TODO: improve
- doc.add(new TextField("foobar", "boo", Field.Store.NO));
+ doc.add(new TextField("foobar", TestUtil.randomSimpleString(random()), Field.Store.NO));
+
}
@Override
Modified: lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java?rev=1603938&r1=1603937&r2=1603938&view=diff
==============================================================================
--- lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java (original)
+++ lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java Thu Jun 19 16:25:31 2014
@@ -75,6 +75,8 @@ public class RandomCodec extends Lucene4
/** unique set of docvalues format names this codec knows about */
public Set<String> dvFormatNames = new HashSet<>();
+ public final Set<String> avoidCodecs;
+
/** memorized field->postingsformat mappings */
// note: we have to sync this map even though its just for debugging/toString,
// otherwise DWPT's .toString() calls that iterate over the map can
@@ -117,6 +119,7 @@ public class RandomCodec extends Lucene4
public RandomCodec(Random random, Set<String> avoidCodecs) {
this.perFieldSeed = random.nextInt();
+ this.avoidCodecs = avoidCodecs;
// TODO: make it possible to specify min/max iterms per
// block via CL:
int minItemsPerBlock = TestUtil.nextInt(random, 2, 100);
Modified: lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/search/SearchEquivalenceTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/search/SearchEquivalenceTestBase.java?rev=1603938&r1=1603937&r2=1603938&view=diff
==============================================================================
--- lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/search/SearchEquivalenceTestBase.java (original)
+++ lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/search/SearchEquivalenceTestBase.java Thu Jun 19 16:25:31 2014
@@ -33,7 +33,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
-import org.apache.lucene.util.automaton.BasicAutomata;
+import org.apache.lucene.util.automaton.Automata;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -57,7 +57,7 @@ public abstract class SearchEquivalenceT
Random random = random();
directory = newDirectory();
stopword = "" + randomChar();
- CharacterRunAutomaton stopset = new CharacterRunAutomaton(BasicAutomata.makeString(stopword));
+ CharacterRunAutomaton stopset = new CharacterRunAutomaton(Automata.makeString(stopword));
analyzer = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false, stopset);
RandomIndexWriter iw = new RandomIndexWriter(random, directory, analyzer);
Document doc = new Document();
Modified: lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java?rev=1603938&r1=1603937&r2=1603938&view=diff
==============================================================================
--- lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java (original)
+++ lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java Thu Jun 19 16:25:31 2014
@@ -20,39 +20,39 @@ package org.apache.lucene.util;
import java.lang.reflect.Array;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
+import java.util.AbstractList;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.IdentityHashMap;
import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
import java.util.NoSuchElementException;
/** Crawls object graph to collect RAM usage for testing */
public final class RamUsageTester {
-
- /**
- * A {@link Filter} that accepts all fields.
- */
- private static final Filter DEFAULT_FILTER = new Filter() {
- @Override
- public boolean accept(Field field) {
- return true;
- }
+ /** An accumulator of object references. This class allows for customizing RAM usage estimation. */
+ public static class Accumulator {
- public boolean accept(Object o) {
- return true;
+ /** Accumulate transitive references for the provided fields of the given
+ * object into <code>queue</code> and return the shallow size of this object. */
+ public long accumulateObject(Object o, long shallowSize, Map<Field, Object> fieldValues, Collection<Object> queue) {
+ for (Object value : fieldValues.values()) {
+ queue.add(value);
+ }
+ return shallowSize;
}
- };
-
- /** A filter that allows to decide on what to take into account when measuring RAM usage. */
- public static interface Filter {
-
- /** Whether the provided field should be taken into account when measuring RAM usage. */
- boolean accept(Field field);
-
- /** Whether the provided field value should be taken into account when measuring RAM usage. */
- boolean accept(Object o);
+ /** Accumulate transitive references for the provided values of the given
+ * array into <code>queue</code> and return the shallow size of this array. */
+ public long accumulateArray(Object array, long shallowSize, List<Object> values, Collection<Object> queue) {
+ queue.addAll(values);
+ return shallowSize;
+ }
}
@@ -65,13 +65,13 @@ public final class RamUsageTester {
* (it isn't side-effect free). After the method exits, this memory
* should be GCed.</p>
*/
- public static long sizeOf(Object obj, Filter filter) {
- return measureObjectSize(obj, filter);
+ public static long sizeOf(Object obj, Accumulator accumulator) {
+ return measureObjectSize(obj, accumulator);
}
/** Same as calling <code>sizeOf(obj, DEFAULT_FILTER)</code>. */
public static long sizeOf(Object obj) {
- return sizeOf(obj, DEFAULT_FILTER);
+ return sizeOf(obj, new Accumulator());
}
/**
@@ -89,7 +89,7 @@ public final class RamUsageTester {
* or complex graphs (a max. recursion depth on my machine was ~5000 objects linked in a chain
* so not too much).
*/
- private static long measureObjectSize(Object root, Filter filter) {
+ private static long measureObjectSize(Object root, Accumulator accumulator) {
// Objects seen so far.
final IdentityHashSet<Object> seen = new IdentityHashSet<>();
// Class cache with reference Field and precalculated shallow size.
@@ -114,25 +114,28 @@ public final class RamUsageTester {
* Consider an array, possibly of primitive types. Push any of its references to
* the processing stack and accumulate this array's shallow size.
*/
- long size = RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
+ final long shallowSize = RamUsageEstimator.shallowSizeOf(ob);
final int len = Array.getLength(ob);
- if (len > 0) {
- Class<?> componentClazz = obClazz.getComponentType();
- if (componentClazz.isPrimitive()) {
- size += (long) len * RamUsageEstimator.shallowSizeOfInstance(componentClazz);
- } else {
- size += (long) RamUsageEstimator.NUM_BYTES_OBJECT_REF * len;
-
- // Push refs for traversal later.
- for (int i = len; --i >= 0 ;) {
- final Object o = Array.get(ob, i);
- if (o != null && !seen.contains(o) && filter.accept(o)) {
- stack.add(o);
+ final List<Object> values;
+ Class<?> componentClazz = obClazz.getComponentType();
+ if (componentClazz.isPrimitive()) {
+ values = Collections.emptyList();
+ } else {
+ values = new AbstractList<Object>() {
+
+ @Override
+ public Object get(int index) {
+ return Array.get(ob, index);
+ }
+
+ @Override
+ public int size() {
+ return len;
}
- }
+
+ };
}
- }
- totalSize += RamUsageEstimator.alignObjectSize(size);
+ totalSize += accumulator.accumulateArray(ob, shallowSize, values, stack);
} else {
/*
* Consider an object. Push any references it has to the processing stack
@@ -144,17 +147,12 @@ public final class RamUsageTester {
classCache.put(obClazz, cachedInfo = createCacheEntry(obClazz));
}
+ Map<Field, Object> fieldValues = new HashMap<>();
for (Field f : cachedInfo.referenceFields) {
- if (filter.accept(f)) {
- // Fast path to eliminate redundancies.
- final Object o = f.get(ob);
- if (o != null && !seen.contains(o) && filter.accept(o)) {
- stack.add(o);
- }
- }
+ fieldValues.put(f, f.get(ob));
}
- totalSize += cachedInfo.alignedShallowInstanceSize;
+ totalSize += accumulator.accumulateObject(ob, cachedInfo.alignedShallowInstanceSize, fieldValues, stack);
} catch (IllegalAccessException e) {
// this should never happen as we enabled setAccessible().
throw new RuntimeException("Reflective field access failed?", e);
Modified: lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/util/automaton/AutomatonTestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/util/automaton/AutomatonTestUtil.java?rev=1603938&r1=1603937&r2=1603938&view=diff
==============================================================================
--- lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/util/automaton/AutomatonTestUtil.java (original)
+++ lucene/dev/branches/solr-5473/lucene/test-framework/src/java/org/apache/lucene/util/automaton/AutomatonTestUtil.java Thu Jun 19 16:25:31 2014
@@ -20,7 +20,6 @@ package org.apache.lucene.util.automaton
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.IdentityHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -31,7 +30,6 @@ import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.UnicodeUtil;
-import org.apache.lucene.util.fst.Util;
/**
* Utilities for testing automata.
@@ -92,40 +90,40 @@ public class AutomatonTestUtil {
/** picks a random int code point, avoiding surrogates;
* throws IllegalArgumentException if this transition only
* accepts surrogates */
- private static int getRandomCodePoint(final Random r, final Transition t) {
+ private static int getRandomCodePoint(final Random r, int min, int max) {
final int code;
- if (t.max < UnicodeUtil.UNI_SUR_HIGH_START ||
- t.min > UnicodeUtil.UNI_SUR_HIGH_END) {
+ if (max < UnicodeUtil.UNI_SUR_HIGH_START ||
+ min > UnicodeUtil.UNI_SUR_HIGH_END) {
// easy: entire range is before or after surrogates
- code = t.min+r.nextInt(t.max-t.min+1);
- } else if (t.min >= UnicodeUtil.UNI_SUR_HIGH_START) {
- if (t.max > UnicodeUtil.UNI_SUR_LOW_END) {
+ code = min+r.nextInt(max-min+1);
+ } else if (min >= UnicodeUtil.UNI_SUR_HIGH_START) {
+ if (max > UnicodeUtil.UNI_SUR_LOW_END) {
// after surrogates
- code = 1+UnicodeUtil.UNI_SUR_LOW_END+r.nextInt(t.max-UnicodeUtil.UNI_SUR_LOW_END);
+ code = 1+UnicodeUtil.UNI_SUR_LOW_END+r.nextInt(max-UnicodeUtil.UNI_SUR_LOW_END);
} else {
- throw new IllegalArgumentException("transition accepts only surrogates: " + t);
+ throw new IllegalArgumentException("transition accepts only surrogates: min=" + min + " max=" + max);
}
- } else if (t.max <= UnicodeUtil.UNI_SUR_LOW_END) {
- if (t.min < UnicodeUtil.UNI_SUR_HIGH_START) {
+ } else if (max <= UnicodeUtil.UNI_SUR_LOW_END) {
+ if (min < UnicodeUtil.UNI_SUR_HIGH_START) {
// before surrogates
- code = t.min + r.nextInt(UnicodeUtil.UNI_SUR_HIGH_START - t.min);
+ code = min + r.nextInt(UnicodeUtil.UNI_SUR_HIGH_START - min);
} else {
- throw new IllegalArgumentException("transition accepts only surrogates: " + t);
+ throw new IllegalArgumentException("transition accepts only surrogates: min=" + min + " max=" + max);
}
} else {
// range includes all surrogates
- int gap1 = UnicodeUtil.UNI_SUR_HIGH_START - t.min;
- int gap2 = t.max - UnicodeUtil.UNI_SUR_LOW_END;
+ int gap1 = UnicodeUtil.UNI_SUR_HIGH_START - min;
+ int gap2 = max - UnicodeUtil.UNI_SUR_LOW_END;
int c = r.nextInt(gap1+gap2);
if (c < gap1) {
- code = t.min + c;
+ code = min + c;
} else {
code = UnicodeUtil.UNI_SUR_LOW_END + c - gap1 + 1;
}
}
- assert code >= t.min && code <= t.max && (code < UnicodeUtil.UNI_SUR_HIGH_START || code > UnicodeUtil.UNI_SUR_LOW_END):
- "code=" + code + " min=" + t.min + " max=" + t.max;
+ assert code >= min && code <= max && (code < UnicodeUtil.UNI_SUR_HIGH_START || code > UnicodeUtil.UNI_SUR_LOW_END):
+ "code=" + code + " min=" + min + " max=" + max;
return code;
}
@@ -140,11 +138,13 @@ public class AutomatonTestUtil {
private final Map<Transition,Boolean> leadsToAccept;
private final Automaton a;
+ private final Transition[][] transitions;
private static class ArrivingTransition {
- final State from;
+ final int from;
final Transition t;
- public ArrivingTransition(State from, Transition t) {
+
+ public ArrivingTransition(int from, Transition t) {
this.from = from;
this.t = t;
}
@@ -152,32 +152,30 @@ public class AutomatonTestUtil {
public RandomAcceptedStrings(Automaton a) {
this.a = a;
- if (a.isSingleton()) {
- leadsToAccept = null;
- return;
+ if (a.getNumStates() == 0) {
+ throw new IllegalArgumentException("this automaton accepts nothing");
}
+ this.transitions = a.getSortedTransitions();
- // must use IdentityHashmap because two Transitions w/
- // different start nodes can be considered the same
- leadsToAccept = new IdentityHashMap<>();
- final Map<State,List<ArrivingTransition>> allArriving = new HashMap<>();
+ leadsToAccept = new HashMap<>();
+ final Map<Integer,List<ArrivingTransition>> allArriving = new HashMap<>();
- final LinkedList<State> q = new LinkedList<>();
- final Set<State> seen = new HashSet<>();
+ final LinkedList<Integer> q = new LinkedList<>();
+ final Set<Integer> seen = new HashSet<>();
// reverse map the transitions, so we can quickly look
// up all arriving transitions to a given state
- for(State s: a.getNumberedStates()) {
- for(int i=0;i<s.numTransitions;i++) {
- final Transition t = s.transitionsArray[i];
- List<ArrivingTransition> tl = allArriving.get(t.to);
+ int numStates = a.getNumStates();
+ for(int s=0;s<numStates;s++) {
+ for(Transition t : transitions[s]) {
+ List<ArrivingTransition> tl = allArriving.get(t.dest);
if (tl == null) {
tl = new ArrayList<>();
- allArriving.put(t.to, tl);
+ allArriving.put(t.dest, tl);
}
tl.add(new ArrivingTransition(s, t));
}
- if (s.accept) {
+ if (a.isAccept(s)) {
q.add(s);
seen.add(s);
}
@@ -185,12 +183,12 @@ public class AutomatonTestUtil {
// Breadth-first search, from accept states,
// backwards:
- while(!q.isEmpty()) {
- final State s = q.removeFirst();
+ while (q.isEmpty() == false) {
+ final int s = q.removeFirst();
List<ArrivingTransition> arriving = allArriving.get(s);
if (arriving != null) {
for(ArrivingTransition at : arriving) {
- final State from = at.from;
+ final int from = at.from;
if (!seen.contains(from)) {
q.add(from);
seen.add(from);
@@ -204,62 +202,49 @@ public class AutomatonTestUtil {
public int[] getRandomAcceptedString(Random r) {
final List<Integer> soFar = new ArrayList<>();
- if (a.isSingleton()) {
- // accepts only one
- final String s = a.singleton;
-
- int charUpto = 0;
- while(charUpto < s.length()) {
- final int cp = s.codePointAt(charUpto);
- charUpto += Character.charCount(cp);
- soFar.add(cp);
- }
- } else {
- State s = a.initial;
+ int s = 0;
- while(true) {
+ while(true) {
- if (s.accept) {
- if (s.numTransitions == 0) {
- // stop now
+ if (a.isAccept(s)) {
+ if (a.getNumTransitions(s) == 0) {
+ // stop now
+ break;
+ } else {
+ if (r.nextBoolean()) {
break;
- } else {
- if (r.nextBoolean()) {
- break;
- }
}
}
+ }
- if (s.numTransitions == 0) {
- throw new RuntimeException("this automaton has dead states");
- }
+ if (a.getNumTransitions(s) == 0) {
+ throw new RuntimeException("this automaton has dead states");
+ }
- boolean cheat = r.nextBoolean();
+ boolean cheat = r.nextBoolean();
- final Transition t;
- if (cheat) {
- // pick a transition that we know is the fastest
- // path to an accept state
- List<Transition> toAccept = new ArrayList<>();
- for(int i=0;i<s.numTransitions;i++) {
- final Transition t0 = s.transitionsArray[i];
- if (leadsToAccept.containsKey(t0)) {
- toAccept.add(t0);
- }
- }
- if (toAccept.size() == 0) {
- // this is OK -- it means we jumped into a cycle
- t = s.transitionsArray[r.nextInt(s.numTransitions)];
- } else {
- t = toAccept.get(r.nextInt(toAccept.size()));
+ final Transition t;
+ if (cheat) {
+ // pick a transition that we know is the fastest
+ // path to an accept state
+ List<Transition> toAccept = new ArrayList<>();
+ for(Transition t0 : transitions[s]) {
+ if (leadsToAccept.containsKey(t0)) {
+ toAccept.add(t0);
}
+ }
+ if (toAccept.size() == 0) {
+ // this is OK -- it means we jumped into a cycle
+ t = transitions[s][r.nextInt(transitions[s].length)];
} else {
- t = s.transitionsArray[r.nextInt(s.numTransitions)];
+ t = toAccept.get(r.nextInt(toAccept.size()));
}
- soFar.add(getRandomCodePoint(r, t));
- s = t.to;
+ } else {
+ t = transitions[s][r.nextInt(transitions[s].length)];
}
+ soFar.add(getRandomCodePoint(r, t.min, t.max));
+ s = t.dest;
}
return ArrayUtil.toIntArray(soFar);
@@ -270,19 +255,21 @@ public class AutomatonTestUtil {
public static Automaton randomAutomaton(Random random) {
// get two random Automata from regexps
Automaton a1 = new RegExp(AutomatonTestUtil.randomRegexp(random), RegExp.NONE).toAutomaton();
- if (random.nextBoolean())
- a1 = BasicOperations.complement(a1);
+ if (random.nextBoolean()) {
+ a1 = Operations.complement(a1);
+ }
Automaton a2 = new RegExp(AutomatonTestUtil.randomRegexp(random), RegExp.NONE).toAutomaton();
- if (random.nextBoolean())
- a2 = BasicOperations.complement(a2);
-
+ if (random.nextBoolean()) {
+ a2 = Operations.complement(a2);
+ }
+
// combine them in random ways
- switch(random.nextInt(4)) {
- case 0: return BasicOperations.concatenate(a1, a2);
- case 1: return BasicOperations.union(a1, a2);
- case 2: return BasicOperations.intersection(a1, a2);
- default: return BasicOperations.minus(a1, a2);
+ switch (random.nextInt(4)) {
+ case 0: return Operations.concatenate(a1, a2);
+ case 1: return Operations.union(a1, a2);
+ case 2: return Operations.intersection(a1, a2);
+ default: return Operations.minus(a1, a2);
}
}
@@ -323,70 +310,81 @@ public class AutomatonTestUtil {
/**
* Simple, original brics implementation of Brzozowski minimize()
*/
- public static void minimizeSimple(Automaton a) {
- if (a.isSingleton())
- return;
- determinizeSimple(a, SpecialOperations.reverse(a));
- determinizeSimple(a, SpecialOperations.reverse(a));
+ public static Automaton minimizeSimple(Automaton a) {
+ Set<Integer> initialSet = new HashSet<Integer>();
+ a = determinizeSimple(Operations.reverse(a, initialSet), initialSet);
+ initialSet.clear();
+ a = determinizeSimple(Operations.reverse(a, initialSet), initialSet);
+ return a;
}
/**
* Simple, original brics implementation of determinize()
*/
- public static void determinizeSimple(Automaton a) {
- if (a.deterministic || a.isSingleton())
- return;
- Set<State> initialset = new HashSet<>();
- initialset.add(a.initial);
- determinizeSimple(a, initialset);
+ public static Automaton determinizeSimple(Automaton a) {
+ Set<Integer> initialset = new HashSet<>();
+ initialset.add(0);
+ return determinizeSimple(a, initialset);
}
-
+
/**
* Simple, original brics implementation of determinize()
* Determinizes the given automaton using the given set of initial states.
*/
- public static void determinizeSimple(Automaton a, Set<State> initialset) {
+ public static Automaton determinizeSimple(Automaton a, Set<Integer> initialset) {
+ if (a.getNumStates() == 0) {
+ return a;
+ }
int[] points = a.getStartPoints();
// subset construction
- Map<Set<State>, Set<State>> sets = new HashMap<>();
- LinkedList<Set<State>> worklist = new LinkedList<>();
- Map<Set<State>, State> newstate = new HashMap<>();
+ Map<Set<Integer>, Set<Integer>> sets = new HashMap<>();
+ LinkedList<Set<Integer>> worklist = new LinkedList<>();
+ Map<Set<Integer>, Integer> newstate = new HashMap<>();
sets.put(initialset, initialset);
worklist.add(initialset);
- a.initial = new State();
- newstate.put(initialset, a.initial);
+ Automaton.Builder result = new Automaton.Builder();
+ result.createState();
+ newstate.put(initialset, 0);
+ Transition t = new Transition();
while (worklist.size() > 0) {
- Set<State> s = worklist.removeFirst();
- State r = newstate.get(s);
- for (State q : s)
- if (q.accept) {
- r.accept = true;
+ Set<Integer> s = worklist.removeFirst();
+ int r = newstate.get(s);
+ for (int q : s) {
+ if (a.isAccept(q)) {
+ result.setAccept(r, true);
break;
}
+ }
for (int n = 0; n < points.length; n++) {
- Set<State> p = new HashSet<>();
- for (State q : s)
- for (Transition t : q.getTransitions())
- if (t.min <= points[n] && points[n] <= t.max)
- p.add(t.to);
+ Set<Integer> p = new HashSet<>();
+ for (int q : s) {
+ int count = a.initTransition(q, t);
+ for(int i=0;i<count;i++) {
+ a.getNextTransition(t);
+ if (t.min <= points[n] && points[n] <= t.max) {
+ p.add(t.dest);
+ }
+ }
+ }
+
if (!sets.containsKey(p)) {
sets.put(p, p);
worklist.add(p);
- newstate.put(p, new State());
+ newstate.put(p, result.createState());
}
- State q = newstate.get(p);
+ int q = newstate.get(p);
int min = points[n];
int max;
- if (n + 1 < points.length)
+ if (n + 1 < points.length) {
max = points[n + 1] - 1;
- else
+ } else {
max = Character.MAX_CODE_POINT;
- r.addTransition(new Transition(min, max, q));
+ }
+ result.addTransition(r, q, min, max);
}
}
- a.deterministic = true;
- a.clearNumberedStates();
- a.removeDeadTransitions();
+
+ return Operations.removeDeadStates(result.finish());
}
/**
@@ -403,11 +401,7 @@ public class AutomatonTestUtil {
*/
public static Set<IntsRef> getFiniteStringsRecursive(Automaton a, int limit) {
HashSet<IntsRef> strings = new HashSet<>();
- if (a.isSingleton()) {
- if (limit > 0) {
- strings.add(Util.toUTF32(a.singleton, new IntsRef()));
- }
- } else if (!getFiniteStrings(a.initial, new HashSet<State>(), strings, new IntsRef(), limit)) {
+ if (!getFiniteStrings(a, 0, new HashSet<Integer>(), strings, new IntsRef(), limit)) {
return strings;
}
return strings;
@@ -418,24 +412,27 @@ public class AutomatonTestUtil {
* false if more than <code>limit</code> strings are found.
* <code>limit</code><0 means "infinite".
*/
- private static boolean getFiniteStrings(State s, HashSet<State> pathstates,
+ private static boolean getFiniteStrings(Automaton a, int s, HashSet<Integer> pathstates,
HashSet<IntsRef> strings, IntsRef path, int limit) {
pathstates.add(s);
- for (Transition t : s.getTransitions()) {
- if (pathstates.contains(t.to)) {
+ Transition t = new Transition();
+ int count = a.initTransition(s, t);
+ for (int i=0;i<count;i++) {
+ a.getNextTransition(t);
+ if (pathstates.contains(t.dest)) {
return false;
}
for (int n = t.min; n <= t.max; n++) {
path.grow(path.length+1);
path.ints[path.length] = n;
path.length++;
- if (t.to.accept) {
+ if (a.isAccept(t.dest)) {
strings.add(IntsRef.deepCopyOf(path));
if (limit >= 0 && strings.size() > limit) {
return false;
}
}
- if (!getFiniteStrings(t.to, pathstates, strings, path, limit)) {
+ if (!getFiniteStrings(a, t.dest, pathstates, strings, path, limit)) {
return false;
}
path.length--;
@@ -452,8 +449,10 @@ public class AutomatonTestUtil {
* this is only used to test the correctness of our faster implementation.
*/
public static boolean isFiniteSlow(Automaton a) {
- if (a.isSingleton()) return true;
- return isFiniteSlow(a.initial, new HashSet<State>());
+ if (a.getNumStates() == 0) {
+ return true;
+ }
+ return isFiniteSlow(a, 0, new HashSet<Integer>());
}
/**
@@ -462,22 +461,48 @@ public class AutomatonTestUtil {
*/
// TODO: not great that this is recursive... in theory a
// large automata could exceed java's stack
- private static boolean isFiniteSlow(State s, HashSet<State> path) {
+ private static boolean isFiniteSlow(Automaton a, int s, HashSet<Integer> path) {
path.add(s);
- for (Transition t : s.getTransitions())
- if (path.contains(t.to) || !isFiniteSlow(t.to, path)) return false;
+ Transition t = new Transition();
+ int count = a.initTransition(s, t);
+ for (int i=0;i<count;i++) {
+ a.getNextTransition(t);
+ if (path.contains(t.dest) || !isFiniteSlow(a, t.dest, path)) {
+ return false;
+ }
+ }
path.remove(s);
return true;
}
-
/**
* Checks that an automaton has no detached states that are unreachable
* from the initial state.
*/
public static void assertNoDetachedStates(Automaton a) {
- int numStates = a.getNumberOfStates();
- a.clearNumberedStates(); // force recomputation of cached numbered states
- assert numStates == a.getNumberOfStates() : "automaton has " + (numStates - a.getNumberOfStates()) + " detached states";
+ Automaton a2 = Operations.removeDeadStates(a);
+ assert a.getNumStates() == a2.getNumStates() : "automaton has " + (a.getNumStates() - a2.getNumStates()) + " detached states";
}
+
+ /** Returns true if the automaton is deterministic. */
+ public static boolean isDeterministicSlow(Automaton a) {
+ Transition t = new Transition();
+ int numStates = a.getNumStates();
+ for(int s=0;s<numStates;s++) {
+ int count = a.initTransition(s, t);
+ int lastMax = -1;
+ for(int i=0;i<count;i++) {
+ a.getNextTransition(t);
+ if (t.min <= lastMax) {
+ assert a.isDeterministic() == false;
+ return false;
+ }
+ lastMax = t.max;
+ }
+ }
+
+ assert a.isDeterministic() == true;
+ return true;
+ }
+
}
Modified: lucene/dev/branches/solr-5473/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr-5473/solr/CHANGES.txt?rev=1603938&r1=1603937&r2=1603938&view=diff
==============================================================================
--- lucene/dev/branches/solr-5473/solr/CHANGES.txt (original)
+++ lucene/dev/branches/solr-5473/solr/CHANGES.txt Thu Jun 19 16:25:31 2014
@@ -84,14 +84,21 @@ Other Changes
* SOLR-5868: HttpClient should be configured to use ALLOW_ALL_HOSTNAME hostname
verifier to simplify SSL setup. (Steve Davids via Mark Miller)
-* SOLR-6169: Properly remove CoreAdminHandler handleAlias action (Alan Woodward)
+* SOLR-6169: Finish removal of CoreAdminHandler handleAlias action begun in 4.9
+ (Alan Woodward)
================== 4.10.0 =================
-Bug fixes
+Bug Fixes
+----------------------
* SOLR-6095 : SolrCloud cluster can end up without an overseer with overseer roles (Noble Paul, Shalin Mangar)
+Other Changes
+---------------------
+
+* SOLR-6173: Fixed wrong failure message in TestDistributedSearch. (shalin)
+
================== 4.9.0 ==================
Versions of Major Components
@@ -144,6 +151,12 @@ New Features
* SOLR-6150: Add new AnalyticsQuery to support pluggable analytics
(Joel Bernstein)
+* SOLR-6125: Allow SolrIndexWriter to close without waiting for merges
+ (Christine Poerschke via Alan Woodward)
+
+* SOLR-6064: DebugComponent track output should be returned as a JSON
+ object rather than a list (Christine Poerschke, Alan Woodward)
+
Bug Fixes
----------------------
@@ -188,10 +201,7 @@ Bug Fixes
* SOLR-6158: Relative configSetBase directories were resolved relative to the
container CWD, rather than solr.home. (Simon Endele, Alan Woodward)
-* SOLR-6056: Don't publish recovery state until recovery runs to avoid overwhelming
- the overseer state queue. (Raintung Li, Mark Miller, shalin)
-
-* SOLR-5426: Fixed a bug in ReverseWildCardFilter that could cause
+* SOLR-5426: Fixed a bug in ReverseWildCardFilter that could cause
InvalidTokenOffsetsException when highlighting. (Uwe Schindler, Arun Kumar, via hossman)
* SOLR-6175: DebugComponent throws NPE on shard exceptions when using shards.tolerant.
@@ -258,6 +268,12 @@ Other Changes
* SOLR-6153: ReplicationHandler backup response format should contain backup name.
(Varun Thacker via shalin)
+* SOLR-6169: Remove broken handleAlias action in CoreAdminHandler (Alan
+ Woodward)
+
+* SOLR-6128: Removed deprecated analysis factories and fieldTypes from the example
+ schema.xml (hossman)
+
Optimizations
----------------------
Modified: lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/SolrLogFormatter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/SolrLogFormatter.java?rev=1603938&r1=1603937&r2=1603938&view=diff
==============================================================================
--- lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/SolrLogFormatter.java (original)
+++ lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/SolrLogFormatter.java Thu Jun 19 16:25:31 2014
@@ -263,7 +263,7 @@ sb.append("(group_name=").append(tg.getN
private Map<String,Object> getReplicaProps(ZkController zkController, SolrCore core) {
final String collection = core.getCoreDescriptor().getCloudDescriptor().getCollectionName();
- Replica replica = zkController.getClusterState().getReplica(collection, core.getCoreDescriptor().getCloudDescriptor().getCoreNodeName());
+ Replica replica = zkController.getClusterState().getReplica(collection, core.getCoreDescriptor().getCloudDescriptor().getCoreNodeName(),true);
if(replica!=null) {
return replica.getProperties();
}
Modified: lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/cloud/Overseer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1603938&r1=1603937&r2=1603938&view=diff
==============================================================================
--- lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original)
+++ lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/cloud/Overseer.java Thu Jun 19 16:25:31 2014
@@ -33,6 +33,7 @@ import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -104,6 +105,8 @@ public class Overseer {
private Map clusterProps;
private boolean isClosed = false;
+ private final Map<String, Object> updateNodes = new ConcurrentHashMap<>();
+ private boolean isClusterStateModified = false;
public ClusterStateUpdater(final ZkStateReader reader, final String myId, Stats zkStats) {
@@ -117,6 +120,7 @@ public class Overseer {
this.myId = myId;
this.reader = reader;
clusterProps = reader.getClusterProps();
+ reader.ephemeralCollectionData = Collections.unmodifiableMap(updateNodes);
}
public Stats getStateUpdateQueueStats() {
@@ -257,6 +261,7 @@ public class Overseer {
stateUpdateQueue.poll();
if (isClosed || System.nanoTime() - lastUpdatedTime > TimeUnit.NANOSECONDS.convert(STATE_UPDATE_DELAY, TimeUnit.MILLISECONDS)) break;
+ if(!updateNodes.isEmpty()) break;
// if an event comes in the next 100ms batch it together
head = stateUpdateQueue.peek(100);
}
@@ -293,9 +298,28 @@ public class Overseer {
}
private void updateZkStates(ClusterState clusterState) throws KeeperException, InterruptedException {
- lastUpdatedTime = System.nanoTime();
- zkClient.setData(ZkStateReader.CLUSTER_STATE, ZkStateReader.toJSON(clusterState), true);
+ if(!updateNodes.isEmpty()) {
+ for (Entry<String, Object> e : updateNodes.entrySet()) {
+ if (e.getValue() == null) {
+ if (zkClient.exists(e.getKey(), true)) zkClient.delete(e.getKey(), 0, true);
+ } else {
+ if (zkClient.exists(e.getKey(), true)) {
+ log.info("going to update_collection {}", e.getKey());
+ zkClient.setData(e.getKey(), ZkStateReader.toJSON(e.getValue()), true);
+ } else {
+ log.info("going to create_collection {}", e.getValue());
+ zkClient.create(e.getKey(), ZkStateReader.toJSON(e.getValue()), CreateMode.PERSISTENT, true);
+ }
+ }
+ }
+ updateNodes.clear();
+ }
+ if(isClusterStateModified) {
+ lastUpdatedTime = System.nanoTime();
+ zkClient.setData(ZkStateReader.CLUSTER_STATE, ZkStateReader.toJSON(clusterState), true);
+ isClusterStateModified = false;
+ }
}
private void checkIfIamStillLeader() {
@@ -326,8 +350,12 @@ public class Overseer {
}
} finally {
//if I am not shutting down, Then I need to rejoin election
- if (zkController != null && !zkController.getCoreContainer().isShutDown()) {
- zkController.rejoinOverseerElection(null, false);
+ try {
+ if (zkController != null && !zkController.getCoreContainer().isShutDown()) {
+ zkController.rejoinOverseerElection(null, false);
+ }
+ } catch (Exception e) {
+ log.warn("Unable to rejoinElection ",e);
}
}
}
@@ -673,7 +701,7 @@ public class Overseer {
}
Slice slice = clusterState.getSlice(collection, sliceName);
-
+
Map<String,Object> replicaProps = new LinkedHashMap<>();
replicaProps.putAll(message.getProperties());
@@ -691,7 +719,7 @@ public class Overseer {
replicaProps.remove(ZkStateReader.SHARD_ID_PROP);
replicaProps.remove(ZkStateReader.COLLECTION_PROP);
replicaProps.remove(QUEUE_OPERATION);
-
+
// remove any props with null values
Set<Entry<String,Object>> entrySet = replicaProps.entrySet();
List<String> removeKeys = new ArrayList<>();
@@ -839,10 +867,18 @@ public class Overseer {
}
collectionProps.put(DocCollection.DOC_ROUTER, routerSpec);
- if(message.getStr("fromApi") == null) collectionProps.put("autoCreated","true");
- DocCollection newCollection = new DocCollection(collectionName, newSlices, collectionProps, router);
- return newState(state, singletonMap(newCollection.getName(), newCollection));
+ if(message.getStr("fromApi") == null) collectionProps.put("autoCreated","true");
+ String znode = message.getInt(DocCollection.STATE_FORMAT, 1) == 1 ? null : ZkStateReader.getCollectionPath(collectionName);
+ DocCollection newCollection = new DocCollection(collectionName, newSlices, collectionProps, router , -1,znode);
+ isClusterStateModified = true;
+ log.info("state version {} {}", collectionName, newCollection.getStateFormat());
+ if (newCollection.getStateFormat() > 1) {
+ updateNodes.put(ZkStateReader.getCollectionPath(collectionName),
+ new ClusterState(-1, Collections.<String>emptySet(), singletonMap(newCollection.getName(), newCollection), state.getStateReader()));
+ return state;
}
+ return newState(state, singletonMap(newCollection.getName(), newCollection));
+ }
/*
* Return an already assigned id or null if not assigned
@@ -879,30 +915,27 @@ public class Overseer {
}
return null;
}
-
+
private ClusterState updateSlice(ClusterState state, String collectionName, Slice slice) {
// System.out.println("###!!!### OLD CLUSTERSTATE: " + JSONUtil.toJSON(state.getCollectionStates()));
// System.out.println("Updating slice:" + slice);
-
+ DocCollection newCollection = null;
DocCollection coll = state.getCollectionOrNull(collectionName) ;
Map<String,Slice> slices;
- Map<String,Object> props;
- DocRouter router;
-
if (coll == null) {
// when updateSlice is called on a collection that doesn't exist, it's currently when a core is publishing itself
// without explicitly creating a collection. In this current case, we assume custom sharding with an "implicit" router.
- slices = new HashMap<>(1);
- props = new HashMap<>(1);
+ slices = new LinkedHashMap<>(1);
+ slices.put(slice.getName(), slice);
+ Map<String,Object> props = new HashMap<>(1);
props.put(DocCollection.DOC_ROUTER, ZkNodeProps.makeMap("name",ImplicitDocRouter.NAME));
- router = new ImplicitDocRouter();
+ newCollection = new DocCollection(collectionName, slices, props, new ImplicitDocRouter());
} else {
- props = coll.getProperties();
- router = coll.getRouter();
slices = new LinkedHashMap<>(coll.getSlicesMap()); // make a shallow copy
+ slices.put(slice.getName(), slice);
+ newCollection = coll.copyWith(slices);
}
- slices.put(slice.getName(), slice);
- DocCollection newCollection = new DocCollection(collectionName, slices, props, router);
+
// System.out.println("###!!!### NEW CLUSTERSTATE: " + JSONUtil.toJSON(newCollections));
@@ -961,27 +994,54 @@ public class Overseer {
}
- DocCollection newCollection = new DocCollection(coll.getName(), slices, coll.getProperties(), coll.getRouter());
+ DocCollection newCollection = coll.copyWith(slices);
return newState(state, singletonMap(collectionName, newCollection));
}
- private ClusterState newState(ClusterState state, Map<String, DocCollection> colls) {
- return state.copyWith(colls);
+ private ClusterState newState(ClusterState state, Map<String, DocCollection> colls) {
+ for (Entry<String, DocCollection> e : colls.entrySet()) {
+ DocCollection c = e.getValue();
+ if (c == null) {
+ isClusterStateModified = true;
+ state = state.copyWith(singletonMap(e.getKey(), (DocCollection) null));
+ continue;
+ }
+
+ if (c.getStateFormat() >1) {
+ state.getStateReader().updateWatchedCollection(c);
+ updateNodes.put(ZkStateReader.getCollectionPath(c.getName()), new ClusterState(-1, Collections.<String>emptySet(), singletonMap(c.getName(), c), state.getStateReader()));
+ } else {
+ isClusterStateModified = true;
+ state = state.copyWith(singletonMap(e.getKey(), c));
+ }
}
+ return state;
+ }
- /*
- * Remove collection from cloudstate
- */
- private ClusterState removeCollection(final ClusterState clusterState, ZkNodeProps message) {
- final String collection = message.getStr("name");
- if (!checkKeyExistence(message, "name")) return clusterState;
- DocCollection coll = clusterState.getCollectionOrNull(collection);
- if(coll !=null) {
+ /*
+ * Remove collection from cloudstate
+ */
+ private ClusterState removeCollection(final ClusterState clusterState, ZkNodeProps message) {
+ final String collection = message.getStr("name");
+ if (!checkKeyExistence(message, "name")) return clusterState;
+ DocCollection coll = clusterState.getCollectionOrNull(collection);
+ if(coll !=null) {
+ isClusterStateModified = true;
+ if(coll.getStateFormat()>1){
+ try {
+ log.info("Deleting state for collection : {}", collection);
+ zkClient.delete(ZkStateReader.getCollectionPath(collection),-1,true);
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,"Unable to remove collection state :"+collection);
+
+ }
+ return clusterState;
+ } else{
return clusterState.copyWith(singletonMap(collection,(DocCollection)null));
}
- return clusterState;
}
-
+ return clusterState;
+ }
/*
* Remove collection slice from cloudstate
*/
@@ -997,7 +1057,7 @@ public class Overseer {
Map<String, Slice> newSlices = new LinkedHashMap<>(coll.getSlicesMap());
newSlices.remove(sliceId);
- DocCollection newCollection = new DocCollection(coll.getName(), newSlices, coll.getProperties(), coll.getRouter());
+ DocCollection newCollection = coll.copyWith(newSlices);
return newState(clusterState, singletonMap(collection,newCollection));
}
@@ -1075,8 +1135,8 @@ public class Overseer {
return newState(clusterState,singletonMap(collection, (DocCollection) null));
} else {
- DocCollection newCollection = new DocCollection(coll.getName(), newSlices, coll.getProperties(), coll.getRouter());
- return newState(clusterState,singletonMap(collection,newCollection));
+ DocCollection newCollection = coll.copyWith(newSlices);
+ return newState(clusterState,singletonMap(collection,newCollection));
}
}
Modified: lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java?rev=1603938&r1=1603937&r2=1603938&view=diff
==============================================================================
--- lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java (original)
+++ lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java Thu Jun 19 16:25:31 2014
@@ -690,7 +690,13 @@ public class OverseerCollectionProcessor
Set<String> collections = clusterState.getCollections();
for (String name : collections) {
Map<String, Object> collectionStatus = null;
- collectionStatus = getCollectionStatus((Map<String, Object>) stateMap.get(name), name, shard);
+ if (clusterState.getCollection(name).getStateFormat()>1) {
+ bytes = ZkStateReader.toJSON(clusterState.getCollection(name));
+ Map<String, Object> docCollection = (Map<String, Object>) ZkStateReader.fromJSON(bytes);
+ collectionStatus = getCollectionStatus(docCollection, name, shard);
+ } else {
+ collectionStatus = getCollectionStatus((Map<String, Object>) stateMap.get(name), name, shard);
+ }
if (collectionVsAliases.containsKey(name) && !collectionVsAliases.get(name).isEmpty()) {
collectionStatus.put("aliases", collectionVsAliases.get(name));
}
Modified: lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/cloud/ZkController.java?rev=1603938&r1=1603937&r2=1603938&view=diff
==============================================================================
--- lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/cloud/ZkController.java (original)
+++ lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/cloud/ZkController.java Thu Jun 19 16:25:31 2014
@@ -1109,6 +1109,16 @@ public final class ZkController {
}
CloudDescriptor cloudDescriptor = cd.getCloudDescriptor();
+ boolean removeWatch = true;
+ for (SolrCore solrCore : cc.getCores()) {//if there is no SolrCoe which is a member of this collection, remove the watch
+ CloudDescriptor cloudDesc = solrCore.getCoreDescriptor().getCloudDescriptor();
+ if (cloudDesc != null && cloudDescriptor.getCollectionName().equals(cloudDesc.getCollectionName())) {
+ //means
+ removeWatch = false;
+ break;
+ }
+ }
+ if(removeWatch) zkStateReader.removeZKWatch(collection);
ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION,
Overseer.DELETECORE, ZkStateReader.CORE_NAME_PROP, coreName,
ZkStateReader.NODE_NAME_PROP, getNodeName(),
Modified: lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/core/SolrConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/core/SolrConfig.java?rev=1603938&r1=1603937&r2=1603938&view=diff
==============================================================================
--- lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/core/SolrConfig.java (original)
+++ lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/core/SolrConfig.java Thu Jun 19 16:25:31 2014
@@ -315,6 +315,7 @@ public class SolrConfig extends Config {
return new UpdateHandlerInfo(get("updateHandler/@class",null),
getInt("updateHandler/autoCommit/maxDocs",-1),
getInt("updateHandler/autoCommit/maxTime",-1),
+ getBool("updateHandler/indexWriter/closeWaitsForMerges",true),
getBool("updateHandler/autoCommit/openSearcher",true),
getInt("updateHandler/commitIntervalLowerBound",-1),
getInt("updateHandler/autoSoftCommit/maxDocs",-1),
@@ -497,6 +498,7 @@ public class SolrConfig extends Config {
public final String className;
public final int autoCommmitMaxDocs,autoCommmitMaxTime,commitIntervalLowerBound,
autoSoftCommmitMaxDocs,autoSoftCommmitMaxTime;
+ public final boolean indexWriterCloseWaitsForMerges;
public final boolean openSearcher; // is opening a new searcher part of hard autocommit?
public final boolean commitWithinSoftCommit;
@@ -505,11 +507,12 @@ public class SolrConfig extends Config {
* @param autoCommmitMaxTime set -1 as default
* @param commitIntervalLowerBound set -1 as default
*/
- public UpdateHandlerInfo(String className, int autoCommmitMaxDocs, int autoCommmitMaxTime, boolean openSearcher, int commitIntervalLowerBound,
+ public UpdateHandlerInfo(String className, int autoCommmitMaxDocs, int autoCommmitMaxTime, boolean indexWriterCloseWaitsForMerges, boolean openSearcher, int commitIntervalLowerBound,
int autoSoftCommmitMaxDocs, int autoSoftCommmitMaxTime, boolean commitWithinSoftCommit) {
this.className = className;
this.autoCommmitMaxDocs = autoCommmitMaxDocs;
this.autoCommmitMaxTime = autoCommmitMaxTime;
+ this.indexWriterCloseWaitsForMerges = indexWriterCloseWaitsForMerges;
this.openSearcher = openSearcher;
this.commitIntervalLowerBound = commitIntervalLowerBound;
Modified: lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java?rev=1603938&r1=1603937&r2=1603938&view=diff
==============================================================================
--- lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java (original)
+++ lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java Thu Jun 19 16:25:31 2014
@@ -143,7 +143,7 @@ public class CollectionsHandler extends
if (action == null) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown action: " + a);
}
-
+
switch (action) {
case CREATE: {
this.handleCreateAction(req, rsp);
@@ -320,36 +320,36 @@ public class CollectionsHandler extends
SolrQueryResponse rsp) throws KeeperException, InterruptedException {
handleResponse(operation, m, rsp, DEFAULT_ZK_TIMEOUT);
}
-
+
private void handleResponse(String operation, ZkNodeProps m,
SolrQueryResponse rsp, long timeout) throws KeeperException, InterruptedException {
long time = System.nanoTime();
if(m.containsKey(ASYNC) && m.get(ASYNC) != null) {
-
+
String asyncId = m.getStr(ASYNC);
-
+
if(asyncId.equals("-1")) {
throw new SolrException(ErrorCode.BAD_REQUEST, "requestid can not be -1. It is reserved for cleanup purposes.");
}
-
+
NamedList<String> r = new NamedList<>();
-
+
if (coreContainer.getZkController().getOverseerCompletedMap().contains(asyncId) ||
coreContainer.getZkController().getOverseerFailureMap().contains(asyncId) ||
coreContainer.getZkController().getOverseerRunningMap().contains(asyncId) ||
overseerCollectionQueueContains(asyncId)) {
r.add("error", "Task with the same requestid already exists.");
-
+
} else {
coreContainer.getZkController().getOverseerCollectionQueue()
.offer(ZkStateReader.toJSON(m));
}
r.add(CoreAdminParams.REQUESTID, (String) m.get(ASYNC));
SolrResponse response = new OverseerSolrResponse(r);
-
+
rsp.getValues().addAll(response.getResponse());
-
+
return;
}
@@ -380,27 +380,27 @@ public class CollectionsHandler extends
}
}
}
-
+
private void handleReloadAction(SolrQueryRequest req, SolrQueryResponse rsp) throws KeeperException, InterruptedException {
log.info("Reloading Collection : " + req.getParamString());
String name = req.getParams().required().get("name");
-
+
ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION,
OverseerCollectionProcessor.RELOADCOLLECTION, "name", name);
handleResponse(OverseerCollectionProcessor.RELOADCOLLECTION, m, rsp);
}
-
+
private void handleSyncShardAction(SolrQueryRequest req, SolrQueryResponse rsp) throws KeeperException, InterruptedException, SolrServerException, IOException {
log.info("Syncing shard : " + req.getParamString());
String collection = req.getParams().required().get("collection");
String shard = req.getParams().required().get("shard");
-
+
ClusterState clusterState = coreContainer.getZkController().getClusterState();
-
+
ZkNodeProps leaderProps = clusterState.getLeader(collection, shard);
ZkCoreNodeProps nodeProps = new ZkCoreNodeProps(leaderProps);
-
+
HttpSolrServer server = new HttpSolrServer(nodeProps.getBaseUrl());
try {
server.setConnectionTimeout(15000);
@@ -414,36 +414,36 @@ public class CollectionsHandler extends
server.shutdown();
}
}
-
+
private void handleCreateAliasAction(SolrQueryRequest req,
SolrQueryResponse rsp) throws Exception {
log.info("Create alias action : " + req.getParamString());
String name = req.getParams().required().get("name");
String collections = req.getParams().required().get("collections");
-
+
ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION,
OverseerCollectionProcessor.CREATEALIAS, "name", name, "collections",
collections);
-
+
handleResponse(OverseerCollectionProcessor.CREATEALIAS, m, rsp);
}
-
+
private void handleDeleteAliasAction(SolrQueryRequest req,
SolrQueryResponse rsp) throws Exception {
log.info("Delete alias action : " + req.getParamString());
String name = req.getParams().required().get("name");
-
+
ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION,
OverseerCollectionProcessor.DELETEALIAS, "name", name);
-
+
handleResponse(OverseerCollectionProcessor.DELETEALIAS, m, rsp);
}
private void handleDeleteAction(SolrQueryRequest req, SolrQueryResponse rsp) throws KeeperException, InterruptedException {
log.info("Deleting Collection : " + req.getParamString());
-
+
String name = req.getParams().required().get("name");
-
+
ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION,
OverseerCollectionProcessor.DELETECOLLECTION, "name", name);
@@ -464,7 +464,7 @@ public class CollectionsHandler extends
throw new SolrException(ErrorCode.BAD_REQUEST,
"Collection name is required to create a new collection");
}
-
+
Map<String,Object> props = ZkNodeProps.makeMap(
Overseer.QUEUE_OPERATION,
OverseerCollectionProcessor.CREATECOLLECTION,
@@ -477,6 +477,7 @@ public class CollectionsHandler extends
MAX_SHARDS_PER_NODE,
CREATE_NODE_SET ,
SHARDS_PROP,
+ DocCollection.STATE_FORMAT,
ASYNC,
"router.");
@@ -553,7 +554,7 @@ public class CollectionsHandler extends
log.info("Deleting Shard : " + req.getParamString());
String name = req.getParams().required().get(ZkStateReader.COLLECTION_PROP);
String shard = req.getParams().required().get(ZkStateReader.SHARD_ID_PROP);
-
+
Map<String,Object> props = new HashMap<>();
props.put(ZkStateReader.COLLECTION_PROP, name);
props.put(Overseer.QUEUE_OPERATION, OverseerCollectionProcessor.DELETESHARD);
Modified: lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java?rev=1603938&r1=1603937&r2=1603938&view=diff
==============================================================================
--- lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java (original)
+++ lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java Thu Jun 19 16:25:31 2014
@@ -17,7 +17,16 @@
package org.apache.solr.handler.component;
-import static org.apache.solr.common.params.CommonParams.FQ;
+import org.apache.lucene.search.Query;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.search.DocList;
+import org.apache.solr.search.QueryParsing;
+import org.apache.solr.util.SolrPluginUtils;
import java.io.IOException;
import java.net.URL;
@@ -32,16 +41,7 @@ import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicLong;
-import org.apache.lucene.search.Query;
-import org.apache.solr.common.SolrDocumentList;
-import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.SimpleOrderedMap;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.search.DocList;
-import org.apache.solr.search.QueryParsing;
-import org.apache.solr.util.SolrPluginUtils;
+import static org.apache.solr.common.params.CommonParams.FQ;
/**
* Adds debugging information to a request.
@@ -183,7 +183,7 @@ public class DebugComponent extends Sear
@SuppressWarnings("unchecked")
NamedList<Object> stageList = (NamedList<Object>) ((NamedList<Object>)rb.getDebugInfo().get("track")).get(stages.get(rb.stage));
if(stageList == null) {
- stageList = new NamedList<>();
+ stageList = new SimpleOrderedMap<>();
rb.addDebug(stageList, "track", stages.get(rb.stage));
}
for(ShardResponse response: sreq.responses) {
@@ -261,7 +261,7 @@ public class DebugComponent extends Sear
private NamedList<String> getTrackResponse(ShardResponse shardResponse) {
- NamedList<String> namedList = new NamedList<>();
+ NamedList<String> namedList = new SimpleOrderedMap<>();
if (shardResponse.getException() != null) {
namedList.add("Exception", shardResponse.getException().getMessage());
return namedList;
Modified: lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java?rev=1603938&r1=1603937&r2=1603938&view=diff
==============================================================================
--- lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java (original)
+++ lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java Thu Jun 19 16:25:31 2014
@@ -17,6 +17,12 @@
package org.apache.solr.parser;
+import java.io.StringReader;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.util.TokenFilterFactory;
import org.apache.lucene.index.Term;
@@ -34,10 +40,9 @@ import org.apache.lucene.search.Wildcard
import org.apache.lucene.util.QueryBuilder;
import org.apache.lucene.util.ToStringUtils;
import org.apache.lucene.util.Version;
+import org.apache.lucene.util.automaton.Automata;
+import org.apache.lucene.util.automaton.Operations;
import org.apache.lucene.util.automaton.Automaton;
-import org.apache.lucene.util.automaton.BasicAutomata;
-import org.apache.lucene.util.automaton.BasicOperations;
-import org.apache.lucene.util.automaton.SpecialOperations;
import org.apache.solr.analysis.ReversedWildcardFilterFactory;
import org.apache.solr.analysis.TokenizerChain;
import org.apache.solr.common.SolrException;
@@ -49,12 +54,6 @@ import org.apache.solr.schema.TextField;
import org.apache.solr.search.QParser;
import org.apache.solr.search.SyntaxError;
-import java.io.StringReader;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
/** This class is overridden by QueryParser in QueryParser.jj
* and acts to separate the majority of the Java code from the .jj grammar file.
*/
@@ -780,16 +779,16 @@ public abstract class SolrQueryParserBas
Automaton automaton = WildcardQuery.toAutomaton(term);
// TODO: we should likely use the automaton to calculate shouldReverse, too.
if (factory.shouldReverse(termStr)) {
- automaton = BasicOperations.concatenate(automaton, BasicAutomata.makeChar(factory.getMarkerChar()));
- SpecialOperations.reverse(automaton);
+ automaton = Operations.concatenate(automaton, Automata.makeChar(factory.getMarkerChar()));
+ automaton = Operations.reverse(automaton);
} else {
// reverse wildcardfilter is active: remove false positives
// fsa representing false positives (markerChar*)
- Automaton falsePositives = BasicOperations.concatenate(
- BasicAutomata.makeChar(factory.getMarkerChar()),
- BasicAutomata.makeAnyString());
+ Automaton falsePositives = Operations.concatenate(
+ Automata.makeChar(factory.getMarkerChar()),
+ Automata.makeAnyString());
// subtract these away
- automaton = BasicOperations.minus(automaton, falsePositives);
+ automaton = Operations.minus(automaton, falsePositives);
}
return new AutomatonQuery(term, automaton) {
// override toString so its completely transparent
Modified: lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/search/SolrQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/search/SolrQueryParser.java?rev=1603938&r1=1603937&r2=1603938&view=diff
==============================================================================
--- lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/search/SolrQueryParser.java (original)
+++ lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/search/SolrQueryParser.java Thu Jun 19 16:25:31 2014
@@ -25,10 +25,7 @@ import org.apache.lucene.analysis.util.T
import org.apache.lucene.index.Term;
import org.apache.lucene.search.*;
import org.apache.lucene.util.ToStringUtils;
-import org.apache.lucene.util.automaton.Automaton;
-import org.apache.lucene.util.automaton.BasicAutomata;
-import org.apache.lucene.util.automaton.BasicOperations;
-import org.apache.lucene.util.automaton.SpecialOperations;
+import org.apache.lucene.util.automaton.Automata;
import org.apache.lucene.analysis.Analyzer;
import org.apache.solr.analysis.ReversedWildcardFilterFactory;
import org.apache.solr.analysis.TokenizerChain;
Modified: lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java?rev=1603938&r1=1603937&r2=1603938&view=diff
==============================================================================
--- lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java (original)
+++ lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java Thu Jun 19 16:25:31 2014
@@ -324,6 +324,7 @@ public class SolrDispatchFilter extends
String coreUrl = getRemotCoreUrl(cores, corename, origCorename);
// don't proxy for internal update requests
SolrParams queryParams = SolrRequestParsers.parseQueryString(req.getQueryString());
+ checkStateIsValid(cores, queryParams.get(CloudSolrServer.STATE_VERSION));
if (coreUrl != null
&& queryParams
.get(DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM) == null) {
@@ -379,6 +380,7 @@ public class SolrDispatchFilter extends
if( "/select".equals( path ) || "/select/".equals( path ) ) {
solrReq = parser.parse( core, path, req );
+ checkStateIsValid(cores,solrReq.getParams().get(CloudSolrServer.STATE_VERSION));
String qt = solrReq.getParams().get( CommonParams.QT );
handler = core.getRequestHandler( qt );
if( handler == null ) {
@@ -434,12 +436,16 @@ public class SolrDispatchFilter extends
}
catch (Throwable ex) {
sendError( core, solrReq, request, (HttpServletResponse)response, ex );
- if (ex instanceof Error) {
- throw (Error) ex;
- }
- if (ex.getCause() != null && ex.getCause() instanceof Error) {
- log.error("An Error was wrapped in another exception - please report complete stacktrace on SOLR-6161", ex);
- throw (Error) ex.getCause();
+ // walk the the entire cause chain to search for an Error
+ Throwable t = ex;
+ while (t != null) {
+ if (t instanceof Error) {
+ if (t != ex) {
+ log.error("An Error was wrapped in another exception - please report complete stacktrace on SOLR-6161", ex);
+ }
+ throw (Error) t;
+ }
+ t = t.getCause();
}
return;
} finally {
@@ -464,6 +470,22 @@ public class SolrDispatchFilter extends
chain.doFilter(request, response);
}
+ private void checkStateIsValid(CoreContainer cores, String stateVer) {
+ if(stateVer != null && !stateVer.isEmpty() && cores.isZooKeeperAware() ){
+ // many have multiple collections separated by |
+ String[] pairs = StringUtils.split(stateVer, '|');
+ for (String pair : pairs) {
+ String[] pcs = StringUtils.split(pair, ':');
+ if(pcs.length == 2 && !pcs[0].isEmpty() && !pcs[1].isEmpty()){
+ Boolean status = cores.getZkController().getZkStateReader().checkValid(pcs[0],Integer.parseInt(pcs[1]));
+
+ if(Boolean.TRUE != status){
+ throw new SolrException(ErrorCode.INVALID_STATE, "STATE STALE: " + pair+ "valid : "+status);
+ }
+ }
+ }
+ }
+ }
private void processAliases(SolrQueryRequest solrReq, Aliases aliases,
List<String> collectionsList) {
Modified: lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/servlet/ZookeeperInfoServlet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/servlet/ZookeeperInfoServlet.java?rev=1603938&r1=1603937&r2=1603938&view=diff
==============================================================================
--- lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/servlet/ZookeeperInfoServlet.java (original)
+++ lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/servlet/ZookeeperInfoServlet.java Thu Jun 19 16:25:31 2014
@@ -49,6 +49,7 @@ import org.apache.zookeeper.KeeperExcept
import org.apache.zookeeper.KeeperException.NoNodeException;
import org.apache.zookeeper.data.Stat;
import org.noggit.CharArr;
+import org.noggit.ObjectBuilder;
import org.noggit.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -110,6 +111,7 @@ public final class ZookeeperInfoServlet
ZKPrinter printer = new ZKPrinter(response, out, cores.getZkController(), addr);
printer.detail = detail;
printer.dump = dump;
+ printer.isTreeView = (params.get("wt") == null); // this is hacky but tree view requests don't come in with the wt set
try {
printer.print(path);
@@ -140,6 +142,8 @@ public final class ZookeeperInfoServlet
boolean detail = false;
boolean dump = false;
+ boolean isTreeView = false;
+
String addr; // the address passed to us
String keeperAddr; // the address we're connected to
@@ -386,6 +390,48 @@ public final class ZookeeperInfoServlet
}
}
+ // pull in external collections too
+ if ("/clusterstate.json".equals(path) && !isTreeView) {
+ SortedMap<String,Object> collectionStates = null;
+ List<String> children = zkClient.getChildren("/collections", null, true);
+ java.util.Collections.sort(children);
+ for (String collection : children) {
+ String collStatePath = String.format(Locale.ROOT, "/collections/%s/state", collection);
+ String childDataStr = null;
+ try {
+ byte[] childData = zkClient.getData(collStatePath, null, null, true);
+ if (childData != null) {
+ childDataStr = (new BytesRef(childData)).utf8ToString();
+ }
+ } catch (KeeperException.NoNodeException nne) {
+ // safe to ignore
+ } catch (Exception childErr) {
+ log.error("Failed to get "+collStatePath+" due to: "+childErr);
+ }
+
+ if (childDataStr != null) {
+ if (collectionStates == null) {
+ // initialize lazily as there may not be any external collections
+ collectionStates = new TreeMap<String,Object>();
+
+ // add the internal collections
+ if (dataStr != null)
+ collectionStates.putAll((Map<String,Object>)ObjectBuilder.fromJSON(dataStr));
+ }
+
+ // now add in the external collections
+ Map<String,Object> extColl = (Map<String,Object>)ObjectBuilder.fromJSON(childDataStr);
+ collectionStates.put(collection, extColl.get(collection));
+ }
+ }
+
+ if (collectionStates != null) {
+ CharArr out = new CharArr();
+ new JSONWriter(out, 2).write(collectionStates);
+ dataStr = out.toString();
+ }
+ }
+
json.writeString("znode");
json.writeNameSeparator();
json.startObject();
Modified: lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=1603938&r1=1603937&r2=1603938&view=diff
==============================================================================
--- lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java (original)
+++ lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java Thu Jun 19 16:25:31 2014
@@ -94,6 +94,8 @@ public class DirectUpdateHandler2 extend
protected boolean commitWithinSoftCommit;
+ protected boolean indexWriterCloseWaitsForMerges;
+
public DirectUpdateHandler2(SolrCore core) {
super(core);
@@ -110,6 +112,7 @@ public class DirectUpdateHandler2 extend
softCommitTracker = new CommitTracker("Soft", core, softCommitDocsUpperBound, softCommitTimeUpperBound, true, true);
commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit;
+ indexWriterCloseWaitsForMerges = updateHandlerInfo.indexWriterCloseWaitsForMerges;
}
@@ -129,6 +132,7 @@ public class DirectUpdateHandler2 extend
softCommitTracker = new CommitTracker("Soft", core, softCommitDocsUpperBound, softCommitTimeUpperBound, updateHandlerInfo.openSearcher, true);
commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit;
+ indexWriterCloseWaitsForMerges = updateHandlerInfo.indexWriterCloseWaitsForMerges;
UpdateLog existingLog = updateHandler.getUpdateLog();
if (this.ulog != null && this.ulog == existingLog) {
@@ -788,7 +792,7 @@ public class DirectUpdateHandler2 extend
if (writer != null) {
try {
- writer.waitForMerges();
+ if (indexWriterCloseWaitsForMerges) writer.waitForMerges();
} finally {
writer.close();
}
Modified: lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/util/SolrLogLayout.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/util/SolrLogLayout.java?rev=1603938&r1=1603937&r2=1603938&view=diff
==============================================================================
--- lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/util/SolrLogLayout.java (original)
+++ lucene/dev/branches/solr-5473/solr/core/src/java/org/apache/solr/util/SolrLogLayout.java Thu Jun 19 16:25:31 2014
@@ -234,7 +234,7 @@ public class SolrLogLayout extends Layou
private Map<String,Object> getReplicaProps(ZkController zkController, SolrCore core) {
final String collection = core.getCoreDescriptor().getCloudDescriptor().getCollectionName();
- Replica replica = zkController.getClusterState().getReplica(collection, zkController.getCoreNodeName(core.getCoreDescriptor()));
+ Replica replica = zkController.getZkStateReader().getClusterState(). getReplica(collection, zkController.getCoreNodeName(core.getCoreDescriptor()), true);
if(replica!=null) {
return replica.getProperties();
}
Modified: lucene/dev/branches/solr-5473/solr/core/src/test/org/apache/solr/TestDistributedSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr-5473/solr/core/src/test/org/apache/solr/TestDistributedSearch.java?rev=1603938&r1=1603937&r2=1603938&view=diff
==============================================================================
--- lucene/dev/branches/solr-5473/solr/core/src/test/org/apache/solr/TestDistributedSearch.java (original)
+++ lucene/dev/branches/solr-5473/solr/core/src/test/org/apache/solr/TestDistributedSearch.java Thu Jun 19 16:25:31 2014
@@ -27,6 +27,7 @@ import org.apache.lucene.util.LuceneTest
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.cloud.ChaosMonkey;
import org.apache.solr.common.SolrException;
@@ -269,10 +270,10 @@ public class TestDistributedSearch exten
query("q","*:*", "sort",i1+" desc", "stats", "true", "stats.field", tdate_b);
handle.put("stats_fields", UNORDERED);
- query("q","*:*", "sort",i1+" desc", "stats", "true",
- "stats.field", "stats_dt",
- "stats.field", i1,
- "stats.field", tdate_a,
+ query("q","*:*", "sort",i1+" desc", "stats", "true",
+ "stats.field", "stats_dt",
+ "stats.field", i1,
+ "stats.field", tdate_a,
"stats.field", tdate_b);
/*** TODO: the failure may come back in "exception"
@@ -430,9 +431,12 @@ public class TestDistributedSearch exten
"stats.field", tdate_a,
"stats.field", tdate_b,
"stats.calcdistinct", "true");
- } catch (Exception e) {
- log.error("Exception on distrib stats request on empty index", e);
- fail("NullPointerException with stats request on empty index");
+ } catch (HttpSolrServer.RemoteSolrException e) {
+ if (e.getMessage().startsWith("java.lang.NullPointerException")) {
+ fail("NullPointerException with stats request on empty index");
+ } else {
+ throw e;
+ }
}
}
Modified: lucene/dev/branches/solr-5473/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr-5473/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java?rev=1603938&r1=1603937&r2=1603938&view=diff
==============================================================================
--- lucene/dev/branches/solr-5473/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java (original)
+++ lucene/dev/branches/solr-5473/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java Thu Jun 19 16:25:31 2014
@@ -19,7 +19,6 @@ package org.apache.solr.analysis;
import java.io.IOException;
import java.lang.reflect.Field;
-
import java.util.HashMap;
import java.util.Map;
@@ -27,8 +26,8 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.search.AutomatonQuery;
import org.apache.lucene.search.Query;
+import org.apache.lucene.util.automaton.Operations;
import org.apache.lucene.util.automaton.Automaton;
-import org.apache.lucene.util.automaton.SpecialOperations;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.IndexSchema;
@@ -158,14 +157,11 @@ public class TestReversedWildcardFilterF
/** fragile assert: depends on our implementation, but cleanest way to check for now */
private boolean wasReversed(SolrQueryParser qp, String query) throws Exception {
Query q = qp.parse(query);
- if (!(q instanceof AutomatonQuery))
+ if (!(q instanceof AutomatonQuery)) {
return false;
- // this is a hack to get the protected Automaton field in AutomatonQuery,
- // may break in later lucene versions - we have no getter... for good reasons.
- final Field automatonField = AutomatonQuery.class.getDeclaredField("automaton");
- automatonField.setAccessible(true);
- Automaton automaton = (Automaton) automatonField.get(q);
- String prefix = SpecialOperations.getCommonPrefix(automaton);
+ }
+ Automaton automaton = ((AutomatonQuery) q).getAutomaton();
+ String prefix = Operations.getCommonPrefix(Operations.determinize(automaton));
return prefix.length() > 0 && prefix.charAt(0) == '\u0001';
}