You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by is...@apache.org on 2017/02/12 13:18:29 UTC
[05/18] lucene-solr:jira/solr-5944: Updating branch by merging latest
changes from master
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fc5a9f0/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java b/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java
index b9e1e4a..a34febc 100644
--- a/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java
+++ b/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java
@@ -55,6 +55,7 @@ import org.apache.solr.servlet.SolrRequestParsers.MultipartRequestParser;
import org.apache.solr.servlet.SolrRequestParsers.FormDataRequestParser;
import org.apache.solr.servlet.SolrRequestParsers.RawRequestParser;
import org.apache.solr.servlet.SolrRequestParsers.StandardRequestParser;
+import org.easymock.EasyMock;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -481,6 +482,7 @@ public class SolrRequestParserTest extends SolrTestCaseJ4 {
// we dont pass a content-length to let the security mechanism limit it:
expect(request.getQueryString()).andReturn("foo=1&bar=2").anyTimes();
expect(request.getInputStream()).andReturn(new ByteServletInputStream(body.getBytes(StandardCharsets.US_ASCII)));
+ expect(request.getAttribute(EasyMock.anyObject(String.class))).andReturn(null).anyTimes();
replay(request);
SolrRequestParsers parsers = new SolrRequestParsers(h.getCore().getSolrConfig());
@@ -518,7 +520,7 @@ public class SolrRequestParserTest extends SolrTestCaseJ4 {
expect(request.getRequestURI()).andReturn(uri).anyTimes();
expect(request.getContentType()).andReturn(contentType).anyTimes();
expect(request.getContentLength()).andReturn(contentLength).anyTimes();
- expect(request.getAttribute(SolrRequestParsers.REQUEST_TIMER_SERVLET_ATTRIBUTE)).andReturn(null).anyTimes();
+ expect(request.getAttribute(EasyMock.anyObject(String.class))).andReturn(null).anyTimes();
return request;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fc5a9f0/solr/core/src/test/org/apache/solr/spelling/suggest/RandomTestDictionaryFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/spelling/suggest/RandomTestDictionaryFactory.java b/solr/core/src/test/org/apache/solr/spelling/suggest/RandomTestDictionaryFactory.java
new file mode 100644
index 0000000..6e3172d
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/spelling/suggest/RandomTestDictionaryFactory.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.spelling.suggest;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+
+import org.apache.lucene.search.spell.Dictionary;
+import org.apache.lucene.search.suggest.InputIterator;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefIterator;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Factory for a dictionary with an iterator over bounded-length random strings (with fixed
+ * weight of 1 and null payloads) that only operates when RandomDictionary.enabledSysProp
+ * is set; this will be true from the time a RandomDictionary has been constructed until
+ * its enabledSysProp has been cleared.
+ */
+public class RandomTestDictionaryFactory extends DictionaryFactory {
+ public static final String RAND_DICT_MAX_ITEMS = "randDictMaxItems";
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private static final long DEFAULT_MAX_ITEMS = 100_000_000;
+
+ @Override
+ public RandomTestDictionary create(SolrCore core, SolrIndexSearcher searcher) {
+ if(params == null) {
+ // should not happen; implies setParams was not called
+ throw new IllegalStateException("Value of params not set");
+ }
+ String name = (String)params.get(CommonParams.NAME);
+ if (name == null) { // Shouldn't happen since this is the component name
+ throw new IllegalArgumentException(CommonParams.NAME + " is a mandatory parameter");
+ }
+ long maxItems = DEFAULT_MAX_ITEMS;
+ Object specifiedMaxItems = params.get(RAND_DICT_MAX_ITEMS);
+ if (specifiedMaxItems != null) {
+ maxItems = Long.parseLong(specifiedMaxItems.toString());
+ }
+ return new RandomTestDictionary(name, maxItems);
+ }
+
+ public static class RandomTestDictionary implements Dictionary {
+ private static final String SYS_PROP_PREFIX = RandomTestDictionary.class.getName() + ".enabled.";
+ private final String enabledSysProp; // Clear this property to stop the input iterator
+ private final long maxItems;
+ private long emittedItems = 0L;
+
+ RandomTestDictionary(String name, long maxItems) {
+ enabledSysProp = getEnabledSysProp(name);
+ this.maxItems = maxItems;
+ synchronized (RandomTestDictionary.class) {
+ if (System.getProperty(enabledSysProp) != null) {
+ throw new RuntimeException("System property '" + enabledSysProp + "' is already in use.");
+ }
+ System.setProperty(enabledSysProp, "true");
+ }
+ }
+
+ public static String getEnabledSysProp(String suggesterName) {
+ return SYS_PROP_PREFIX + suggesterName;
+ }
+
+ @Override
+ public InputIterator getEntryIterator() throws IOException {
+ return new InputIterator.InputIteratorWrapper(new RandomByteRefIterator());
+ }
+
+ private class RandomByteRefIterator implements BytesRefIterator {
+ private static final int MAX_LENGTH = 100;
+
+ @Override
+ public BytesRef next() throws IOException {
+ BytesRef next = null;
+ if (System.getProperty(enabledSysProp) != null) {
+ if (emittedItems < maxItems) {
+ ++emittedItems;
+ next = new BytesRef(TestUtil.randomUnicodeString(LuceneTestCase.random(), MAX_LENGTH));
+ if (emittedItems % 1000 == 0) {
+ log.info(enabledSysProp + " emitted " + emittedItems + " items.");
+ }
+ } else {
+ log.info(enabledSysProp + " disabled after emitting " + emittedItems + " items.");
+ System.clearProperty(enabledSysProp); // disable once maxItems has been reached
+ emittedItems = 0L;
+ }
+ } else {
+ log.warn(enabledSysProp + " invoked when disabled");
+ emittedItems = 0L;
+ }
+ return next;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fc5a9f0/solr/core/src/test/org/apache/solr/store/blockcache/BlockCacheTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/store/blockcache/BlockCacheTest.java b/solr/core/src/test/org/apache/solr/store/blockcache/BlockCacheTest.java
index 8e2edfe..795518d 100644
--- a/solr/core/src/test/org/apache/solr/store/blockcache/BlockCacheTest.java
+++ b/solr/core/src/test/org/apache/solr/store/blockcache/BlockCacheTest.java
@@ -18,6 +18,7 @@ package org.apache.solr.store.blockcache;
import java.util.Arrays;
import java.util.Random;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.util.LuceneTestCase;
@@ -106,4 +107,137 @@ public class BlockCacheTest extends LuceneTestCase {
random.nextBytes(buf);
return buf;
}
+
+ // given a position, return the appropriate byte.
+ // always returns the same thing so we don't actually have to store the bytes redundantly to check them.
+ private static byte getByte(long pos) {
+ // knuth multiplicative hash method, then take top 8 bits
+ return (byte) ((((int)pos) * (int)(2654435761L)) >> 24);
+
+ // just the lower bits of the block number, to aid in debugging...
+ // return (byte)(pos>>10);
+ }
+
+ @Test
+ @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-10121")
+ public void testBlockCacheConcurrent() throws Exception {
+ Random rnd = random();
+
+ /***
+ final int blocksInTest = 256;
+ final int blockSize = 1024;
+ final int slabSize = blockSize * 128;
+ final long totalMemory = 2 * slabSize;
+ ***/
+
+ final int blocksInTest = 16384; // pick something that won't fit in memory, but is small enough to cause a medium hit rate. 16MB of blocks is double the total memory size of the cache.
+ final int blockSize = 1024;
+ final int slabSize = blockSize * 4096;
+ final long totalMemory = 2 * slabSize; // should give us 2 slabs (8MB)
+
+ final int nThreads=2;
+ final int nReads=1000000;
+ final int readsPerThread=nReads/nThreads;
+ final int readLastBlockOdds=10; // odds (1 in N) of the next block operation being on the same block as the previous operation... helps flush concurrency issues
+
+ final BlockCache blockCache = new BlockCache(new Metrics(), true, totalMemory, slabSize, blockSize);
+
+ final AtomicBoolean failed = new AtomicBoolean(false);
+ final AtomicLong hitsInCache = new AtomicLong();
+ final AtomicLong missesInCache = new AtomicLong();
+ final AtomicLong storeFails = new AtomicLong();
+ final AtomicLong lastBlock = new AtomicLong();
+
+ final int file = 0;
+
+
+ Thread[] threads = new Thread[nThreads];
+ for (int i=0; i<threads.length; i++) {
+ final int threadnum = i;
+ final long seed = rnd.nextLong();
+
+ threads[i] = new Thread() {
+ Random r;
+ BlockCacheKey blockCacheKey = new BlockCacheKey();
+ byte[] buffer = new byte[blockSize];
+
+ @Override
+ public void run() {
+ try {
+ r = new Random(seed);
+ blockCacheKey = new BlockCacheKey();
+ blockCacheKey.setFile(file);
+ blockCacheKey.setPath("/foo.txt");
+
+ test(readsPerThread);
+
+ } catch (Throwable e) {
+ failed.set(true);
+ e.printStackTrace();
+ }
+ }
+
+ public void test(int iter) {
+ for (int i=0; i<iter; i++) {
+ test();
+ }
+ }
+
+ public void test() {
+ long block = r.nextInt(blocksInTest);
+ if (r.nextInt(readLastBlockOdds) == 0) block = lastBlock.get(); // some percent of the time, try to read the last block another thread was just reading/writing
+ lastBlock.set(block);
+
+
+ int blockOffset = r.nextInt(blockSize);
+ long globalOffset = block * blockSize + blockOffset;
+ int len = r.nextInt(blockSize - blockOffset) + 1; // TODO: bias toward smaller reads?
+
+ blockCacheKey.setBlock(block);
+
+ if (blockCache.fetch(blockCacheKey, buffer, blockOffset, 0, len)) {
+ hitsInCache.incrementAndGet();
+ // validate returned bytes
+ for (int i = 0; i < len; i++) {
+ long globalPos = globalOffset + i;
+ if (buffer[i] != getByte(globalPos)) {
+ System.out.println("ERROR: read was " + "block=" + block + " blockOffset=" + blockOffset + " len=" + len + " globalPos=" + globalPos + " localReadOffset=" + i + " got=" + buffer[i] + " expected=" + getByte(globalPos));
+ failed.set(true);
+ }
+ }
+ } else {
+ missesInCache.incrementAndGet();
+
+ // OK, we should "get" the data and then cache the block
+ for (int i = 0; i < blockSize; i++) {
+ buffer[i] = getByte(block * blockSize + i);
+ }
+ boolean cached = blockCache.store(blockCacheKey, 0, buffer, 0, blockSize);
+ if (!cached) {
+ storeFails.incrementAndGet();
+ }
+ }
+
+ }
+
+ };
+ }
+
+
+ for (Thread thread : threads) {
+ thread.start();
+ }
+
+ for (Thread thread : threads) {
+ thread.join();
+ }
+
+ System.out.println("# of Elements = " + blockCache.getSize());
+ System.out.println("Cache Hits = " + hitsInCache.get());
+ System.out.println("Cache Misses = " + missesInCache.get());
+ System.out.println("Cache Store Fails = " + storeFails.get());
+
+ assertFalse( failed.get() );
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fc5a9f0/solr/core/src/test/org/apache/solr/update/SoftAutoCommitTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/SoftAutoCommitTest.java b/solr/core/src/test/org/apache/solr/update/SoftAutoCommitTest.java
index c9c9691..261a44b 100644
--- a/solr/core/src/test/org/apache/solr/update/SoftAutoCommitTest.java
+++ b/solr/core/src/test/org/apache/solr/update/SoftAutoCommitTest.java
@@ -17,9 +17,9 @@
package org.apache.solr.update;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import static java.util.concurrent.TimeUnit.SECONDS;
import static org.junit.Assert.assertEquals;
+import java.lang.invoke.MethodHandles;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
@@ -33,6 +33,8 @@ import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.util.AbstractSolrTestCase;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Test auto commit functionality in a way that doesn't suck.
@@ -55,7 +57,7 @@ import org.junit.BeforeClass;
*/
@Slow
public class SoftAutoCommitTest extends AbstractSolrTestCase {
-
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@BeforeClass
public static void beforeClass() throws Exception {
@@ -78,28 +80,34 @@ public class SoftAutoCommitTest extends AbstractSolrTestCase {
core.registerNewSearcherListener(monitor);
updater.registerSoftCommitCallback(monitor);
updater.registerCommitCallback(monitor);
+
+ // isolate searcher getting ready from this test
+ monitor.searcher.poll(5000, MILLISECONDS);
}
@Override
public void setUp() throws Exception {
super.setUp();
- // reset stats
- h.getCoreContainer().reload("collection1");
+
}
public void testSoftAndHardCommitMaxTimeMixedAdds() throws Exception {
-
final int softCommitWaitMillis = 500;
final int hardCommitWaitMillis = 1200;
CommitTracker hardTracker = updater.commitTracker;
CommitTracker softTracker = updater.softCommitTracker;
+ int startingHardCommits = hardTracker.getCommitCount();
+ int startingSoftCommits = softTracker.getCommitCount();
+
softTracker.setTimeUpperBound(softCommitWaitMillis);
softTracker.setDocsUpperBound(-1);
hardTracker.setTimeUpperBound(hardCommitWaitMillis);
hardTracker.setDocsUpperBound(-1);
-
+ // simplify whats going on by only having soft auto commits trigger new searchers
+ hardTracker.setOpenSearcher(false);
+
// Add a single document
long add529 = System.nanoTime();
assertU(adoc("id", "529", "subject", "the doc we care about in this test"));
@@ -107,21 +115,24 @@ public class SoftAutoCommitTest extends AbstractSolrTestCase {
monitor.assertSaneOffers();
// Wait for the soft commit with some fudge
- Long soft529 = monitor.soft.poll(softCommitWaitMillis * 3, MILLISECONDS);
+ Long soft529 = monitor.soft.poll(softCommitWaitMillis * 500, MILLISECONDS);
assertNotNull("soft529 wasn't fast enough", soft529);
monitor.assertSaneOffers();
+
+ // wait for the hard commit
+ Long hard529 = monitor.hard.poll(hardCommitWaitMillis * 5, MILLISECONDS);
+ assertNotNull("hard529 wasn't fast enough", hard529);
+
// check for the searcher, should have happened right after soft commit
- Long searcher529 = monitor.searcher.poll(softCommitWaitMillis * 3, MILLISECONDS);
+ Long searcher529 = monitor.searcher.poll(5000, MILLISECONDS);
assertNotNull("searcher529 wasn't fast enough", searcher529);
monitor.assertSaneOffers();
// toss in another doc, shouldn't affect first hard commit time we poll
assertU(adoc("id", "530", "subject", "just for noise/activity"));
- // wait for the hard commit
- Long hard529 = monitor.hard.poll(hardCommitWaitMillis * 5, MILLISECONDS);
- assertNotNull("hard529 wasn't fast enough", hard529);
+
monitor.assertSaneOffers();
final long soft529Ms = TimeUnit.MILLISECONDS.convert(soft529 - add529, TimeUnit.NANOSECONDS);
@@ -148,17 +159,17 @@ public class SoftAutoCommitTest extends AbstractSolrTestCase {
monitor.assertSaneOffers();
// there may have been (or will be) a second hard commit for 530
- Long hard530 = monitor.hard.poll(hardCommitWaitMillis, MILLISECONDS);
+ Long hard530 = monitor.hard.poll(hardCommitWaitMillis * 5, MILLISECONDS);
assertEquals("Tracker reports too many hard commits",
(null == hard530 ? 1 : 2),
- hardTracker.getCommitCount());
+ hardTracker.getCommitCount() - startingHardCommits);
// there may have been a second soft commit for 530,
// but if so it must have already happend
Long soft530 = monitor.soft.poll(0, MILLISECONDS);
if (null != soft530) {
assertEquals("Tracker reports too many soft commits",
- 2, softTracker.getCommitCount());
+ 2, softTracker.getCommitCount() - startingSoftCommits);
if (null != hard530) {
assertTrue("soft530 after hard530: " +
soft530 + " !<= " + hard530,
@@ -170,7 +181,7 @@ public class SoftAutoCommitTest extends AbstractSolrTestCase {
}
} else {
assertEquals("Tracker reports too many soft commits",
- 1, softTracker.getCommitCount());
+ 1, softTracker.getCommitCount() - startingSoftCommits);
}
if (null != soft530 || null != hard530) {
@@ -178,16 +189,19 @@ public class SoftAutoCommitTest extends AbstractSolrTestCase {
monitor.searcher.poll(0, MILLISECONDS));
}
- monitor.assertSaneOffers();
+ // clear commits
+ monitor.hard.clear();
+ monitor.soft.clear();
- // wait a bit, w/o other action we definitely shouldn't see any
+ // wait a bit, w/o other action we shouldn't see any
// new hard/soft commits
assertNull("Got a hard commit we weren't expecting",
- monitor.hard.poll(2, SECONDS));
+ monitor.hard.poll(1000, MILLISECONDS));
assertNull("Got a soft commit we weren't expecting",
monitor.soft.poll(0, MILLISECONDS));
monitor.assertSaneOffers();
+ monitor.searcher.clear();
}
public void testSoftAndHardCommitMaxTimeDelete() throws Exception {
@@ -198,10 +212,16 @@ public class SoftAutoCommitTest extends AbstractSolrTestCase {
CommitTracker hardTracker = updater.commitTracker;
CommitTracker softTracker = updater.softCommitTracker;
+ int startingHardCommits = hardTracker.getCommitCount();
+ int startingSoftCommits = softTracker.getCommitCount();
+
softTracker.setTimeUpperBound(softCommitWaitMillis);
softTracker.setDocsUpperBound(-1);
hardTracker.setTimeUpperBound(hardCommitWaitMillis);
hardTracker.setDocsUpperBound(-1);
+ // we don't want to overlap soft and hard opening searchers - this now blocks commits and we
+ // are looking for prompt timings
+ hardTracker.setOpenSearcher(false);
// add a doc and force a commit
assertU(adoc("id", "529", "subject", "the doc we care about in this test"));
@@ -238,7 +258,7 @@ public class SoftAutoCommitTest extends AbstractSolrTestCase {
monitor.assertSaneOffers();
// Wait for the soft commit with some fudge
- soft529 = monitor.soft.poll(softCommitWaitMillis * 3, MILLISECONDS);
+ soft529 = monitor.soft.poll(softCommitWaitMillis * 5, MILLISECONDS);
assertNotNull("soft529 wasn't fast enough", soft529);
monitor.assertSaneOffers();
@@ -251,7 +271,7 @@ public class SoftAutoCommitTest extends AbstractSolrTestCase {
assertU(adoc("id", "550", "subject", "just for noise/activity"));
// wait for the hard commit
- hard529 = monitor.hard.poll(hardCommitWaitMillis * 3, MILLISECONDS);
+ hard529 = monitor.hard.poll(hardCommitWaitMillis * 5, MILLISECONDS);
assertNotNull("hard529 wasn't fast enough", hard529);
monitor.assertSaneOffers();
@@ -276,18 +296,27 @@ public class SoftAutoCommitTest extends AbstractSolrTestCase {
searcher529 + " !<= " + hard529,
searcher529 <= hard529);
+ // ensure we wait for the last searcher we triggered with 550
+ monitor.searcher.poll(5000, MILLISECONDS);
+
+ // ensure we wait for the commits on 550
+ monitor.hard.poll(5000, MILLISECONDS);
+ monitor.soft.poll(5000, MILLISECONDS);
+
// clear commits
monitor.hard.clear();
monitor.soft.clear();
- // wait a bit, w/o other action we definitely shouldn't see any
+ // wait a bit, w/o other action we shouldn't see any
// new hard/soft commits
assertNull("Got a hard commit we weren't expecting",
- monitor.hard.poll(2, SECONDS));
+ monitor.hard.poll(1000, MILLISECONDS));
assertNull("Got a soft commit we weren't expecting",
- monitor.soft.poll(0, MILLISECONDS));
+ monitor.soft.poll(0, MILLISECONDS));
monitor.assertSaneOffers();
+
+ monitor.searcher.clear();
}
public void testSoftAndHardCommitMaxTimeRapidAdds() throws Exception {
@@ -302,32 +331,43 @@ public class SoftAutoCommitTest extends AbstractSolrTestCase {
softTracker.setDocsUpperBound(-1);
hardTracker.setTimeUpperBound(hardCommitWaitMillis);
hardTracker.setDocsUpperBound(-1);
+ // we don't want to overlap soft and hard opening searchers - this now blocks commits and we
+ // are looking for prompt timings
+ hardTracker.setOpenSearcher(false);
// try to add 5 docs really fast
long fast5start = System.nanoTime();
for( int i=0;i<5; i++ ) {
assertU(adoc("id", ""+500 + i, "subject", "five fast docs"));
}
- long fast5end = System.nanoTime() - TimeUnit.NANOSECONDS.convert(200, TimeUnit.MILLISECONDS); // minus a tad of slop
+ long fast5end = System.nanoTime() - TimeUnit.NANOSECONDS.convert(300, TimeUnit.MILLISECONDS); // minus a tad of slop
long fast5time = 1 + TimeUnit.MILLISECONDS.convert(fast5end - fast5start, TimeUnit.NANOSECONDS);
// total time for all 5 adds determines the number of soft to expect
long expectedSoft = (long)Math.ceil((double) fast5time / softCommitWaitMillis);
long expectedHard = (long)Math.ceil((double) fast5time / hardCommitWaitMillis);
+
+ expectedSoft = Math.max(1, expectedSoft);
+ expectedHard = Math.max(1, expectedHard);
// note: counting from 1 for multiplication
for (int i = 1; i <= expectedSoft; i++) {
- // Wait for the soft commit with some fudge
+ // Wait for the soft commit with plenty of fudge to survive nasty envs
Long soft = monitor.soft.poll(softCommitWaitMillis * 2, MILLISECONDS);
- assertNotNull(i + ": soft wasn't fast enough", soft);
- monitor.assertSaneOffers();
-
- // have to assume none of the docs were added until
- // very end of the add window
- long softMs = TimeUnit.MILLISECONDS.convert(soft - fast5end, TimeUnit.NANOSECONDS);
- assertTrue(i + ": soft occurred too fast: " +
- softMs + " < (" + softCommitWaitMillis + " * " + i + ")",
- softMs >= (softCommitWaitMillis * i));
+ if (soft != null || i == 1) {
+ assertNotNull(i + ": soft wasn't fast enough", soft);
+ monitor.assertSaneOffers();
+
+ // have to assume none of the docs were added until
+ // very end of the add window
+ long softMs = TimeUnit.MILLISECONDS.convert(soft - fast5end, TimeUnit.NANOSECONDS);
+ assertTrue(i + ": soft occurred too fast: " +
+ softMs + " < (" + softCommitWaitMillis + " * " + i + ")",
+ softMs >= (softCommitWaitMillis * i));
+ } else {
+ // we may have guessed wrong and there were fewer commits
+ assertNull("Got a soft commit we weren't expecting", monitor.soft.poll(2000, MILLISECONDS));
+ }
}
// note: counting from 1 for multiplication
@@ -345,7 +385,24 @@ public class SoftAutoCommitTest extends AbstractSolrTestCase {
hardMs + " < (" + hardCommitWaitMillis + " * " + i + ")",
hardMs >= (hardCommitWaitMillis * i));
}
+
+ // we are only guessing how many commits we may see, allow one extra of each
+ monitor.soft.poll(softCommitWaitMillis + 200, MILLISECONDS);
+ monitor.hard.poll(hardCommitWaitMillis + 200, MILLISECONDS);
+ // clear commits
+ monitor.hard.clear();
+ monitor.soft.clear();
+
+ // wait a bit, w/o other action we shouldn't see any
+ // new hard/soft commits
+ assertNull("Got a hard commit we weren't expecting",
+ monitor.hard.poll(1000, MILLISECONDS));
+ assertNull("Got a soft commit we weren't expecting",
+ monitor.soft.poll(0, MILLISECONDS));
+
+ monitor.assertSaneOffers();
+
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fc5a9f0/solr/core/src/test/org/apache/solr/update/SolrIndexMetricsTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/SolrIndexMetricsTest.java b/solr/core/src/test/org/apache/solr/update/SolrIndexMetricsTest.java
index 4f5ea69..850ce03 100644
--- a/solr/core/src/test/org/apache/solr/update/SolrIndexMetricsTest.java
+++ b/solr/core/src/test/org/apache/solr/update/SolrIndexMetricsTest.java
@@ -63,7 +63,7 @@ public class SolrIndexMetricsTest extends SolrTestCaseJ4 {
assertTrue(metrics.entrySet().stream().filter(e -> e.getKey().startsWith("INDEX")).count() >= 12);
// this is variable, depending on the codec and the number of created files
- assertTrue(metrics.entrySet().stream().filter(e -> e.getKey().startsWith("DIRECTORY")).count() > 50);
+ assertTrue(metrics.entrySet().stream().filter(e -> e.getKey().startsWith("DIRECTORY")).count() > 20);
// check basic index meters
Timer timer = (Timer)metrics.get("INDEX.merge.minor");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fc5a9f0/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java b/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java
index 74360e3..5386721 100644
--- a/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java
+++ b/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java
@@ -181,7 +181,7 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase {
// The following should work: full update to doc 0, in-place update for doc 0, delete doc 0
private void outOfOrderDBQsTest() throws Exception {
- del("*:*");
+ clearIndex();
commit();
buildRandomIndex(0);
@@ -241,12 +241,12 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase {
}
log.info("outOfOrderDeleteUpdatesIndividualReplicaTest: This test passed fine...");
- del("*:*");
+ clearIndex();
commit();
}
private void docValuesUpdateTest() throws Exception {
- del("*:*");
+ clearIndex();
commit();
// number of docs we're testing (0 <= id), index may contain additional random docs (id < 0)
@@ -397,7 +397,7 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase {
private void ensureRtgWorksWithPartialUpdatesTest() throws Exception {
- del("*:*");
+ clearIndex();
commit();
float inplace_updatable_float = 1;
@@ -496,7 +496,7 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase {
private void outOfOrderUpdatesIndividualReplicaTest() throws Exception {
- del("*:*");
+ clearIndex();
commit();
buildRandomIndex(0);
@@ -561,14 +561,14 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase {
}
log.info("outOfOrderUpdatesIndividualReplicaTest: This test passed fine...");
- del("*:*");
+ clearIndex();
commit();
}
// The following should work: full update to doc 0, in-place update for doc 0, delete doc 0
private void outOfOrderDeleteUpdatesIndividualReplicaTest() throws Exception {
- del("*:*");
+ clearIndex();
commit();
buildRandomIndex(0);
@@ -627,7 +627,7 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase {
}
log.info("outOfOrderDeleteUpdatesIndividualReplicaTest: This test passed fine...");
- del("*:*");
+ clearIndex();
commit();
}
@@ -641,7 +641,7 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase {
DV(id=x, val=5, ver=3)
*/
private void reorderedDBQsWithInPlaceUpdatesShouldNotThrowReplicaInLIRTest() throws Exception {
- del("*:*");
+ clearIndex();
commit();
buildRandomIndex(0);
@@ -728,12 +728,12 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase {
}
log.info("reorderedDBQsWithInPlaceUpdatesShouldNotThrowReplicaInLIRTest: This test passed fine...");
- del("*:*");
+ clearIndex();
commit();
}
private void delayedReorderingFetchesMissingUpdateFromLeaderTest() throws Exception {
- del("*:*");
+ clearIndex();
commit();
float inplace_updatable_float = 1F;
@@ -795,7 +795,7 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase {
// This is to ensure that the fetch missing update from leader doesn't bomb out if the
// document has been deleted on the leader later on
{
- del("*:*");
+ clearIndex();
commit();
shardToJetty.get(SHARD1).get(1).jetty.getDebugFilter().unsetDelay();
@@ -1038,7 +1038,7 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase {
* dbq("inp:14",version=4)
*/
private void testDBQUsingUpdatedFieldFromDroppedUpdate() throws Exception {
- del("*:*");
+ clearIndex();
commit();
float inplace_updatable_float = 1F;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fc5a9f0/solr/core/src/test/org/apache/solr/util/JsonValidatorTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/util/JsonValidatorTest.java b/solr/core/src/test/org/apache/solr/util/JsonValidatorTest.java
new file mode 100644
index 0000000..4492586
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/util/JsonValidatorTest.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.util;
+
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.api.ApiBag;
+import org.apache.solr.common.util.ValidatingJsonMap;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.Utils;
+
+import static org.apache.solr.common.util.ValidatingJsonMap.NOT_NULL;
+import static org.apache.solr.common.util.Utils.toJSONString;
+
+public class JsonValidatorTest extends SolrTestCaseJ4 {
+
+ public void testSchema() {
+ checkSchema("collections.Commands");
+ checkSchema("collections.collection.Commands");
+ checkSchema("collections.collection.shards.Commands");
+ checkSchema("collections.collection.shards.shard.Commands");
+ checkSchema("cores.Commands");
+ checkSchema("cores.core.Commands");
+ checkSchema("node.Commands");
+ checkSchema("cluster.security.BasicAuth.Commands");
+ checkSchema("cluster.security.RuleBasedAuthorization");
+ checkSchema("core.config.Commands");
+ checkSchema("core.SchemaEdit");
+ checkSchema("cluster.configs.Commands");
+ }
+
+
+ public void testSchemaValidation() {
+ ValidatingJsonMap spec = ApiBag.getSpec("collections.Commands").getSpec();
+ Map createSchema = spec.getMap("commands", NOT_NULL).getMap("create-alias", NOT_NULL);
+ JsonSchemaValidator validator = new JsonSchemaValidator(createSchema);
+ List<String> errs = validator.validateJson(Utils.fromJSONString("{name : x, collections: [ c1 , c2]}"));
+ assertNull(toJSONString(errs), errs);
+ errs = validator.validateJson(Utils.fromJSONString("{name : x, collections: c1 }"));
+ assertNull(toJSONString(errs), errs);
+ errs = validator.validateJson(Utils.fromJSONString("{name : x, x:y, collections: [ c1 , c2]}"));
+ assertNotNull(toJSONString(errs), errs);
+ assertTrue(toJSONString(errs), errs.get(0).contains("Unknown"));
+ errs = validator.validateJson(Utils.fromJSONString("{name : 123, collections: c1 }"));
+ assertNotNull(toJSONString(errs), errs);
+ assertTrue(toJSONString(errs), errs.get(0).contains("Expected type"));
+ errs = validator.validateJson(Utils.fromJSONString("{x:y, collections: [ c1 , c2]}"));
+ assertEquals(toJSONString(errs), 2, errs.size());
+ assertTrue(toJSONString(errs), StrUtils.join(errs, '|').contains("Missing field"));
+ assertTrue(toJSONString(errs), StrUtils.join(errs, '|').contains("Unknown"));
+ errs = validator.validateJson(Utils.fromJSONString("{name : x, collections: [ 1 , 2]}"));
+ assertFalse(toJSONString(errs), errs.isEmpty());
+ assertTrue(toJSONString(errs), errs.get(0).contains("Expected elements of type"));
+ validator = new JsonSchemaValidator("{" +
+ " type:object," +
+ " properties: {" +
+ " age : {type: number}," +
+ " adult : {type: boolean}," +
+ " name: {type: string}}}");
+ errs = validator.validateJson(Utils.fromJSONString("{name:x, age:21, adult:true}"));
+ assertNull(errs);
+ errs = validator.validateJson(Utils.fromJSONString("{name:x, age:'21', adult:'true'}"));
+ assertNull(errs);
+
+ errs = validator.validateJson(Utils.fromJSONString("{name:x, age:'x21', adult:'true'}"));
+ assertEquals(1, errs.size());
+ try {
+ validator = new JsonSchemaValidator("{" +
+ " type:object," +
+ " properties: {" +
+ " age : {type: int}," +
+ " adult : {type: Boolean}," +
+ " name: {type: string}}}");
+ fail("should have failed");
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("Unknown type"));
+ }
+
+ try {
+ new JsonSchemaValidator("{" +
+ " type:object," +
+ " x : y," +
+ " properties: {" +
+ " age : {type: number}," +
+ " adult : {type: boolean}," +
+ " name: {type: string}}}");
+ fail("should have failed");
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("Unknown key"));
+ }
+ try {
+ new JsonSchemaValidator("{" +
+ " type:object," +
+ " propertes: {" +
+ " age : {type: number}," +
+ " adult : {type: boolean}," +
+ " name: {type: string}}}");
+ fail("should have failed");
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("Unknown key : propertes"));
+ }
+
+ validator = new JsonSchemaValidator("{" +
+ " type:object," +
+ " properties: {" +
+ " age : {type: number}," +
+ " sex: {type: string, enum:[M, F]}," +
+ " adult : {type: boolean}," +
+ " name: {type: string}}}");
+ errs = validator.validateJson(Utils.fromJSONString("{name: 'Joe Average' , sex:M}"));
+ assertNull("errs are " + errs, errs);
+ errs = validator.validateJson(Utils.fromJSONString("{name: 'Joe Average' , sex:m}"));
+ assertEquals(1, errs.size());
+ assertTrue(errs.get(0).contains("value of enum"));
+
+ String schema = "{\n" +
+ " 'type': 'object',\n" +
+ " 'properties': {\n" +
+ " 'links': {\n" +
+ " 'type': 'array',\n" +
+ " 'items':{" +
+ " 'type': 'object',\n" +
+ " 'properties': {\n" +
+ " 'rel': {\n" +
+ " 'type': 'string'\n" +
+ " },\n" +
+ " 'href': {\n" +
+ " 'type': 'string'\n" +
+ " }\n" +
+ " }\n" +
+ " }\n" +
+ " }\n" +
+ "\n" +
+ " }\n" +
+ "}";
+ validator = new JsonSchemaValidator(schema);
+ validator.validateJson(Utils.fromJSONString("{\n" +
+ " 'links': [\n" +
+ " {\n" +
+ " 'rel': 'x',\n" +
+ " 'href': 'x'\n" +
+ " },\n" +
+ " {\n" +
+ " 'rel': 'x',\n" +
+ " 'href': 'x'\n" +
+ " },\n" +
+ " {\n" +
+ " 'rel': 'x',\n" +
+ " 'href': 'x'\n" +
+ " }\n" +
+ " ]\n" +
+ "}"));
+
+
+
+
+ }
+
+ private void checkSchema(String name) {
+ ValidatingJsonMap spec = ApiBag.getSpec(name).getSpec();
+ Map commands = (Map) spec.get("commands");
+ for (Object o : commands.entrySet()) {
+ Map.Entry cmd = (Map.Entry) o;
+ try {
+ JsonSchemaValidator validator = new JsonSchemaValidator((Map) cmd.getValue());
+ } catch (Exception e) {
+ throw new RuntimeException("Error in command " + cmd.getKey() + " in schema " + name, e);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fc5a9f0/solr/core/src/test/org/apache/solr/util/TestObjectReleaseTracker.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/util/TestObjectReleaseTracker.java b/solr/core/src/test/org/apache/solr/util/TestObjectReleaseTracker.java
index f7e6943..e7a7637 100644
--- a/solr/core/src/test/org/apache/solr/util/TestObjectReleaseTracker.java
+++ b/solr/core/src/test/org/apache/solr/util/TestObjectReleaseTracker.java
@@ -18,6 +18,7 @@ package org.apache.solr.util;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestRuleLimitSysouts.Limit;
+import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.util.ObjectReleaseTracker;
import org.junit.Test;
@@ -29,12 +30,12 @@ public class TestObjectReleaseTracker extends LuceneTestCase {
public void testObjectReleaseTracker() {
ObjectReleaseTracker.track(new Object());
ObjectReleaseTracker.release(new Object());
- assertNotNull(ObjectReleaseTracker.clearObjectTrackerAndCheckEmpty(1));
- assertNull(ObjectReleaseTracker.clearObjectTrackerAndCheckEmpty(1));
+ assertNotNull(SolrTestCaseJ4.clearObjectTrackerAndCheckEmpty(1));
+ assertNull(SolrTestCaseJ4.clearObjectTrackerAndCheckEmpty(1));
Object obj = new Object();
ObjectReleaseTracker.track(obj);
ObjectReleaseTracker.release(obj);
- assertNull(ObjectReleaseTracker.clearObjectTrackerAndCheckEmpty(1));
+ assertNull(SolrTestCaseJ4.clearObjectTrackerAndCheckEmpty(1));
Object obj1 = new Object();
ObjectReleaseTracker.track(obj1);
@@ -46,7 +47,7 @@ public class TestObjectReleaseTracker extends LuceneTestCase {
ObjectReleaseTracker.release(obj1);
ObjectReleaseTracker.release(obj2);
ObjectReleaseTracker.release(obj3);
- assertNull(ObjectReleaseTracker.clearObjectTrackerAndCheckEmpty(1));
+ assertNull(SolrTestCaseJ4.clearObjectTrackerAndCheckEmpty(1));
ObjectReleaseTracker.track(obj1);
ObjectReleaseTracker.track(obj2);
@@ -55,7 +56,7 @@ public class TestObjectReleaseTracker extends LuceneTestCase {
ObjectReleaseTracker.release(obj1);
ObjectReleaseTracker.release(obj2);
// ObjectReleaseTracker.release(obj3);
- assertNotNull(ObjectReleaseTracker.clearObjectTrackerAndCheckEmpty(1));
- assertNull(ObjectReleaseTracker.clearObjectTrackerAndCheckEmpty(1));
+ assertNotNull(SolrTestCaseJ4.clearObjectTrackerAndCheckEmpty(1));
+ assertNull(SolrTestCaseJ4.clearObjectTrackerAndCheckEmpty(1));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fc5a9f0/solr/example/example-DIH/build.xml
----------------------------------------------------------------------
diff --git a/solr/example/example-DIH/build.xml b/solr/example/example-DIH/build.xml
index fb82edf..77abc2d 100644
--- a/solr/example/example-DIH/build.xml
+++ b/solr/example/example-DIH/build.xml
@@ -24,6 +24,7 @@
<!-- example tests are currently elsewhere -->
<target name="test"/>
+ <target name="test-nocompile"/>
<!-- this module has no javadocs -->
<target name="javadocs"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fc5a9f0/solr/server/build.xml
----------------------------------------------------------------------
diff --git a/solr/server/build.xml b/solr/server/build.xml
index ae4e650..d3d7af6 100644
--- a/solr/server/build.xml
+++ b/solr/server/build.xml
@@ -22,6 +22,7 @@
<!-- example tests are currently elsewhere -->
<target name="test"/>
+ <target name="test-nocompile"/>
<!-- this module has no javadocs -->
<target name="javadocs"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fc5a9f0/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml b/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
index 69a1519..a9ddb25 100644
--- a/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
+++ b/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
@@ -850,7 +850,7 @@
</requestHandler>
<!-- A request handler that returns indented JSON by default -->
- <requestHandler name="/query" class="solr.SearchHandler">
+ <requestHandler name="/query" class="solr.SearchHandler" registerPath="/,/v2">
<lst name="defaults">
<str name="echoParams">explicit</str>
<str name="wt">json</str>
@@ -958,7 +958,7 @@
</requestHandler>
- <initParams path="/update/**,/query,/select,/tvrh,/elevate,/spell,/browse">
+ <initParams path="/update/**,/query,/select,/tvrh,/elevate,/spell,/browse,update">
<lst name="defaults">
<str name="df">text</str>
</lst>
@@ -966,12 +966,12 @@
<!-- The following are implicitly added
<requestHandler name="/update/json" class="solr.UpdateRequestHandler">
- <lst name="defaults">
+ <lst name="invariants">
<str name="stream.contentType">application/json</str>
</lst>
</requestHandler>
<requestHandler name="/update/csv" class="solr.UpdateRequestHandler">
- <lst name="defaults">
+ <lst name="invariants">
<str name="stream.contentType">application/csv</str>
</lst>
</requestHandler>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fc5a9f0/solr/solrj/src/java/org/apache/solr/client/solrj/SolrRequest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/SolrRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/SolrRequest.java
index 3e31edf..4dbba5b 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/SolrRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/SolrRequest.java
@@ -21,10 +21,14 @@ import org.apache.solr.common.util.ContentStream;
import java.io.IOException;
import java.io.Serializable;
+import java.util.Arrays;
import java.util.Collection;
+import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import static java.util.Collections.unmodifiableSet;
+
/**
*
*
@@ -35,9 +39,16 @@ public abstract class SolrRequest<T extends SolrResponse> implements Serializabl
public enum METHOD {
GET,
POST,
- PUT
+ PUT,
+ DELETE
};
+ public static final Set<String> SUPPORTED_METHODS = unmodifiableSet(new HashSet<>(Arrays.<String>asList(
+ METHOD.GET.toString(),
+ METHOD.POST.toString(),
+ METHOD.PUT.toString(),
+ METHOD.DELETE.toString())));
+
private METHOD method = METHOD.GET;
private String path = null;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fc5a9f0/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java
index 3b69484..d0263c8 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java
@@ -23,6 +23,7 @@ import java.net.ConnectException;
import java.net.SocketException;
import java.nio.file.Path;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -83,7 +84,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
-import static org.apache.solr.common.params.CommonParams.ADMIN_PATHS;
+import static org.apache.solr.common.params.CommonParams.AUTHC_PATH;
+import static org.apache.solr.common.params.CommonParams.AUTHZ_PATH;
+import static org.apache.solr.common.params.CommonParams.COLLECTIONS_HANDLER_PATH;
+import static org.apache.solr.common.params.CommonParams.CONFIGSETS_HANDLER_PATH;
+import static org.apache.solr.common.params.CommonParams.CORES_HANDLER_PATH;
/**
* SolrJ client class to communicate with SolrCloud.
@@ -1036,6 +1041,15 @@ public class CloudSolrClient extends SolrClient {
collection = (reqParams != null) ? reqParams.get("collection", getDefaultCollection()) : getDefaultCollection();
return requestWithRetryOnStaleState(request, 0, collection);
}
+ private static final Set<String> ADMIN_PATHS = new HashSet<>(Arrays.asList(
+ CORES_HANDLER_PATH,
+ COLLECTIONS_HANDLER_PATH,
+ CONFIGSETS_HANDLER_PATH,
+ AUTHC_PATH,
+ AUTHZ_PATH,
+ "/v2/cluster/security/authentication",
+ "/v2/cluster/security/authorization"
+ ));
/**
* As this class doesn't watch external collections on the client side,
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fc5a9f0/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java
index decd5e8..7ee90e1 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java
@@ -22,6 +22,7 @@ import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.zip.GZIPInputStream;
@@ -111,28 +112,46 @@ public class HttpClientUtil {
// cannot be established within x ms. with a
// java.net.SocketTimeoutException: Connection timed out
public static final String PROP_CONNECTION_TIMEOUT = "connTimeout";
-
+
+ /**
+ * A Java system property to select the {@linkplain HttpClientBuilderFactory} used for
+ * configuring the {@linkplain HttpClientBuilder} instance by default.
+ */
+ public static final String SYS_PROP_HTTP_CLIENT_BUILDER_FACTORY = "solr.httpclient.builder.factory";
+
static final DefaultHttpRequestRetryHandler NO_RETRY = new DefaultHttpRequestRetryHandler(
0, false);
private static volatile SolrHttpClientBuilder httpClientBuilder;
-
+
private static SolrHttpClientContextBuilder httpClientRequestContextBuilder = new SolrHttpClientContextBuilder();
-
+
+ private static volatile SchemaRegistryProvider schemaRegistryProvider;
+ private static volatile String cookiePolicy;
+ private static final List<HttpRequestInterceptor> interceptors = Collections.synchronizedList(new ArrayList<HttpRequestInterceptor>());
+
+
static {
resetHttpClientBuilder();
+
+ // Configure the HttpClientBuilder if user has specified the factory type.
+ String factoryClassName = System.getProperty(SYS_PROP_HTTP_CLIENT_BUILDER_FACTORY);
+ if (factoryClassName != null) {
+ logger.debug ("Using " + factoryClassName);
+ try {
+ HttpClientBuilderFactory factory = (HttpClientBuilderFactory)Class.forName(factoryClassName).newInstance();
+ httpClientBuilder = factory.getHttpClientBuilder(Optional.of(SolrHttpClientBuilder.create()));
+ } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
+ throw new RuntimeException("Unable to instantiate Solr HttpClientBuilderFactory", e);
+ }
+ }
}
public static abstract class SchemaRegistryProvider {
/** Must be non-null */
public abstract Registry<ConnectionSocketFactory> getSchemaRegistry();
}
-
- private static volatile SchemaRegistryProvider schemaRegistryProvider;
- private static volatile String cookiePolicy;
- private static final List<HttpRequestInterceptor> interceptors = Collections.synchronizedList(new ArrayList<HttpRequestInterceptor>());
-
private static class DynamicInterceptor implements HttpRequestInterceptor {
@Override
@@ -151,7 +170,7 @@ public class HttpClientUtil {
}
}
-
+
public static void setHttpClientBuilder(SolrHttpClientBuilder newHttpClientBuilder) {
httpClientBuilder = newHttpClientBuilder;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fc5a9f0/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Krb5HttpClientBuilder.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Krb5HttpClientBuilder.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Krb5HttpClientBuilder.java
index 7f3cf29..1bcf96b 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Krb5HttpClientBuilder.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Krb5HttpClientBuilder.java
@@ -138,8 +138,11 @@ public class Krb5HttpClientBuilder implements HttpClientBuilderFactory {
});
HttpClientUtil.addRequestInterceptor(bufferedEntityInterceptor);
}
+ } else {
+ logger.warn("{} is configured without specifying system property '{}'",
+ getClass().getName(), LOGIN_CONFIG_PROP);
}
-
+
return builder;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fc5a9f0/solr/solrj/src/java/org/apache/solr/client/solrj/impl/PreemptiveAuth.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/PreemptiveAuth.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/PreemptiveAuth.java
new file mode 100644
index 0000000..3334d9f
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/PreemptiveAuth.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.impl;
+
+import java.io.IOException;
+
+import org.apache.http.HttpException;
+import org.apache.http.HttpRequest;
+import org.apache.http.HttpRequestInterceptor;
+import org.apache.http.auth.AuthScheme;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.AuthState;
+import org.apache.http.auth.Credentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.protocol.ClientContext;
+import org.apache.http.protocol.HttpContext;
+
+/**
+ * This HTTP request interceptor adds HTTP authentication credentials to every outgoing
+ * request. This implementation is required since Solr client is not capable of performing
+ * non preemptive authentication. By adding the Http authentication credentials to every request,
+ * this interceptor enables "preemptive" authentication.
+ */
+public class PreemptiveAuth implements HttpRequestInterceptor {
+ private AuthScheme authScheme = null;
+
+ public PreemptiveAuth(AuthScheme authScheme) {
+ this.authScheme = authScheme;
+ }
+
+ @Override
+ public void process(final HttpRequest request, final HttpContext context) throws HttpException,
+ IOException {
+
+ AuthState authState = (AuthState) context.getAttribute(ClientContext.TARGET_AUTH_STATE);
+ // If no auth scheme available yet, try to initialize it preemptively
+ if (authState.getAuthScheme() == null) {
+ CredentialsProvider credsProvider = (CredentialsProvider) context
+ .getAttribute(ClientContext.CREDS_PROVIDER);
+ Credentials creds = credsProvider.getCredentials(AuthScope.ANY);
+ authState.update(authScheme, creds);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fc5a9f0/solr/solrj/src/java/org/apache/solr/client/solrj/impl/PreemptiveBasicAuthClientBuilderFactory.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/PreemptiveBasicAuthClientBuilderFactory.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/PreemptiveBasicAuthClientBuilderFactory.java
new file mode 100644
index 0000000..76ce990
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/PreemptiveBasicAuthClientBuilderFactory.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.impl;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Optional;
+import java.util.Properties;
+
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.impl.auth.BasicScheme;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder.CredentialsProviderProvider;
+import org.apache.solr.common.params.MapSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.StrUtils;
+
+/**
+ * HttpClientConfigurer implementation providing support for preemptive Http Basic authentication
+ * scheme.
+ */
+@SuppressWarnings({"rawtypes", "unchecked"})
+public class PreemptiveBasicAuthClientBuilderFactory implements HttpClientBuilderFactory {
+ /**
+ * A system property used to specify a properties file containing default parameters used for
+ * creating a HTTP client. This is specifically useful for configuring the HTTP basic auth
+ * credentials (i.e. username/password). The name of the property must match the relevant
+ * Solr config property name.
+ */
+ public static final String SYS_PROP_HTTP_CLIENT_CONFIG = "solr.httpclient.config";
+
+ /**
+ * A system property to configure the Basic auth credentials via a java system property.
+ * Since this will expose the password on the command-line, it is not very secure. But
+ * this mechanism is added for backwards compatibility.
+ */
+ public static final String SYS_PROP_BASIC_AUTH_CREDENTIALS = "basicauth";
+
+ private static SolrParams defaultParams;
+ private static PreemptiveAuth requestInterceptor = new PreemptiveAuth(new BasicScheme());
+
+ static {
+ String credentials = System.getProperty(SYS_PROP_BASIC_AUTH_CREDENTIALS);
+ String configFile = System.getProperty(SYS_PROP_HTTP_CLIENT_CONFIG);
+
+ if (credentials != null && configFile != null) {
+ throw new RuntimeException("Basic authentication credentials passed via a configuration file"
+ + " as well as java system property. Please choose one mechanism!");
+ }
+
+ if (credentials != null) {
+ List<String> ss = StrUtils.splitSmart(credentials, ':');
+ if (ss.size() != 2) {
+ throw new RuntimeException("Please provide 'basicauth' in the 'user:password' format");
+ }
+ Properties defaultProps = new Properties();
+ defaultProps.setProperty(HttpClientUtil.PROP_BASIC_AUTH_USER, ss.get(0));
+ defaultProps.setProperty(HttpClientUtil.PROP_BASIC_AUTH_PASS, ss.get(1));
+ defaultParams = new MapSolrParams(new HashMap(defaultProps));
+ }
+
+ if(configFile != null) {
+ try {
+ Properties defaultProps = new Properties();
+ defaultProps.load(new InputStreamReader(new FileInputStream(configFile), StandardCharsets.UTF_8));
+ defaultParams = new MapSolrParams(new HashMap(defaultProps));
+ } catch (IOException e) {
+ throw new IllegalArgumentException("Unable to read the Http client config file", e);
+ }
+ }
+ }
+
+ /**
+ * This method enables configuring system wide defaults (apart from using a config file based approach).
+ */
+ public static void setDefaultSolrParams(SolrParams params) {
+ defaultParams = params;
+ }
+
+ @Override
+ public void close() throws IOException {
+ HttpClientUtil.removeRequestInterceptor(requestInterceptor);
+ }
+
+ @Override
+ public SolrHttpClientBuilder getHttpClientBuilder(Optional<SolrHttpClientBuilder> builder) {
+ return builder.isPresent() ?
+ initHttpClientBuilder(builder.get())
+ : initHttpClientBuilder(SolrHttpClientBuilder.create());
+ }
+
+ private SolrHttpClientBuilder initHttpClientBuilder(SolrHttpClientBuilder builder) {
+ final String basicAuthUser = defaultParams.get(HttpClientUtil.PROP_BASIC_AUTH_USER);
+ final String basicAuthPass = defaultParams.get(HttpClientUtil.PROP_BASIC_AUTH_PASS);
+ if(basicAuthUser == null || basicAuthPass == null) {
+ throw new IllegalArgumentException("username & password must be specified with " + getClass().getName());
+ }
+
+ builder.setDefaultCredentialsProvider(new CredentialsProviderProvider() {
+ @Override
+ public CredentialsProvider getCredentialsProvider() {
+ CredentialsProvider credsProvider = new BasicCredentialsProvider();
+ credsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(basicAuthUser, basicAuthPass));
+ return credsProvider;
+ }
+ });
+
+ HttpClientUtil.addRequestInterceptor(requestInterceptor);
+ return builder;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fc5a9f0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/AbsoluteValueEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/AbsoluteValueEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/AbsoluteValueEvaluator.java
new file mode 100644
index 0000000..38b3bb5
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/AbsoluteValueEvaluator.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ *
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.List;
+import java.util.Locale;
+
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class AbsoluteValueEvaluator extends NumberEvaluator {
+ protected static final long serialVersionUID = 1L;
+
+ public AbsoluteValueEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+ super(expression, factory);
+
+ if(1 != subEvaluators.size()){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - expecting one value but found %d",expression,subEvaluators.size()));
+ }
+ }
+
+ @Override
+ public Number evaluate(Tuple tuple) throws IOException {
+
+ List<BigDecimal> results = evaluateAll(tuple);
+
+ // we're still doing these checks because if we ever add an array-flatten evaluator,
+ // one found in the constructor could become != 1
+ if(1 != results.size()){
+ throw new IOException(String.format(Locale.ROOT,"%s(...) only works with a 1 value but %d were provided", constructingFactory.getFunctionName(getClass()), results.size()));
+ }
+
+ if(null == results.get(0)){
+ return null;
+ }
+
+ return normalizeType(results.get(0).abs());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fc5a9f0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/AddEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/AddEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/AddEvaluator.java
new file mode 100644
index 0000000..317741e
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/AddEvaluator.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ *
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.List;
+import java.util.Locale;
+
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class AddEvaluator extends NumberEvaluator {
+ protected static final long serialVersionUID = 1L;
+
+ public AddEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+ super(expression, factory);
+
+ if(subEvaluators.size() < 2){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - expecting at least two values but found %d",expression,subEvaluators.size()));
+ }
+ }
+
+ @Override
+ public Number evaluate(Tuple tuple) throws IOException {
+
+ List<BigDecimal> results = evaluateAll(tuple);
+
+ if(results.stream().anyMatch(item -> null == item)){
+ return null;
+ }
+
+ BigDecimal result = null;
+ if(results.size() > 0){
+ result = results.get(0);
+ for(int idx = 1; idx < results.size(); ++idx){
+ result = result.add(results.get(idx));
+ }
+ }
+
+ return normalizeType(result);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fc5a9f0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/AndEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/AndEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/AndEvaluator.java
new file mode 100644
index 0000000..290bd98
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/AndEvaluator.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ *
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Locale;
+import java.util.stream.Collectors;
+
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class AndEvaluator extends BooleanEvaluator {
+ protected static final long serialVersionUID = 1L;
+
+ public AndEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+ super(expression, factory);
+
+ if(subEvaluators.size() < 2){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - expecting at least two values but found %d",expression,subEvaluators.size()));
+ }
+ }
+
+ @Override
+ public Boolean evaluate(Tuple tuple) throws IOException {
+
+ List<Object> results = evaluateAll(tuple);
+
+ if(results.size() < 2){
+ String message = null;
+ if(1 == results.size()){
+ message = String.format(Locale.ROOT,"%s(...) only works with at least 2 values but 1 was provided", constructingFactory.getFunctionName(getClass()));
+ }
+ else{
+ message = String.format(Locale.ROOT,"%s(...) only works with at least 2 values but 0 were provided", constructingFactory.getFunctionName(getClass()));
+ }
+ throw new IOException(message);
+ }
+
+ Checker checker = constructChecker(results.get(0));
+ if(results.stream().anyMatch(result -> null == result && !checker.isNullAllowed())){
+ throw new IOException(String.format(Locale.ROOT,"Unable to check %s(...) because a null value was found", constructingFactory.getFunctionName(getClass())));
+ }
+ if(results.stream().anyMatch(result -> !checker.isCorrectType(result))){
+ throw new IOException(String.format(Locale.ROOT,"Unable to check %s(...) of differing types [%s]", constructingFactory.getFunctionName(getClass()), results.stream().map(item -> item.getClass().getSimpleName()).collect(Collectors.joining(","))));
+ }
+
+ for(int idx = 1; idx < results.size(); ++idx){
+ if(!checker.test(results.get(0), results.get(idx))){
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ private Checker constructChecker(Object fromValue) throws IOException{
+ if(null == fromValue){
+ throw new IOException(String.format(Locale.ROOT,"Unable to check %s(...) because a null value was found", constructingFactory.getFunctionName(getClass())));
+ }
+ else if(fromValue instanceof Boolean){
+ return new BooleanChecker(){
+ @Override
+ public boolean test(Object left, Object right) {
+ return (boolean)left && (boolean)right;
+ }
+ };
+ }
+
+ throw new IOException(String.format(Locale.ROOT,"Unable to check %s(...) for values of type '%s'", constructingFactory.getFunctionName(getClass()), fromValue.getClass().getSimpleName()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fc5a9f0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/BooleanEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/BooleanEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/BooleanEvaluator.java
new file mode 100644
index 0000000..bf21f1d
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/BooleanEvaluator.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ *
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public abstract class BooleanEvaluator extends ComplexEvaluator {
+ protected static final long serialVersionUID = 1L;
+
+ public BooleanEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+ super(expression, factory);
+ }
+
+ // restrict result to a Boolean
+ public abstract Boolean evaluate(Tuple tuple) throws IOException;
+
+ public List<Object> evaluateAll(final Tuple tuple) throws IOException {
+ List<Object> results = new ArrayList<Object>();
+ for(StreamEvaluator subEvaluator : subEvaluators){
+ results.add(subEvaluator.evaluate(tuple));
+ }
+
+ return results;
+ }
+
+ public interface Checker {
+ default boolean isNullAllowed(){
+ return false;
+ }
+ boolean isCorrectType(Object value);
+ boolean test(Object left, Object right);
+ }
+
+ public interface NullChecker extends Checker {
+ default boolean isNullAllowed(){
+ return true;
+ }
+ default boolean isCorrectType(Object value){
+ return true;
+ }
+ default boolean test(Object left, Object right){
+ return null == left && null == right;
+ }
+ }
+
+ public interface BooleanChecker extends Checker {
+ default boolean isCorrectType(Object value){
+ return value instanceof Boolean;
+ }
+ }
+
+ public interface NumberChecker extends Checker {
+ default boolean isCorrectType(Object value){
+ return value instanceof Number;
+ }
+ }
+
+ public interface StringChecker extends Checker {
+ default boolean isCorrectType(Object value){
+ return value instanceof String;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fc5a9f0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ComplexEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ComplexEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ComplexEvaluator.java
new file mode 100644
index 0000000..1e56d12
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ComplexEvaluator.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.UUID;
+
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public abstract class ComplexEvaluator implements StreamEvaluator {
+ protected static final long serialVersionUID = 1L;
+
+ protected UUID nodeId = UUID.randomUUID();
+
+ protected StreamFactory constructingFactory;
+ protected List<StreamEvaluator> subEvaluators = new ArrayList<StreamEvaluator>();
+
+ public ComplexEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+ constructingFactory = factory;
+
+ // We have to do this because order of the parameters matter
+ List<StreamExpressionParameter> parameters = factory.getOperandsOfType(expression, StreamExpressionParameter.class);
+
+ for(StreamExpressionParameter parameter : parameters){
+ if(parameter instanceof StreamExpression){
+ // possible evaluator
+ StreamExpression streamExpression = (StreamExpression)parameter;
+ if(factory.doesRepresentTypes(streamExpression, ComplexEvaluator.class)){
+ subEvaluators.add(factory.constructEvaluator(streamExpression));
+ }
+ else if(factory.doesRepresentTypes(streamExpression, SimpleEvaluator.class)){
+ subEvaluators.add(factory.constructEvaluator(streamExpression));
+ }
+ else{
+ // Will be treated as a field name
+ subEvaluators.add(new FieldEvaluator(streamExpression.toString()));
+ }
+ }
+ else if(parameter instanceof StreamExpressionValue){
+ if(0 != ((StreamExpressionValue)parameter).getValue().length()){
+ // special case - if evaluates to a number, boolean, or null then we'll treat it
+ // as a RawValueEvaluator
+ Object value = factory.constructPrimitiveObject(((StreamExpressionValue)parameter).getValue());
+ if(null == value || value instanceof Boolean || value instanceof Number){
+ subEvaluators.add(new RawValueEvaluator(value));
+ }
+ else if(value instanceof String){
+ subEvaluators.add(new FieldEvaluator((String)value));
+ }
+ }
+ }
+ }
+
+ if(expression.getParameters().size() != subEvaluators.size()){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - unknown operands found - expecting only StreamEvaluators or field names", expression));
+ }
+ }
+
+ @Override
+ public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
+ StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
+
+ for(StreamEvaluator evaluator : subEvaluators){
+ expression.addParameter(evaluator.toExpression(factory));
+ }
+ return expression;
+ }
+
+ @Override
+ public Explanation toExplanation(StreamFactory factory) throws IOException {
+ return new Explanation(nodeId.toString())
+ .withExpressionType(ExpressionType.EVALUATOR)
+ .withFunctionName(factory.getFunctionName(getClass()))
+ .withImplementingClass(getClass().getName())
+ .withExpression(toExpression(factory).toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fc5a9f0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ConditionalEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ConditionalEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ConditionalEvaluator.java
new file mode 100644
index 0000000..499e2f8
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ConditionalEvaluator.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ *
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public abstract class ConditionalEvaluator extends ComplexEvaluator {
+ protected static final long serialVersionUID = 1L;
+
+ public ConditionalEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+ super(expression, factory);
+ }
+
+ public List<Object> evaluateAll(final Tuple tuple) throws IOException {
+ List<Object> results = new ArrayList<Object>();
+ for(StreamEvaluator subEvaluator : subEvaluators){
+ results.add(subEvaluator.evaluate(tuple));
+ }
+
+ return results;
+ }
+
+ public interface Checker {
+ default boolean isNullAllowed(){
+ return false;
+ }
+ boolean isCorrectType(Object value);
+ boolean test(Object left, Object right);
+ }
+
+ public interface BooleanChecker extends Checker {
+ default boolean isCorrectType(Object value){
+ return value instanceof Boolean;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fc5a9f0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/DivideEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/DivideEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/DivideEvaluator.java
new file mode 100644
index 0000000..f21a7f3
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/DivideEvaluator.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ *
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.MathContext;
+import java.util.List;
+import java.util.Locale;
+
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class DivideEvaluator extends NumberEvaluator {
+ protected static final long serialVersionUID = 1L;
+
+ public DivideEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+ super(expression, factory);
+
+ if(2 != subEvaluators.size()){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - expecting two values but found %d",expression,subEvaluators.size()));
+ }
+ }
+
+ @Override
+ public Number evaluate(Tuple tuple) throws IOException {
+
+ List<BigDecimal> results = evaluateAll(tuple);
+
+ // we're still doing these checks because if we ever add an array-flatten evaluator,
+ // two found in the constructor could become != 2
+ if(2 != results.size()){
+ String message = null;
+ if(1 == results.size()){
+ message = String.format(Locale.ROOT,"%s(...) only works with a 2 values (numerator,denominator) but 1 was provided", constructingFactory.getFunctionName(getClass()));
+ }
+ else{
+ message = String.format(Locale.ROOT,"%s(...) only works with a 2 values (numerator,denominator) but %d were provided", constructingFactory.getFunctionName(getClass()), results.size());
+ }
+ throw new IOException(message);
+ }
+
+ BigDecimal numerator = results.get(0);
+ BigDecimal denominator = results.get(1);
+
+ if(null == numerator){
+ throw new IOException(String.format(Locale.ROOT,"Unable to %s(...) with a null numerator", constructingFactory.getFunctionName(getClass())));
+ }
+
+ if(null == denominator){
+ throw new IOException(String.format(Locale.ROOT,"Unable to %s(...) with a null denominator", constructingFactory.getFunctionName(getClass())));
+ }
+
+ if(0 == denominator.compareTo(BigDecimal.ZERO)){
+ throw new IOException(String.format(Locale.ROOT,"Unable to %s(...) with a 0 denominator", constructingFactory.getFunctionName(getClass())));
+ }
+
+ return normalizeType(numerator.divide(denominator, MathContext.DECIMAL64));
+ }
+}