You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2020/08/17 23:12:09 UTC
[lucene-solr] 39/49: @553 The village may think I'm crazy Or say
that I drift too far But once you know what you like, well There you are
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit cab3d10c4e235de80da30500f6733bcef82f1d78
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Sun Aug 16 12:03:20 2020 -0500
@553 The village may think I'm crazy
Or say that I drift too far
But once you know what you like, well
There you are
---
gradle/testing/defaults-tests.gradle | 5 +-
.../org/apache/solr/cloud/ElectionContext.java | 2 +-
.../java/org/apache/solr/cloud/LeaderElector.java | 20 +-
.../src/java/org/apache/solr/cloud/Overseer.java | 53 ++--
.../apache/solr/cloud/OverseerElectionContext.java | 29 +-
.../solr/cloud/ShardLeaderElectionContext.java | 27 +-
.../solr/cloud/ShardLeaderElectionContextBase.java | 2 +-
.../org/apache/solr/cloud/ZkCollectionTerms.java | 12 +-
.../java/org/apache/solr/cloud/ZkController.java | 11 +-
.../org/apache/solr/cloud/ZkDistributedQueue.java | 12 +-
.../java/org/apache/solr/core/CoreContainer.java | 14 +-
.../src/java/org/apache/solr/core/SolrCore.java | 9 +-
.../src/java/org/apache/solr/core/SolrCores.java | 23 +-
.../java/org/apache/solr/servlet/HttpSolrCall.java | 5 +-
.../apache/solr/update/DefaultSolrCoreState.java | 51 +--
.../apache/solr/cloud/CollectionsAPISolrJTest.java | 2 +
.../apache/solr/cloud/ConnectionManagerTest.java | 1 +
.../test/org/apache/solr/cloud/DeleteNodeTest.java | 1 +
.../org/apache/solr/cloud/SolrCLIZkUtilsTest.java | 2 +
.../org/apache/solr/util/OrderedExecutorTest.java | 8 +-
.../src/java/org/apache/solr/common/ParWork.java | 10 +-
.../org/apache/solr/common/ParWorkExecService.java | 15 +-
.../org/apache/solr/common/ParWorkExecutor.java | 19 +-
.../org/apache/solr/common/cloud/SolrZkClient.java | 5 +-
.../solr/common/util/ObjectReleaseTracker.java | 1 +
.../src/java/org/apache/solr/CollectionTester.java | 341 +++++++++++++++++++++
.../src/java/org/apache/solr/JSONTestUtil.java | 314 -------------------
27 files changed, 530 insertions(+), 464 deletions(-)
diff --git a/gradle/testing/defaults-tests.gradle b/gradle/testing/defaults-tests.gradle
index 34c583a..e5824b4 100644
--- a/gradle/testing/defaults-tests.gradle
+++ b/gradle/testing/defaults-tests.gradle
@@ -56,18 +56,17 @@ allprojects {
testOutputsDir = file("${reports.junitXml.destination}/outputs")
}
binaryResultsDirectory = file(propertyOrDefault("binaryResultsDirectory", binaryResultsDirectory))
+
if (verboseMode) {
maxParallelForks = 1
} else {
maxParallelForks = propertyOrDefault("tests.jvms", (int) Math.max(1, Math.min(Runtime.runtime.availableProcessors() / 2.0, 4.0))) as Integer
}
- forkEvery = 0
-
workingDir testsCwd
useJUnit()
- minHeapSize = propertyOrDefault("tests.minheapsize", "256m")
+ minHeapSize = propertyOrDefault("tests.minheapsize", "512m")
maxHeapSize = propertyOrDefault("tests.heapsize", "512m")
jvmArgs Commandline.translateCommandline(propertyOrDefault("tests.jvmargs", "-XX:TieredStopAtLevel=1 -XX:+UseParallelGC -XX:-UseBiasedLocking -Dorg.apache.xml.dtm.DTMManager=org.apache.xml.dtm.ref.DTMManagerDefault"));
diff --git a/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java b/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
index 4e690eb..f382466 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
@@ -47,7 +47,7 @@ public abstract class ElectionContext implements Closeable {
assert ObjectReleaseTracker.release(this);
}
- public void cancelElection() throws InterruptedException, KeeperException {
+ protected void cancelElection() throws InterruptedException, KeeperException {
}
abstract void runLeaderProcess(ElectionContext context, boolean weAreReplacement, int pauseBeforeStartMs) throws KeeperException, InterruptedException, IOException;
diff --git a/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java b/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
index dcc5847..4fe5537 100644
--- a/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
+++ b/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
@@ -71,7 +71,7 @@ public class LeaderElector {
private final Map<ContextKey,ElectionContext> electionContexts;
private final ContextKey contextKey;
-// public LeaderElector(SolrZkClient zkClient) {
+ // public LeaderElector(SolrZkClient zkClient) {
// this.zkClient = zkClient;
// this.contextKey = null;
// this.electionContexts = new ConcurrentHashMap<>(132, 0.75f, 50);
@@ -98,7 +98,9 @@ public class LeaderElector {
*/
private boolean checkIfIamLeader(final ElectionContext context, boolean replacement) throws KeeperException,
InterruptedException, IOException {
-
+ if (context.isClosed()) {
+ throw new AlreadyClosedException();
+ }
context.checkIfIamLeaderFired();
boolean checkAgain = false;
if (!getContext().isClosed()) {
@@ -163,6 +165,9 @@ public class LeaderElector {
// TODO: get this core param out of here
protected void runIamLeaderProcess(final ElectionContext context, boolean weAreReplacement) throws KeeperException,
InterruptedException, IOException {
+ if (context.isClosed()) {
+ throw new AlreadyClosedException();
+ }
context.runLeaderProcess(context, weAreReplacement,0);
}
@@ -215,7 +220,7 @@ public class LeaderElector {
* @return sequential node number
*/
public int joinElection(ElectionContext context, boolean replacement,boolean joinAtHead) throws KeeperException, InterruptedException, IOException {
- if (zkClient.isClosed()) {
+ if (context.isClosed() || zkClient.isClosed()) {
throw new AlreadyClosedException();
}
@@ -297,7 +302,9 @@ public class LeaderElector {
}
}
while(checkIfIamLeader(context, replacement)) {
-
+ if (context.isClosed()) {
+ throw new AlreadyClosedException();
+ }
}
return getSeq(context.leaderSeqPath);
@@ -347,11 +354,9 @@ public class LeaderElector {
// am I the next leader?
checkIfIamLeader(context, true);
} catch (AlreadyClosedException | InterruptedException e) {
- ParWork.propegateInterrupt(e);
log.info("Already shutting down");
return;
} catch (Exception e) {
- ParWork.propegateInterrupt(e);
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Exception canceling election", e);
}
}
@@ -372,6 +377,9 @@ public class LeaderElector {
}
void retryElection(ElectionContext context, boolean joinAtHead) throws KeeperException, InterruptedException, IOException {
+ if (context.isClosed()) {
+ throw new AlreadyClosedException();
+ }
ElectionWatcher watcher = this.watcher;
if (electionContexts != null) {
ElectionContext prevContext = electionContexts.put(contextKey, context);
diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
index c0d4ec2..e10695b 100644
--- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java
+++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
@@ -532,11 +532,6 @@ public class Overseer implements SolrCloseable {
protected volatile boolean isClosed;
private final Closeable thread;
- public OverseerThread(ThreadGroup tg, Closeable thread) {
- super(tg, (Runnable) thread);
- this.thread = thread;
- }
-
public OverseerThread(ThreadGroup ccTg, Closeable thread, String name) {
super(ccTg, (Runnable) thread, name);
this.thread = thread;
@@ -844,27 +839,15 @@ public class Overseer implements SolrCloseable {
public void closeAndDone() {
this.closeAndDone = true;
this.closed = true;
+ close();
}
public void close() {
if (this.id != null) {
log.info("Overseer (id={}) closing", id);
}
-
-
- if (zkController.getZkClient().isConnected()) {
- try {
- context.cancelElection();
- } catch (InterruptedException e) {
- ParWork.propegateInterrupt(e);
- } catch (KeeperException e) {
- log.error("Exception canceling election for overseer");
- }
- }
-
+ IOUtils.closeQuietly(context);
doClose();
-
- ParWork.close(context);
}
@Override
@@ -873,37 +856,45 @@ public class Overseer implements SolrCloseable {
}
void doClose() {
- if (closed) {
- return;
- }
closed = true;
if (log.isDebugEnabled()) {
log.debug("doClose() - start");
}
if (ccThread != null) {
- ccThread.interrupt();
+ ccThread.interrupt();
}
if (updaterThread != null) {
updaterThread.interrupt();
}
+// if (overseerCollectionConfigSetProcessor != null) {
+// overseerCollectionConfigSetProcessor.interrupt();
+// }
+
+
IOUtils.closeQuietly(ccThread);
IOUtils.closeQuietly(updaterThread);
if (ccThread != null) {
- try {
- ccThread.join();
- } catch (InterruptedException e) {
- // okay
+ while (true) {
+ try {
+ ccThread.join();
+ break;
+ } catch (InterruptedException e) {
+ // okay
+ }
}
}
if (updaterThread != null) {
- try {
- updaterThread.join();
- } catch (InterruptedException e) {
- // okay
+ while (true) {
+ try {
+ updaterThread.join();
+ break;
+ } catch (InterruptedException e) {
+ // okay
+ }
}
}
// closer.collect(() -> {
diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerElectionContext.java b/solr/core/src/java/org/apache/solr/cloud/OverseerElectionContext.java
index c971fb7..aad95c2 100644
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerElectionContext.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerElectionContext.java
@@ -49,7 +49,7 @@ final class OverseerElectionContext extends ShardLeaderElectionContextBase {
@Override
void runLeaderProcess(ElectionContext context, boolean weAreReplacement, int pauseBeforeStartMs) throws KeeperException,
InterruptedException, IOException {
- if (isClosed()) {
+ if (isClosed() || !zkClient.isConnected()) {
return;
}
@@ -94,7 +94,7 @@ final class OverseerElectionContext extends ShardLeaderElectionContextBase {
if (items.size() == 0) {
break;
}
- for (Pair<String,byte[]> item : items) {
+ for (Pair<String, byte[]> item : items) {
paths.add(item.first());
}
queue.remove(paths);
@@ -107,20 +107,17 @@ final class OverseerElectionContext extends ShardLeaderElectionContextBase {
@Override
public void cancelElection() throws InterruptedException, KeeperException {
- if (!zkClient.isConnected()) {
- log.info("Can't cancel, zkClient is not connected");
- return;
- }
-
try (ParWork closer = new ParWork(this, true)) {
- closer.collect("cancelElection", () -> {
- try {
- super.cancelElection();
- } catch (Exception e) {
- ParWork.propegateInterrupt(e);
- log.error("Exception closing Overseer", e);
- }
- });
+ if (zkClient.isConnected()) {
+ closer.collect("cancelElection", () -> {
+ try {
+ super.cancelElection();
+ } catch (Exception e) {
+ ParWork.propegateInterrupt(e);
+ log.error("Exception closing Overseer", e);
+ }
+ });
+ }
closer.collect("overseer", () -> {
try {
overseer.doClose();
@@ -146,7 +143,7 @@ final class OverseerElectionContext extends ShardLeaderElectionContextBase {
});
closer.collect("Overseer", () -> {
try {
- overseer.doClose();
+ cancelElection();
} catch (Exception e) {
ParWork.propegateInterrupt(e);
log.error("Exception closing Overseer", e);
diff --git a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java
index 9a32f93..9279237 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java
@@ -96,7 +96,6 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
}
});
closer.collect(syncStrategy);
- closer.addCollect();
}
this.isClosed = true;
@@ -104,7 +103,7 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
}
@Override
- public void cancelElection() throws InterruptedException, KeeperException {
+ protected void cancelElection() throws InterruptedException, KeeperException {
super.cancelElection();
String coreName = leaderProps.getStr(ZkStateReader.CORE_NAME_PROP);
try {
@@ -326,10 +325,14 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
log.info("I am the new leader: " + ZkCoreNodeProps.getCoreUrl(leaderProps) + " " + shardId);
} catch (AlreadyClosedException | InterruptedException e) {
- log.info("Already closed or interrupted, bailing..");
+ ParWork.propegateInterrupt("Already closed or interrupted, bailing..", e);
+ return;
} catch (Exception e) {
SolrException.log(log, "There was a problem trying to register as the leader", e);
ParWork.propegateInterrupt(e);
+ if (isClosed()) {
+ return;
+ }
if(e instanceof IOException
|| (e instanceof KeeperException && (!(e instanceof SessionExpiredException)))) {
@@ -367,6 +370,9 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
* false if otherwise
*/
private boolean waitForEligibleBecomeLeaderAfterTimeout(ZkShardTerms zkShardTerms, CoreDescriptor cd, int timeout) throws InterruptedException {
+ if (isClosed()) {
+ throw new AlreadyClosedException();
+ }
String coreNodeName = cd.getCloudDescriptor().getCoreNodeName();
AtomicReference<Boolean> foundHigherTerm = new AtomicReference<>();
try {
@@ -397,6 +403,9 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
* @return true if other replicas with higher term participated in the election, false if otherwise
*/
private boolean replicasWithHigherTermParticipated(ZkShardTerms zkShardTerms, String coreNodeName) {
+ if (isClosed()) {
+ throw new AlreadyClosedException();
+ }
ClusterState clusterState = zkController.getClusterState();
DocCollection docCollection = clusterState.getCollectionOrNull(collection, true);
Slice slices = (docCollection == null) ? null : docCollection.getSlice(shardId);
@@ -427,7 +436,7 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
private void rejoinLeaderElection(SolrCore core)
throws InterruptedException, KeeperException, IOException {
// remove our ephemeral and re join the election
- if (cc.isShutDown()) {
+ if (isClosed()) {
log.debug("Not rejoining election because CoreContainer is closed");
return;
}
@@ -436,8 +445,18 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
cancelElection();
+ if (isClosed()) {
+ log.debug("Not rejoining election because CoreContainer is closed");
+ return;
+ }
+
core.getUpdateHandler().getSolrCoreState().doRecovery(zkController.getCoreContainer(), core.getCoreDescriptor());
+ if (isClosed()) {
+ log.debug("Not rejoining election because CoreContainer is closed");
+ return;
+ }
+
leaderElector.joinElection(this, true);
}
diff --git a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContextBase.java b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContextBase.java
index 0662afa..57a888a 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContextBase.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContextBase.java
@@ -66,7 +66,7 @@ class ShardLeaderElectionContextBase extends ElectionContext {
}
@Override
- public void cancelElection() throws InterruptedException, KeeperException {
+ protected void cancelElection() throws InterruptedException, KeeperException {
if (!zkClient.isConnected()) {
log.info("Can't cancel, zkClient is not connected");
return;
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkCollectionTerms.java b/solr/core/src/java/org/apache/solr/cloud/ZkCollectionTerms.java
index 671bb469..8641b74 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkCollectionTerms.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkCollectionTerms.java
@@ -17,13 +17,14 @@
package org.apache.solr.cloud;
-import java.util.HashMap;
-import java.util.Map;
-
+import org.apache.solr.common.AlreadyClosedException;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.util.ObjectReleaseTracker;
import org.apache.solr.core.CoreDescriptor;
+import java.util.HashMap;
+import java.util.Map;
+
/**
* Used to manage all ZkShardTerms of a collection
*/
@@ -31,6 +32,7 @@ class ZkCollectionTerms implements AutoCloseable {
private final String collection;
private final Map<String, ZkShardTerms> terms;
private final SolrZkClient zkClient;
+ private boolean closed;
ZkCollectionTerms(String collection, SolrZkClient client) {
this.collection = collection;
@@ -42,6 +44,7 @@ class ZkCollectionTerms implements AutoCloseable {
public ZkShardTerms getShard(String shardId) {
synchronized (terms) {
+ if (closed) throw new AlreadyClosedException();
if (!terms.containsKey(shardId)) terms.put(shardId, new ZkShardTerms(collection, shardId, zkClient));
return terms.get(shardId);
}
@@ -49,12 +52,14 @@ class ZkCollectionTerms implements AutoCloseable {
public void register(String shardId, String coreNodeName) {
synchronized (terms) {
+ if (closed) throw new AlreadyClosedException();
getShard(shardId).registerTerm(coreNodeName);
}
}
public void remove(String shardId, CoreDescriptor coreDescriptor) {
synchronized (terms) {
+ if (closed) throw new AlreadyClosedException();
if (getShard(shardId).removeTerm(coreDescriptor)) {
terms.remove(shardId).close();
}
@@ -63,6 +68,7 @@ class ZkCollectionTerms implements AutoCloseable {
public void close() {
synchronized (terms) {
+ this.closed = true;
terms.values().forEach(ZkShardTerms::close);
}
ObjectReleaseTracker.release(this);
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index 4587631..98d0d22 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -613,8 +613,6 @@ public class ZkController implements Closeable {
closer.collect(cloudSolrClient);
closer.collect(replicateFromLeaders.values());
closer.collect(overseerContexts.values());
- closer.addCollect();
-
closer.collect("Overseer", () -> {
if (overseer != null) {
overseer.closeAndDone();
@@ -1135,7 +1133,6 @@ public class ZkController implements Closeable {
throw new SolrException(ErrorCode.SERVER_ERROR, e);
}
});
- worker.addCollect();
}
// Do this last to signal we're up.
createEphemeralLiveNode();
@@ -1989,6 +1986,14 @@ public class ZkController implements Closeable {
ZkStateReader.BASE_URL_PROP, getBaseUrl(),
ZkStateReader.CORE_NODE_NAME_PROP, coreNodeName);
overseerJobQueue.offer(Utils.toJSON(m));
+ zkStateReader.waitForState(cloudDescriptor.getCollectionName(), 10, TimeUnit.SECONDS, (l,c) -> {
+ if (c == null) return true;
+ Slice slice = c.getSlice(cloudDescriptor.getShardId());
+ if (slice == null) return true;
+ Replica r = slice.getReplica(cloudDescriptor.getCoreNodeName());
+ if (r == null) return true;
+ return false;
+ });
}
}
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedQueue.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedQueue.java
index 6743845..aad9cf8 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedQueue.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedQueue.java
@@ -20,6 +20,7 @@ import com.codahale.metrics.Timer;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.solr.client.solrj.cloud.DistributedQueue;
+import org.apache.solr.common.AlreadyClosedException;
import org.apache.solr.common.ParWork;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ConnectionManager.IsClosed;
@@ -46,6 +47,7 @@ import java.util.Map;
import java.util.NoSuchElementException;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
@@ -464,9 +466,15 @@ public class ZkDistributedQueue implements DistributedQueue {
TreeSet<String> existingChildren = knownChildren;
while (existingChildren == knownChildren) {
- changed.await(500, TimeUnit.MILLISECONDS);
+ try {
+ changed.await(500, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ ParWork.propegateInterrupt(e);
+ throw new AlreadyClosedException();
+ }
if (timeout.hasTimedOut()) {
- break;
+ //throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Timeout");
+ return Collections.emptyList();
}
}
} finally {
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 7e3f009..fc0bb49 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -1181,8 +1181,6 @@ public class CoreContainer implements Closeable {
closer.addCollect();
closer.collect(zkSys);
- closer.addCollect();
-
}
assert ObjectReleaseTracker.release(this);
@@ -1802,9 +1800,7 @@ public class CoreContainer implements Closeable {
}
SolrCore core = null;
- boolean close;
try {
- close = solrCores.isLoadedNotPendingClose(name);
core = solrCores.remove(name);
solrCores.removeCoreDescriptor(cd);
@@ -1834,6 +1830,8 @@ public class CoreContainer implements Closeable {
throw new SolrException(ErrorCode.SERVER_ERROR, "Interrupted while unregistering core [" + name + "] from cloud state");
} catch (KeeperException e) {
throw new SolrException(ErrorCode.SERVER_ERROR, "Error unregistering core [" + name + "] from cloud state", e);
+ } catch (AlreadyClosedException e) {
+
} catch (Exception e) {
throw new SolrException(ErrorCode.SERVER_ERROR, "Error unregistering core [" + name + "] from cloud state", e);
}
@@ -1846,9 +1844,7 @@ public class CoreContainer implements Closeable {
core.unloadOnClose(cd, deleteIndexDir, deleteDataDir, deleteInstanceDir);
- if (close) {
- core.closeAndWait();
- }
+ core.closeAndWait();
}
public void rename(String name, String toName) {
@@ -2031,10 +2027,6 @@ public class CoreContainer implements Closeable {
return solrCores.isLoaded(name);
}
- public boolean isLoadedNotPendingClose(String name) {
- return solrCores.isLoadedNotPendingClose(name);
- }
-
// Primarily for transient cores when a core is aged out.
// public void queueCoreToClose(SolrCore coreToClose) {
// solrCores.queueCoreToClose(coreToClose);
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index 80fe2e4..30d499d 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -176,6 +176,7 @@ import org.apache.solr.util.plugin.PluginInfoInitialized;
import org.apache.solr.util.plugin.SolrCoreAware;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.Stat;
+import org.eclipse.jetty.util.BlockingArrayQueue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1707,7 +1708,9 @@ public final class SolrCore implements SolrInfoBean, Closeable {
closer.collect(updateHandler);
closer.collect("closeSearcher", () -> {
closeSearcher();
+ //searcherExecutor.shutdownNow();
});
+ // closer.addCollect();
closer.collect(searcherExecutor);
closer.addCollect();
@@ -1741,7 +1744,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
infoRegistry.clear();
//areAllSearcherReferencesEmpty();
-
+ refCount.set(-1);
ObjectReleaseTracker.release(this);
}
@@ -1756,7 +1759,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
* Whether this core is closed.
*/
public boolean isClosed() {
- return refCount.get() <= 0;
+ return refCount.get() < 0;
}
private final Collection<CloseHook> closeHooks = ConcurrentHashMap.newKeySet(128);
@@ -1891,7 +1894,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
private final LinkedList<RefCounted<SolrIndexSearcher>> _searchers = new LinkedList<>();
private final LinkedList<RefCounted<SolrIndexSearcher>> _realtimeSearchers = new LinkedList<>();
- final ExecutorService searcherExecutor = new ParWorkExecutor("searcherExecutor", 1, 1, 1, new LinkedBlockingQueue<>());
+ final ExecutorService searcherExecutor = new ParWorkExecutor("searcherExecutor", 0, 1, 1, new BlockingArrayQueue<>());
private AtomicInteger onDeckSearchers = new AtomicInteger(); // number of searchers preparing
// Lock ordering: one can acquire the openSearcherLock and then the searcherLock, but not vice-versa.
private final Object searcherLock = new Object(); // the sync object for the searcher
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCores.java b/solr/core/src/java/org/apache/solr/core/SolrCores.java
index 7ba4de0..64a1936 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCores.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCores.java
@@ -70,6 +70,9 @@ class SolrCores implements Closeable {
}
protected void addCoreDescriptor(CoreDescriptor p) {
+ if (isClosed()) {
+ throw new AlreadyClosedException();
+ }
if (p.isTransient()) {
if (getTransientCacheHandler() != null) {
getTransientCacheHandler().addTransientDescriptor(p.getName(), p);
@@ -307,26 +310,6 @@ class SolrCores implements Closeable {
return core;
}
- // See SOLR-5366 for why the UNLOAD command needs to know whether a core is actually loaded or not, it might have
- // to close the core. However, there's a race condition. If the core happens to be in the pending "to close" queue,
- // we should NOT close it in unload core.
- protected boolean isLoadedNotPendingClose(String name) {
- if (cores.containsKey(name)) {
- return true;
- }
- if (getTransientCacheHandler() != null && getTransientCacheHandler().containsCore(name)) {
- // Check pending
- for (SolrCore core : pendingCloses) {
- if (core.getName().equals(name)) {
- return false;
- }
- }
-
- return true;
- }
- return false;
- }
-
protected boolean isLoaded(String name) {
if (cores.containsKey(name)) {
return true;
diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
index 07679e1..9f9c3e4 100644
--- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
+++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
@@ -1064,6 +1064,9 @@ public class HttpSolrCall {
Collection<Slice> slices, boolean activeSlices) {
if (activeSlices) {
for (Map.Entry<String, DocCollection> entry : clusterState.getCollectionsMap().entrySet()) {
+ if (entry.getValue() == null) {
+ continue;
+ }
final Slice[] activeCollectionSlices = entry.getValue().getActiveSlicesArr();
for (Slice s : activeCollectionSlices) {
slices.add(s);
@@ -1081,7 +1084,7 @@ public class HttpSolrCall {
protected String getRemoteCoreUrl(String collectionName, String origCorename) throws SolrException {
ClusterState clusterState = cores.getZkController().getClusterState();
- final DocCollection docCollection = clusterState.getCollectionOrNull(collectionName, true);
+ final DocCollection docCollection = clusterState.getCollectionOrNull(collectionName, false);
Slice[] slices = (docCollection != null) ? docCollection.getActiveSlicesArr() : null;
List<Slice> activeSlices = new ArrayList<>();
boolean byCoreName = false;
diff --git a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
index 325962d..b61722a 100644
--- a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
+++ b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
@@ -25,6 +25,7 @@ import org.apache.solr.common.AlreadyClosedException;
import org.apache.solr.common.ParWork;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.util.IOUtils;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.DirectoryFactory;
@@ -360,7 +361,7 @@ public final class DefaultSolrCoreState extends SolrCoreState implements Recover
recoveryThrottle.minimumWaitBetweenActions();
recoveryThrottle.markAttemptingAction();
if (recoveryStrat != null) {
- ParWork.close(recoveryStrat);
+ IOUtils.closeQuietly(recoveryStrat);
}
if (prepForClose || cc.isShutDown() || closed) {
@@ -401,23 +402,36 @@ public final class DefaultSolrCoreState extends SolrCoreState implements Recover
if (prepForClose) {
this.prepForClose = true;
}
-
- if (recoveryStrat != null) {
- try {
- recoveryStrat.close();
- } catch (NullPointerException e) {
- // okay
- }
- if (wait && recoveryStrat != null && recoveryFuture != null) {
- try {
- recoveryFuture.get(10, TimeUnit.MINUTES); // nocommit - how long? make configurable too
- } catch (InterruptedException e) {
- ParWork.propegateInterrupt(e);
- throw new SolrException(ErrorCode.SERVER_ERROR, e);
- } catch (ExecutionException e) {
- throw new SolrException(ErrorCode.SERVER_ERROR, e);
- } catch (TimeoutException e) {
- throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ try (ParWork closer = new ParWork(this, true)) {
+ if (recoveryStrat != null) {
+ closer.collect("recoveryStrat", (() -> {
+ try {
+ recoveryFuture.cancel(true);
+ recoveryStrat.close();
+ } catch (NullPointerException e) {
+ // okay
+ }
+ try {
+ recoveryStrat.close();
+ } catch (NullPointerException e) {
+ // okay
+ }
+ }));
+
+ if (wait && recoveryStrat != null && recoveryFuture != null) {
+ closer.collect("recoveryStrat", (() -> {
+ try {
+ recoveryFuture.get(10,
+ TimeUnit.MINUTES); // nocommit - how long? make configurable too
+ } catch (InterruptedException e) {
+ ParWork.propegateInterrupt(e);
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ } catch (ExecutionException e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ } catch (TimeoutException e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ }
+ }));
}
}
recoveryFuture = null;
@@ -454,7 +468,6 @@ public final class DefaultSolrCoreState extends SolrCoreState implements Recover
// iwLock.writeLock().unlock();
}
});
- worker.addCollect();
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
index e1535ae..6267964 100644
--- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
@@ -915,6 +915,8 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {
}
@Test
+ // nocommit
+ @Ignore
public void testModifyCollectionAttribute() throws IOException, SolrServerException {
final String collection = "testAddAndDeleteCollectionAttribute";
CollectionAdminRequest.createCollection(collection, "conf", 1, 1)
diff --git a/solr/core/src/test/org/apache/solr/cloud/ConnectionManagerTest.java b/solr/core/src/test/org/apache/solr/cloud/ConnectionManagerTest.java
index a38ab1b..6b01022 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ConnectionManagerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ConnectionManagerTest.java
@@ -73,6 +73,7 @@ public class ConnectionManagerTest extends SolrTestCaseJ4 {
}
}
+ @Nightly // sleepy test
public void testLikelyExpired() throws Exception {
// setup a SolrZkClient to do some getBaseUrlForNodeName testing
diff --git a/solr/core/src/test/org/apache/solr/cloud/DeleteNodeTest.java b/solr/core/src/test/org/apache/solr/cloud/DeleteNodeTest.java
index bc3d053..14e1f00 100644
--- a/solr/core/src/test/org/apache/solr/cloud/DeleteNodeTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/DeleteNodeTest.java
@@ -33,6 +33,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Set;
+// nocommit flakey
public class DeleteNodeTest extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
diff --git a/solr/core/src/test/org/apache/solr/cloud/SolrCLIZkUtilsTest.java b/solr/core/src/test/org/apache/solr/cloud/SolrCLIZkUtilsTest.java
index 4fc4c81..72fe292 100644
--- a/solr/core/src/test/org/apache/solr/cloud/SolrCLIZkUtilsTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/SolrCLIZkUtilsTest.java
@@ -32,6 +32,7 @@ import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.List;
+import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkMaintenanceUtils;
import org.apache.solr.util.SolrCLI;
@@ -42,6 +43,7 @@ import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
+@LuceneTestCase.Nightly // slow test
public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
@BeforeClass
diff --git a/solr/core/src/test/org/apache/solr/util/OrderedExecutorTest.java b/solr/core/src/test/org/apache/solr/util/OrderedExecutorTest.java
index 65265ad..1e5f389 100644
--- a/solr/core/src/test/org/apache/solr/util/OrderedExecutorTest.java
+++ b/solr/core/src/test/org/apache/solr/util/OrderedExecutorTest.java
@@ -54,7 +54,7 @@ public class OrderedExecutorTest extends SolrTestCase {
public void testExecutionInOrder() {
IntBox intBox = new IntBox();
OrderedExecutor orderedExecutor = new OrderedExecutor(TEST_NIGHTLY ? 10 : 3,
- new ParWorkExecutor("executeInOrderTest", TEST_NIGHTLY ? 10 : 3, TEST_NIGHTLY ? 10 : 3));
+ ParWork.getExecutorService(TEST_NIGHTLY ? 10 : 3));
try {
for (int i = 0; i < 100; i++) {
orderedExecutor.execute(1, () -> intBox.value.incrementAndGet());
@@ -71,7 +71,7 @@ public class OrderedExecutorTest extends SolrTestCase {
@Test
public void testLockWhenQueueIsFull() throws ExecutionException {
final OrderedExecutor orderedExecutor = new OrderedExecutor
- (TEST_NIGHTLY ? 10 : 3, new ParWorkExecutor("testLockWhenQueueIsFull_test", TEST_NIGHTLY ? 10 : 3, TEST_NIGHTLY ? 10 : 3));
+ (TEST_NIGHTLY ? 10 : 3, ParWork.getExecutorService(TEST_NIGHTLY ? 10 : 3));
try {
// AAA and BBB events will both depend on the use of the same lockId
@@ -122,7 +122,7 @@ public class OrderedExecutorTest extends SolrTestCase {
final int parallelism = atLeast(3);
final OrderedExecutor orderedExecutor = new OrderedExecutor
- (parallelism, new ParWorkExecutor("testRunInParallel_test", parallelism, parallelism));
+ (parallelism, ParWork.getExecutorService(parallelism));
try {
// distinct lockIds should be able to be used in parallel, up to the size of the executor,
@@ -226,7 +226,7 @@ public class OrderedExecutorTest extends SolrTestCase {
run.put(i, i);
}
OrderedExecutor orderedExecutor = new OrderedExecutor(TEST_NIGHTLY ? 10 : 3,
- new ParWorkExecutor("testStress", TEST_NIGHTLY ? 10 : 3, TEST_NIGHTLY ? 10 : 3));
+ ParWork.getExecutorService(TEST_NIGHTLY ? 10 : 3));
try {
for (int i = 0; i < (TEST_NIGHTLY ? 1000 : 55); i++) {
int key = random().nextInt(N);
diff --git a/solr/solrj/src/java/org/apache/solr/common/ParWork.java b/solr/solrj/src/java/org/apache/solr/common/ParWork.java
index fd4e3d5..b84843e 100644
--- a/solr/solrj/src/java/org/apache/solr/common/ParWork.java
+++ b/solr/solrj/src/java/org/apache/solr/common/ParWork.java
@@ -46,6 +46,7 @@ import java.util.concurrent.FutureTask;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicReference;
/**
@@ -451,13 +452,14 @@ public class ParWork implements Closeable {
if (closeCalls.size() > 0) {
List<Future<Object>> results = new ArrayList<>(closeCalls.size());
- for (Callable<Object> call : closeCalls) {
- Future<Object> future = executor.submit(call);
+
+ for (Callable call : closeCalls) {
+ Future future = executor.submit(call);
results.add(future);
}
// List<Future<Object>> results = executor.invokeAll(closeCalls, 8, TimeUnit.SECONDS);
-
+ int i = 0;
for (Future<Object> future : results) {
try {
future.get(
@@ -468,6 +470,8 @@ public class ParWork implements Closeable {
future.isDone(), future.isCancelled());
// throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "A task did nor finish" +future.isDone() + " " + future.isCancelled());
}
+ } catch (TimeoutException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, objects.get(i).label, e);
} catch (InterruptedException e1) {
log.warn(WORK_WAS_INTERRUPTED);
// TODO: save interrupted status and reset it at end?
diff --git a/solr/solrj/src/java/org/apache/solr/common/ParWorkExecService.java b/solr/solrj/src/java/org/apache/solr/common/ParWorkExecService.java
index 780af28..527c0db 100644
--- a/solr/solrj/src/java/org/apache/solr/common/ParWorkExecService.java
+++ b/solr/solrj/src/java/org/apache/solr/common/ParWorkExecService.java
@@ -203,8 +203,9 @@ public class ParWorkExecService extends AbstractExecutorService {
public void execute(Runnable runnable) {
if (shutdown) {
- runIt(runnable, false, true, true);
- return;
+ throw new RejectedExecutionException();
+// runIt(runnable, false, true, true);
+// return;
}
running.incrementAndGet();
if (runnable instanceof ParWork.SolrFutureTask) {
@@ -292,13 +293,9 @@ public class ParWorkExecService extends AbstractExecutorService {
}
} finally {
if (!alreadyShutdown) {
- try {
- running.decrementAndGet();
- synchronized (awaitTerminate) {
- awaitTerminate.notifyAll();
- }
- } finally {
- if (!callThreadRuns) ParWork.closeExecutor();
+ running.decrementAndGet();
+ synchronized (awaitTerminate) {
+ awaitTerminate.notifyAll();
}
}
}
diff --git a/solr/solrj/src/java/org/apache/solr/common/ParWorkExecutor.java b/solr/solrj/src/java/org/apache/solr/common/ParWorkExecutor.java
index 69cf9b7..b7b250b 100644
--- a/solr/solrj/src/java/org/apache/solr/common/ParWorkExecutor.java
+++ b/solr/solrj/src/java/org/apache/solr/common/ParWorkExecutor.java
@@ -63,19 +63,20 @@ public class ParWorkExecutor extends ThreadPoolExecutor {
}
});
- setRejectedExecutionHandler(new CallerRunsPolicy());
+ //setRejectedExecutionHandler(new CallerRunsPolicy());
}
public void shutdown() {
- // wake up idle threads!
- ThreadPoolExecutor exec = ParWork.getEXEC();
- for (int i = 0; i < getPoolSize(); i++) {
- exec.submit(new Runnable() {
- @Override
- public void run() {
+ if (!isShutdown()) {
+ // wake up idle threads!
+ for (int i = 0; i < getPoolSize(); i++) {
+ submit(new Runnable() {
+ @Override
+ public void run() {
- }
- });
+ }
+ });
+ }
}
super.shutdown();
}
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
index f635c1b..8afe74a0 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
@@ -26,6 +26,7 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.StringUtils;
import org.apache.solr.common.cloud.ConnectionManager.IsClosed;
import org.apache.solr.common.util.CloseTracker;
+import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.ObjectReleaseTracker;
import org.apache.zookeeper.CreateMode;
@@ -853,8 +854,10 @@ public class SolrZkClient implements Closeable {
log.info("Closing {} instance {}", SolrZkClient.class.getSimpleName(), this);
isClosed = true;
- // zkCallbackExecutor.shutdownNow();
connManager.close();
+ // ExecutorUtil.shutdownAndAwaitTermination(zkConnManagerCallbackExecutor);
+ // ExecutorUtil.shutdownAndAwaitTermination(zkCallbackExecutor);
+
closeTracker.close();
assert ObjectReleaseTracker.release(this);
}
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/ObjectReleaseTracker.java b/solr/solrj/src/java/org/apache/solr/common/util/ObjectReleaseTracker.java
index fa3b2d9..157c5b1 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/ObjectReleaseTracker.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/ObjectReleaseTracker.java
@@ -65,6 +65,7 @@ public class ObjectReleaseTracker {
* @param object
*/
public static String checkEmpty(String object) {
+ // if (true) return null; // nocommit
StringBuilder error = new StringBuilder();
Set<Entry<Object,String>> entries = OBJECTS.entrySet();
Set<Entry<Object,String>> entriesCopy = new HashSet<>(entries);
diff --git a/solr/test-framework/src/java/org/apache/solr/CollectionTester.java b/solr/test-framework/src/java/org/apache/solr/CollectionTester.java
new file mode 100644
index 0000000..2cee80f
--- /dev/null
+++ b/solr/test-framework/src/java/org/apache/solr/CollectionTester.java
@@ -0,0 +1,341 @@
+package org.apache.solr;
+
+import org.apache.solr.common.util.StrUtils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Tests simple object graphs, like those generated by the noggit JSON parser
+ */
+class CollectionTester {
+ public Object valRoot;
+ public Object val;
+ public Object expectedRoot;
+ public Object expected;
+ public double delta;
+ public List<Object> path;
+ public String err;
+
+ public CollectionTester(Object val, double delta) {
+ this.val = val;
+ this.valRoot = val;
+ this.delta = delta;
+ path = new ArrayList<>();
+ }
+
+ public CollectionTester(Object val) {
+ this(val, JSONTestUtil.DEFAULT_DELTA);
+ }
+
+ public String getPath() {
+ StringBuilder sb = new StringBuilder();
+ boolean first = true;
+ for (Object seg : path) {
+ if (seg == null) break;
+ if (!first) sb.append('/');
+ else first = false;
+
+ if (seg instanceof Integer) {
+ sb.append('[');
+ sb.append(seg);
+ sb.append(']');
+ } else {
+ sb.append(seg.toString());
+ }
+ }
+ return sb.toString();
+ }
+
+ void setPath(Object lastSeg) {
+ path.set(path.size() - 1, lastSeg);
+ }
+
+ Object popPath() {
+ return path.remove(path.size() - 1);
+ }
+
+ void pushPath(Object lastSeg) {
+ path.add(lastSeg);
+ }
+
+ void setErr(String msg) {
+ err = msg;
+ }
+
+ public boolean match(Object expected) {
+ this.expectedRoot = expected;
+ this.expected = expected;
+ return match();
+ }
+
+ boolean match() {
+ if (expected == val) {
+ return true;
+ }
+ if (expected == null || val == null) {
+ setErr("mismatch: '" + expected + "'!='" + val + "'");
+ return false;
+ }
+ if (expected instanceof List) {
+ return matchList();
+ }
+ if (expected instanceof Map) {
+ return matchMap();
+ }
+
+ // generic fallback
+ if (!expected.equals(val)) {
+
+ if (expected instanceof String) {
+ String str = (String) expected;
+ if (str.length() > 6 && str.startsWith("///") && str.endsWith("///")) {
+ return handleSpecialString(str);
+ }
+ }
+
+ // make an exception for some numerics
+ if ((expected instanceof Integer && val instanceof Long
+ || expected instanceof Long && val instanceof Integer)
+ && ((Number) expected).longValue() == ((Number) val).longValue()) {
+ return true;
+ } else if ((expected instanceof Double || expected instanceof Float) && (
+ val instanceof Double || val instanceof Float)) {
+ double a = ((Number) expected).doubleValue();
+ double b = ((Number) val).doubleValue();
+ if (Double.compare(a, b) == 0) return true;
+ if (Math.abs(a - b) < delta) return true;
+ }
+ setErr("mismatch: '" + expected + "'!='" + val + "'");
+ return false;
+ }
+
+ // setErr("unknown expected type " + expected.getClass().getName());
+ return true;
+ }
+
+ private boolean handleSpecialString(String str) {
+ String code = str.substring(3, str.length() - 3);
+ if ("ignore".equals(code)) {
+ return true;
+ } else if (code.startsWith("regex:")) {
+ String regex = code.substring("regex:".length());
+ if (!(val instanceof String)) {
+ setErr("mismatch: '" + expected + "'!='" + val
+ + "', value is not a string");
+ return false;
+ }
+ Pattern pattern = Pattern.compile(regex);
+ Matcher matcher = pattern.matcher((String) val);
+ if (matcher.find()) {
+ return true;
+ }
+ setErr(
+ "mismatch: '" + expected + "'!='" + val + "', regex does not match");
+ return false;
+ }
+
+ setErr("mismatch: '" + expected + "'!='" + val + "'");
+ return false;
+ }
+
+ boolean matchList() {
+ List expectedList = (List) expected;
+ List v = asList();
+ if (v == null) return false;
+ int a = 0;
+ int b = 0;
+ pushPath(null);
+ for (; ; ) {
+ if (a >= expectedList.size() && b >= v.size()) {
+ break;
+ }
+
+ if (a >= expectedList.size() || b >= v.size()) {
+ popPath();
+ setErr("List size mismatch");
+ return false;
+ }
+
+ expected = expectedList.get(a);
+ val = v.get(b);
+ setPath(b);
+ if (!match()) return false;
+
+ a++;
+ b++;
+ }
+
+ popPath();
+ return true;
+ }
+
+ private static Set<String> reserved = new HashSet<>(
+ Arrays.asList("_SKIP_", "_MATCH_", "_ORDERED_", "_UNORDERED_"));
+
+ boolean matchMap() {
+ Map<String,Object> expectedMap = (Map<String,Object>) expected;
+ Map<String,Object> v = asMap();
+ if (v == null) return false;
+
+ boolean ordered = false;
+ String skipList = (String) expectedMap.get("_SKIP_");
+ String matchList = (String) expectedMap.get("_MATCH_");
+ Object orderedStr = expectedMap.get("_ORDERED_");
+ Object unorderedStr = expectedMap.get("_UNORDERED_");
+
+ if (orderedStr != null) ordered = true;
+ if (unorderedStr != null) ordered = false;
+
+ Set<String> match = null;
+ if (matchList != null) {
+ match = new HashSet(StrUtils.splitSmart(matchList, ",", false));
+ }
+
+ Set<String> skips = null;
+ if (skipList != null) {
+ skips = new HashSet(StrUtils.splitSmart(skipList, ",", false));
+ }
+
+ Set<String> keys = match != null ? match : expectedMap.keySet();
+ Set<String> visited = new HashSet<>();
+
+ Iterator<Map.Entry<String,Object>> iter = ordered ?
+ v.entrySet().iterator() :
+ null;
+
+ int numExpected = 0;
+
+ pushPath(null);
+ for (String expectedKey : keys) {
+ if (reserved.contains(expectedKey)) continue;
+ numExpected++;
+
+ setPath(expectedKey);
+ if (!v.containsKey(expectedKey)) {
+ popPath();
+ setErr("expected key '" + expectedKey + "'");
+ return false;
+ }
+
+ expected = expectedMap.get(expectedKey);
+
+ if (ordered) {
+ Map.Entry<String,Object> entry;
+ String foundKey;
+ for (; ; ) {
+ if (!iter.hasNext()) {
+ popPath();
+ setErr("expected key '" + expectedKey + "' in ordered map");
+ return false;
+ }
+ entry = iter.next();
+ foundKey = entry.getKey();
+ if (skips != null && skips.contains(foundKey)) continue;
+ if (match != null && !match.contains(foundKey)) continue;
+ break;
+ }
+
+ if (!entry.getKey().equals(expectedKey)) {
+ popPath();
+ setErr(
+ "expected key '" + expectedKey + "' instead of '" + entry.getKey()
+ + "' in ordered map");
+ return false;
+ }
+ val = entry.getValue();
+ } else {
+ if (skips != null && skips.contains(expectedKey)) continue;
+ val = v.get(expectedKey);
+ }
+
+ if (!match()) return false;
+ }
+
+ popPath();
+
+ // now check if there were any extra keys in the value (as long as there wasn't a specific list to include)
+ if (match == null) {
+ int skipped = 0;
+ if (skips != null) {
+ for (String skipStr : skips)
+ if (v.containsKey(skipStr)) skipped++;
+ }
+ if (numExpected != (v.size() - skipped)) {
+ HashSet<String> set = new HashSet<>(v.keySet());
+ set.removeAll(expectedMap.keySet());
+ setErr("unexpected map keys " + set);
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ public boolean seek(String seekPath) {
+ if (path == null) return true;
+ if (seekPath.startsWith("/")) {
+ seekPath = seekPath.substring(1);
+ }
+ if (seekPath.endsWith("/")) {
+ seekPath = seekPath.substring(0, seekPath.length() - 1);
+ }
+ List<String> pathList = StrUtils.splitSmart(seekPath, "/", false);
+ return seek(pathList);
+ }
+
+ List asList() {
+ // TODO: handle native arrays
+ if (val instanceof List) {
+ return (List) val;
+ }
+ setErr("expected List");
+ return null;
+ }
+
+ Map<String,Object> asMap() {
+ // TODO: handle NamedList
+ if (val instanceof Map) {
+ return (Map<String,Object>) val;
+ }
+ setErr("expected Map");
+ return null;
+ }
+
+ public boolean seek(List<String> seekPath) {
+ if (seekPath.size() == 0) return true;
+ String seg = seekPath.get(0);
+
+ if (seg.charAt(0) == '[') {
+ List listVal = asList();
+ if (listVal == null) return false;
+
+ int arrIdx = Integer.parseInt(seg.substring(1, seg.length() - 1));
+
+ if (arrIdx >= listVal.size()) return false;
+
+ val = listVal.get(arrIdx);
+ pushPath(arrIdx);
+ } else {
+ Map<String,Object> mapVal = asMap();
+ if (mapVal == null) return false;
+
+ // use containsKey rather than get to handle null values
+ if (!mapVal.containsKey(seg)) return false;
+
+ val = mapVal.get(seg);
+ pushPath(seg);
+ }
+
+ // recurse after removing head of the path
+ return seek(seekPath.subList(1, seekPath.size()));
+ }
+
+}
diff --git a/solr/test-framework/src/java/org/apache/solr/JSONTestUtil.java b/solr/test-framework/src/java/org/apache/solr/JSONTestUtil.java
index cc67bc0..852cee0 100644
--- a/solr/test-framework/src/java/org/apache/solr/JSONTestUtil.java
+++ b/solr/test-framework/src/java/org/apache/solr/JSONTestUtil.java
@@ -133,317 +133,3 @@ public class JSONTestUtil {
}
-/** Tests simple object graphs, like those generated by the noggit JSON parser */
-class CollectionTester {
- public Object valRoot;
- public Object val;
- public Object expectedRoot;
- public Object expected;
- public double delta;
- public List<Object> path;
- public String err;
-
- public CollectionTester(Object val, double delta) {
- this.val = val;
- this.valRoot = val;
- this.delta = delta;
- path = new ArrayList<>();
- }
- public CollectionTester(Object val) {
- this(val, JSONTestUtil.DEFAULT_DELTA);
- }
-
- public String getPath() {
- StringBuilder sb = new StringBuilder();
- boolean first=true;
- for (Object seg : path) {
- if (seg==null) break;
- if (!first) sb.append('/');
- else first=false;
-
- if (seg instanceof Integer) {
- sb.append('[');
- sb.append(seg);
- sb.append(']');
- } else {
- sb.append(seg.toString());
- }
- }
- return sb.toString();
- }
-
- void setPath(Object lastSeg) {
- path.set(path.size()-1, lastSeg);
- }
- Object popPath() {
- return path.remove(path.size()-1);
- }
- void pushPath(Object lastSeg) {
- path.add(lastSeg);
- }
-
- void setErr(String msg) {
- err = msg;
- }
-
- public boolean match(Object expected) {
- this.expectedRoot = expected;
- this.expected = expected;
- return match();
- }
-
- boolean match() {
- if (expected == val) {
- return true;
- }
- if (expected == null || val == null) {
- setErr("mismatch: '" + expected + "'!='" + val + "'");
- return false;
- }
- if (expected instanceof List) {
- return matchList();
- }
- if (expected instanceof Map) {
- return matchMap();
- }
-
- // generic fallback
- if (!expected.equals(val)) {
-
- if (expected instanceof String) {
- String str = (String)expected;
- if (str.length() > 6 && str.startsWith("///") && str.endsWith("///")) {
- return handleSpecialString(str);
- }
- }
-
- // make an exception for some numerics
- if ((expected instanceof Integer && val instanceof Long || expected instanceof Long && val instanceof Integer)
- && ((Number)expected).longValue() == ((Number)val).longValue()) {
- return true;
- } else if ((expected instanceof Double || expected instanceof Float) && (val instanceof Double || val instanceof Float)) {
- double a = ((Number)expected).doubleValue();
- double b = ((Number)val).doubleValue();
- if (Double.compare(a,b) == 0) return true;
- if (Math.abs(a-b) < delta) return true;
- }
- setErr("mismatch: '" + expected + "'!='" + val + "'");
- return false;
- }
-
- // setErr("unknown expected type " + expected.getClass().getName());
- return true;
- }
-
- private boolean handleSpecialString(String str) {
- String code = str.substring(3,str.length()-3);
- if ("ignore".equals(code)) {
- return true;
- } else if (code.startsWith("regex:")) {
- String regex = code.substring("regex:".length());
- if (!(val instanceof String)) {
- setErr("mismatch: '" + expected + "'!='" + val + "', value is not a string");
- return false;
- }
- Pattern pattern = Pattern.compile(regex);
- Matcher matcher = pattern.matcher((String)val);
- if (matcher.find()) {
- return true;
- }
- setErr("mismatch: '" + expected + "'!='" + val + "', regex does not match");
- return false;
- }
-
- setErr("mismatch: '" + expected + "'!='" + val + "'");
- return false;
- }
-
- boolean matchList() {
- List expectedList = (List)expected;
- List v = asList();
- if (v == null) return false;
- int a = 0;
- int b = 0;
- pushPath(null);
- for (;;) {
- if (a >= expectedList.size() && b >=v.size()) {
- break;
- }
-
- if (a >= expectedList.size() || b >=v.size()) {
- popPath();
- setErr("List size mismatch");
- return false;
- }
-
- expected = expectedList.get(a);
- val = v.get(b);
- setPath(b);
- if (!match()) return false;
-
- a++; b++;
- }
-
- popPath();
- return true;
- }
-
- private static Set<String> reserved = new HashSet<>(Arrays.asList("_SKIP_","_MATCH_","_ORDERED_","_UNORDERED_"));
-
- boolean matchMap() {
- Map<String,Object> expectedMap = (Map<String,Object>)expected;
- Map<String,Object> v = asMap();
- if (v == null) return false;
-
- boolean ordered = false;
- String skipList = (String)expectedMap.get("_SKIP_");
- String matchList = (String)expectedMap.get("_MATCH_");
- Object orderedStr = expectedMap.get("_ORDERED_");
- Object unorderedStr = expectedMap.get("_UNORDERED_");
-
- if (orderedStr != null) ordered = true;
- if (unorderedStr != null) ordered = false;
-
- Set<String> match = null;
- if (matchList != null) {
- match = new HashSet(StrUtils.splitSmart(matchList,",",false));
- }
-
- Set<String> skips = null;
- if (skipList != null) {
- skips = new HashSet(StrUtils.splitSmart(skipList,",",false));
- }
-
- Set<String> keys = match != null ? match : expectedMap.keySet();
- Set<String> visited = new HashSet<>();
-
- Iterator<Map.Entry<String,Object>> iter = ordered ? v.entrySet().iterator() : null;
-
- int numExpected=0;
-
- pushPath(null);
- for (String expectedKey : keys) {
- if (reserved.contains(expectedKey)) continue;
- numExpected++;
-
- setPath(expectedKey);
- if (!v.containsKey(expectedKey)) {
- popPath();
- setErr("expected key '" + expectedKey + "'");
- return false;
- }
-
- expected = expectedMap.get(expectedKey);
-
- if (ordered) {
- Map.Entry<String,Object> entry;
- String foundKey;
- for(;;) {
- if (!iter.hasNext()) {
- popPath();
- setErr("expected key '" + expectedKey + "' in ordered map");
- return false;
- }
- entry = iter.next();
- foundKey = entry.getKey();
- if (skips != null && skips.contains(foundKey))continue;
- if (match != null && !match.contains(foundKey)) continue;
- break;
- }
-
- if (!entry.getKey().equals(expectedKey)) {
- popPath();
- setErr("expected key '" + expectedKey + "' instead of '"+entry.getKey()+"' in ordered map");
- return false;
- }
- val = entry.getValue();
- } else {
- if (skips != null && skips.contains(expectedKey)) continue;
- val = v.get(expectedKey);
- }
-
- if (!match()) return false;
- }
-
- popPath();
-
- // now check if there were any extra keys in the value (as long as there wasn't a specific list to include)
- if (match == null) {
- int skipped = 0;
- if (skips != null) {
- for (String skipStr : skips)
- if (v.containsKey(skipStr)) skipped++;
- }
- if (numExpected != (v.size() - skipped)) {
- HashSet<String> set = new HashSet<>(v.keySet());
- set.removeAll(expectedMap.keySet());
- setErr("unexpected map keys " + set);
- return false;
- }
- }
-
- return true;
- }
-
- public boolean seek(String seekPath) {
- if (path == null) return true;
- if (seekPath.startsWith("/")) {
- seekPath = seekPath.substring(1);
- }
- if (seekPath.endsWith("/")) {
- seekPath = seekPath.substring(0,seekPath.length()-1);
- }
- List<String> pathList = StrUtils.splitSmart(seekPath, "/", false);
- return seek(pathList);
- }
-
- List asList() {
- // TODO: handle native arrays
- if (val instanceof List) {
- return (List)val;
- }
- setErr("expected List");
- return null;
- }
-
- Map<String,Object> asMap() {
- // TODO: handle NamedList
- if (val instanceof Map) {
- return (Map<String,Object>)val;
- }
- setErr("expected Map");
- return null;
- }
-
- public boolean seek(List<String> seekPath) {
- if (seekPath.size() == 0) return true;
- String seg = seekPath.get(0);
-
- if (seg.charAt(0)=='[') {
- List listVal = asList();
- if (listVal==null) return false;
-
- int arrIdx = Integer.parseInt(seg.substring(1, seg.length()-1));
-
- if (arrIdx >= listVal.size()) return false;
-
- val = listVal.get(arrIdx);
- pushPath(arrIdx);
- } else {
- Map<String,Object> mapVal = asMap();
- if (mapVal==null) return false;
-
- // use containsKey rather than get to handle null values
- if (!mapVal.containsKey(seg)) return false;
-
- val = mapVal.get(seg);
- pushPath(seg);
- }
-
- // recurse after removing head of the path
- return seek(seekPath.subList(1,seekPath.size()));
- }
-
-
-
-}