You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2021/03/04 15:41:34 UTC
[lucene-solr] branch reference_impl_dev updated: @1422 Address a
few more things - I still lost something, but a little back.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/reference_impl_dev by this push:
new 86cc0b6 @1422 Address a few more things - I still lost something, but a little back.
86cc0b6 is described below
commit 86cc0b6111b9cc2f96287616d399e0297866d62e
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Thu Mar 4 09:40:45 2021 -0600
@1422 Address a few more things - I still lost something, but a little back.
Took 2 hours 8 minutes
---
.../org/apache/solr/cloud/OverseerTaskQueue.java | 44 ++++-------------
.../java/org/apache/solr/cloud/ZkController.java | 57 ++++++++++++----------
.../src/java/org/apache/solr/core/SolrCore.java | 9 ++--
.../apache/solr/cloud/CollectionsAPISolrJTest.java | 3 +-
.../apache/solr/common/cloud/ZkStateReader.java | 2 +-
.../src/java/org/apache/solr/SolrTestUtil.java | 2 +-
.../src/resources/logconf/log4j2-std.xml | 8 ++-
7 files changed, 52 insertions(+), 73 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskQueue.java b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskQueue.java
index fb50d23..e920dd3 100644
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskQueue.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskQueue.java
@@ -29,7 +29,6 @@ import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -134,7 +133,7 @@ public class OverseerTaskQueue extends ZkDistributedQueue {
private volatile WatchedEvent event;
LatchWatcher(String path, SolrZkClient zkClient) {
- this.lock = new ReentrantLock();
+ this.lock = new ReentrantLock(true);
this.eventReceived = lock.newCondition();
this.path = path;
this.zkClient = zkClient;
@@ -151,37 +150,24 @@ public class OverseerTaskQueue extends ZkDistributedQueue {
if (log.isDebugEnabled()) log.debug("{} fired on path {} state {} latchEventType {}", event.getType(), event.getPath(), event.getState());
- checkResult();
- }
+ this.event = event;
- private void checkResult() {
+ lock.lock();
try {
- Stat stat = zkClient.exists(path, null, true);
- if (stat != null && stat.getDataLength() > 0) {
- lock.lock();
- try {
- this.event = new WatchedEvent(Event.EventType.NodeDataChanged, Event.KeeperState.SyncConnected, path);
- eventReceived.signalAll();
- } finally {
- lock.unlock();
- }
- }
- } catch (Exception e) {
- log.error("", e);
+ eventReceived.signalAll();
+ } finally {
+ lock.unlock();
}
}
public void await(long timeoutMs) {
- createWatch();
-
- checkResult();
-
if (event != null) {
- close();
return;
}
+ createWatch();
+
TimeOut timeout = new TimeOut(timeoutMs, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME);
lock.lock();
try {
@@ -198,7 +184,6 @@ public class OverseerTaskQueue extends ZkDistributedQueue {
}
} finally {
lock.unlock();
- close();
}
}
@@ -234,18 +219,7 @@ public class OverseerTaskQueue extends ZkDistributedQueue {
private String createData(String path, byte[] data, CreateMode mode)
throws KeeperException, InterruptedException {
for (;;) {
- try {
- return zookeeper.create(path, data, mode, true);
- } catch (KeeperException.NodeExistsException e) {
- log.warn("Found request node already, waiting to see if it frees up ...");
- // TODO: use a watch?
- Thread.sleep(50);
- try {
- return zookeeper.create(path, data, mode, true);
- } catch (KeeperException.NodeExistsException ne) {
- // someone created it
- }
- }
+ return zookeeper.create(path, data, mode, true);
}
}
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index ca16fcb..48182fd 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -2083,25 +2083,27 @@ public class ZkController implements Closeable, Runnable {
}
private void unregisterConfListener(String confDir, Runnable listener) {
- final ConfListeners confListeners = confDirectoryListeners.get(confDir);
- if (confListeners == null) {
- log.warn("{} has no more registered listeners, but a live one attempted to unregister!", confDir);
- return;
- }
- if (confListeners.confDirListeners.remove(listener)) {
- if (log.isDebugEnabled()) log.debug("removed listener for config directory [{}]", confDir);
- }
- if (confListeners.confDirListeners.isEmpty()) {
- // no more listeners for this confDir, remove it from the map
- if (log.isDebugEnabled()) log.debug("No more listeners for config directory [{}]", confDir);
- try {
- zkClient.removeWatches(COLLECTIONS_ZKNODE, confListeners.watcher, Watcher.WatcherType.Any, true);
- } catch (KeeperException.NoWatcherException e) {
+ synchronized (confDirectoryListeners) {
+ final ConfListeners confListeners = confDirectoryListeners.get(confDir);
+ if (confListeners == null) {
+ log.warn("{} has no more registered listeners, but a live one attempted to unregister!", confDir);
+ return;
+ }
+ if (confListeners.confDirListeners.remove(listener)) {
+ if (log.isDebugEnabled()) log.debug("removed listener for config directory [{}]", confDir);
+ }
+ if (confListeners.confDirListeners.isEmpty()) {
+ confDirectoryListeners.remove(confDir);
+ // no more listeners for this confDir, remove it from the map
+ if (log.isDebugEnabled()) log.debug("No more listeners for config directory [{}]", confDir);
+ try {
+ zkClient.removeWatches(COLLECTIONS_ZKNODE, confListeners.watcher, Watcher.WatcherType.Any, true);
+ } catch (KeeperException.NoWatcherException e) {
- } catch (Exception e) {
- log.info("could not remove watch {} {}", e.getClass().getSimpleName(), e.getMessage());
+ } catch (Exception e) {
+ log.info("could not remove watch {} {}", e.getClass().getSimpleName(), e.getMessage());
+ }
}
- confDirectoryListeners.remove(confDir);
}
}
@@ -2115,6 +2117,7 @@ public class ZkController implements Closeable, Runnable {
if (listener == null) {
throw new NullPointerException("listener cannot be null");
}
+
final ConfListeners confDirListeners = getConfDirListeners(confDir);
confDirListeners.confDirListeners.add(listener);
core.addCloseHook(new CloseHook() {
@@ -2141,15 +2144,17 @@ public class ZkController implements Closeable, Runnable {
}
private ConfListeners getConfDirListeners(final String confDir) {
- ConfListeners confDirListeners = confDirectoryListeners.get(confDir);
- if (confDirListeners == null) {
- if (log.isDebugEnabled()) log.debug("watch zkdir {}" , confDir);
- ConfDirWatcher watcher = new ConfDirWatcher(confDir, cc, confDirectoryListeners);
- confDirListeners = new ConfListeners(ConcurrentHashMap.newKeySet(), watcher);
- confDirectoryListeners.put(confDir, confDirListeners);
- setConfWatcher(confDir, watcher, null, cc, confDirectoryListeners, cc.getZkController().getZkClient());
+ synchronized (confDirectoryListeners) {
+ ConfListeners confDirListeners = confDirectoryListeners.get(confDir);
+ if (confDirListeners == null) {
+ if (log.isTraceEnabled()) log.trace("watch zkdir {}", confDir);
+ ConfDirWatcher watcher = new ConfDirWatcher(confDir, cc, confDirectoryListeners);
+ confDirListeners = new ConfListeners(ConcurrentHashMap.newKeySet(), watcher);
+ confDirectoryListeners.put(confDir, confDirListeners);
+ setConfWatcher(confDir, watcher, null, cc, confDirectoryListeners, cc.getZkController().getZkClient());
+ }
+ return confDirListeners;
}
- return confDirListeners;
}
private final Map<String, ConfListeners> confDirectoryListeners = new ConcurrentHashMap<>();
@@ -2206,7 +2211,7 @@ public class ZkController implements Closeable, Runnable {
if (cc.isShutDown() || cc.getZkController().isDcCalled()) {
return false;
}
- listeners.forEach(runnable -> cc.coreContainerExecutor.submit(runnable));
+ listeners.forEach(runnable -> ParWork.getRootSharedExecutor().submit(runnable));
}
return true;
}
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index 39e8a49..1e707b7 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -446,9 +446,6 @@ public final class SolrCore implements SolrInfoBean, Closeable {
private String getIndexPropertyFromPropFile(Directory dir) throws IOException {
IndexInput input;
- if (!Arrays.asList(dir.listAll()).contains(IndexFetcher.INDEX_PROPERTIES)) {
- return dataDir + "index/";
- }
try {
input = dir.openInput(IndexFetcher.INDEX_PROPERTIES, IOContext.DEFAULT);
} catch (FileNotFoundException | NoSuchFileException e) {
@@ -1788,7 +1785,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
return;
}
- log.info("CLOSING SolrCore {}", logid);
+ if (log.isDebugEnabled()) log.debug("CLOSING SolrCore {}", logid);
assert ObjectReleaseTracker.release(this);
searcherReadyLatch.countDown();
@@ -2928,7 +2925,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
}
public void closeSearcher () {
- log.info("{} Closing main searcher on request realtimeSearcher={} searcher={}", logid, realtimeSearcher, _searcher);
+ log.debug("{} Closing main searcher on request realtimeSearcher={} searcher={}", logid, realtimeSearcher, _searcher);
searcherLock.lock();
try {
if (realtimeSearcher != null) {
@@ -3452,7 +3449,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
}
//some files in conf directory may have other than managedschema, overlay, params
- try (ParWork worker = new ParWork("ConfListeners", true, false)) {
+ try (ParWork worker = new ParWork("ConfListeners", true, true)) {
if (cc.isShutDown()) return;
core.confListeners.forEach(runnable -> {
diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
index e74bb79..6685d55 100644
--- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
@@ -241,10 +241,9 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {
}
@Test
- @Ignore // MRM TODO: - testing large numbers
public void testCreateAndDeleteCollection() throws Exception {
String collectionName = "solrj_test";
- CollectionAdminResponse response = CollectionAdminRequest.createCollection(collectionName, "conf", 36, 36) // 24 * 24 = 576
+ CollectionAdminResponse response = CollectionAdminRequest.createCollection(collectionName, "conf", 36, 36) // 36 * 36 = 1296
.process(cluster.getSolrClient());
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index 20a018d..328185a 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -2456,7 +2456,7 @@ public class ZkStateReader implements SolrCloseable, Replica.NodeNameToBaseUrl {
try (ParWork work = new ParWork(this)) {
watchers.stateWatchers.forEach(watcher -> {
// work.collect("", () -> {
- if (log.isDebugEnabled()) log.debug("Notify DocCollectionWatcher {} {}", watcher, collectionState);
+ if (log.isTraceEnabled()) log.debug("Notify DocCollectionWatcher {} {}", watcher, collectionState);
try {
if (watcher.onStateChanged(collectionState)) {
removeDocCollectionWatcher(collection, watcher);
diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestUtil.java b/solr/test-framework/src/java/org/apache/solr/SolrTestUtil.java
index a73ccc4..7a8108c 100644
--- a/solr/test-framework/src/java/org/apache/solr/SolrTestUtil.java
+++ b/solr/test-framework/src/java/org/apache/solr/SolrTestUtil.java
@@ -170,7 +170,7 @@ public class SolrTestUtil {
}
public static void wait(Thread thread) {
- if (thread.getName().contains("ForkJoinPool.") || thread.getName().contains("Log4j2-")) {
+ if ((thread.getName().contains("ForkJoinPool.") || thread.getName().contains("Log4j2-")) && thread.getState() != Thread.State.TERMINATED) {
log.info("Dont wait on ForkJoinPool. or Log4j2-");
return;
}
diff --git a/solr/test-framework/src/resources/logconf/log4j2-std.xml b/solr/test-framework/src/resources/logconf/log4j2-std.xml
index 0dae120..50eed54 100644
--- a/solr/test-framework/src/resources/logconf/log4j2-std.xml
+++ b/solr/test-framework/src/resources/logconf/log4j2-std.xml
@@ -28,9 +28,13 @@
=>%ex{short}}}{10240}%n"/>
</Console>
- <File name="FILE" fileName="${sys:user.home}/solr-test.log" immediateFlush="false" append="false">
+ <File name="FILE_COLOR" fileName="${sys:user.home}/solr-test.log" immediateFlush="false" append="false">
<PatternLayout pattern="%style{%-4r}{yellow} %highlight{%maxLen{%-5p}{6}} %style{(%maxLen{%t}{8})}{yellow,bold} [%style{%X{node_name} %X{core}}{cyan}] %style{%c{1.}}{cyan} %highlight{%m %notEmpty{%ex}}\n"/>
</File>
+ <File name="FILE" fileName="${sys:user.home}/solr-test.log" immediateFlush="false" append="false">
+ <PatternLayout pattern="%maxLen{%-4r %-5p (%t) [%X{node_name} %X{collection} %X{shard} %X{replica} %X{core} %X{trace_id}] %c{1.} %m%notEmpty{
+ =>%ex{short}}}{10240}%n"/>
+ </File>
<Null name="null" />
@@ -79,7 +83,7 @@
<AsyncLogger name="org.apache.solr.client.solrj.impl.Http2SolrClient" level="INFO"/>
<AsyncRoot level="INFO">
- <AppenderRef ref="STDERR"/>
+ <AppenderRef ref="FILE"/>
</AsyncRoot>
</Loggers>
</Configuration>