You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ds...@apache.org on 2024/07/29 13:29:12 UTC
(solr) branch main updated: Refactor: Move Collection Properties backend out of ZkStateReader (#2585)
This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 57fa8b968b1 Refactor: Move Collection Properties backend out of ZkStateReader (#2585)
57fa8b968b1 is described below
commit 57fa8b968b1f6b82e048362a62906cda1bc153e7
Author: aparnasuresh85 <ap...@salesforce.com>
AuthorDate: Mon Jul 29 09:29:06 2024 -0400
Refactor: Move Collection Properties backend out of ZkStateReader (#2585)
ZkStateReader now delegates to CollectionPropertiesZkStateReader for this functionality.
No API change. No functional / observable change.
ZkStateReader has been doing too much!
---
.../cloud/CollectionPropertiesZkStateReader.java | 413 +++++++++++++++++++++
.../apache/solr/common/cloud/ZkStateReader.java | 349 +----------------
2 files changed, 430 insertions(+), 332 deletions(-)
diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/CollectionPropertiesZkStateReader.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/CollectionPropertiesZkStateReader.java
new file mode 100644
index 00000000000..2bb36116afd
--- /dev/null
+++ b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/CollectionPropertiesZkStateReader.java
@@ -0,0 +1,413 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.common.cloud;
+
+import static java.util.Collections.emptyMap;
+
+import java.io.Closeable;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.ZkStateReader.CollectionWatch;
+import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.common.util.SolrNamedThreadFactory;
+import org.apache.solr.common.util.Utils;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Fetches and manages collection properties from a ZooKeeper ensemble */
+public class CollectionPropertiesZkStateReader implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private volatile boolean closed = false;
+
+ private final SolrZkClient zkClient;
+ private final ZkStateReader zkStateReader;
+
+ /** Collection properties being actively watched */
+ private final ConcurrentHashMap<String, VersionedCollectionProps> watchedCollectionProps =
+ new ConcurrentHashMap<>();
+
+ /**
+ * Manages ZooKeeper watchers for each collection. These watchers monitor changes to the
+ * properties of the collection in ZooKeeper. When a change is detected in ZooKeeper, the watcher
+ * triggers an update, which then notifies the relevant "collectionPropsObserver".
+ */
+ private final ConcurrentHashMap<String, PropsWatcher> collectionPropsWatchers =
+ new ConcurrentHashMap<>();
+
+ /**
+ * Manages a list of observers (listeners) for each collection. These observers need to be
+ * notified when the properties of the collection change. When a collection's properties change,
+ * all registered observers for that collection are notified by a "collectionPropWatcher".
+ */
+ private ConcurrentHashMap<String, CollectionWatch<CollectionPropsWatcher>>
+ collectionPropsObservers = new ConcurrentHashMap<>();
+
+ /** Used to submit notifications to Collection Properties watchers in order */
+ private final ExecutorService collectionPropsNotifications =
+ ExecutorUtil.newMDCAwareSingleThreadExecutor(
+ new SolrNamedThreadFactory("collectionPropsNotifications"));
+
+ private final ExecutorService notifications =
+ ExecutorUtil.newMDCAwareCachedThreadPool("cachecleaner");
+
+ // only kept to identify if the cleaner has already been started.
+ private Future<?> collectionPropsCacheCleaner;
+
+ public CollectionPropertiesZkStateReader(ZkStateReader zkStateReader) {
+ this.zkClient = zkStateReader.getZkClient();
+ this.zkStateReader = zkStateReader;
+ }
+
+ /**
+ * Get and cache collection properties for a given collection. If the collection is watched, or
+ * still cached simply return it from the cache, otherwise fetch it directly from zookeeper and
+ * retain the value for at least cacheForMillis milliseconds. Cached properties are watched in
+ * zookeeper and updated automatically. This version of {@code getCollectionProperties} should be
+ * used when properties need to be consulted frequently in the absence of an active {@link
+ * CollectionPropsWatcher}.
+ *
+ * @param collection The collection for which properties are desired
+ * @param cacheForMillis The minimum number of milliseconds to maintain a cache for the specified
+ * collection's properties. Setting a {@code CollectionPropsWatcher} will override this value
+ * and retain the cache for the life of the watcher. A lack of changes in zookeeper may allow
+ * the caching to remain for a greater duration up to the cycle time of {@code CacheCleaner}.
+ * Passing zero for this value will explicitly remove the cached copy if and only if it is due
+ * to expire and no watch exists. Any positive value will extend the expiration time if
+ * required.
+ * @return a map representing the key/value properties for the collection.
+ */
+ public Map<String, String> getCollectionProperties(final String collection, long cacheForMillis) {
+ synchronized (watchedCollectionProps) { // synchronized on the specific collection
+ Watcher watcher = null;
+ if (cacheForMillis > 0) {
+ watcher =
+ collectionPropsWatchers.compute(
+ collection,
+ (c, w) ->
+ w == null ? new PropsWatcher(c, cacheForMillis) : w.renew(cacheForMillis));
+ }
+ VersionedCollectionProps vprops = watchedCollectionProps.get(collection);
+ boolean haveUnexpiredProps = vprops != null && vprops.cacheUntilNs > System.nanoTime();
+ long untilNs =
+ System.nanoTime() + TimeUnit.NANOSECONDS.convert(cacheForMillis, TimeUnit.MILLISECONDS);
+ Map<String, String> properties;
+ if (haveUnexpiredProps) {
+ properties = vprops.props;
+ vprops.cacheUntilNs = Math.max(vprops.cacheUntilNs, untilNs);
+ } else {
+ try {
+ VersionedCollectionProps vcp = fetchCollectionProperties(collection, watcher);
+ properties = vcp.props;
+ if (cacheForMillis > 0) {
+ vcp.cacheUntilNs = untilNs;
+ watchedCollectionProps.put(collection, vcp);
+ } else {
+ // we're synchronized on watchedCollectionProps and we can only get here if we have
+ // found an expired vprops above, so it is safe to remove the cached value and let the
+ // GC free up some mem a bit sooner.
+ if (!collectionPropsObservers.containsKey(collection)) {
+ watchedCollectionProps.remove(collection);
+ }
+ }
+ } catch (Exception e) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "Error reading collection properties",
+ SolrZkClient.checkInterrupted(e));
+ }
+ }
+ return properties;
+ }
+ }
+
+ @Override
+ public void close() {
+ this.closed = true;
+ notifications.shutdownNow();
+ ExecutorUtil.shutdownAndAwaitTermination(notifications);
+ ExecutorUtil.shutdownAndAwaitTermination(collectionPropsNotifications);
+ }
+
+ private static class VersionedCollectionProps {
+ int zkVersion;
+ Map<String, String> props;
+ long cacheUntilNs = 0;
+
+ VersionedCollectionProps(int zkVersion, Map<String, String> props) {
+ this.zkVersion = zkVersion;
+ this.props = props;
+ }
+ }
+
+ /** Watches collection properties */
+ class PropsWatcher implements Watcher {
+ private final String coll;
+ private long watchUntilNs;
+
+ PropsWatcher(String coll) {
+ this.coll = coll;
+ watchUntilNs = 0;
+ }
+
+ PropsWatcher(String coll, long forMillis) {
+ this.coll = coll;
+ watchUntilNs =
+ System.nanoTime() + TimeUnit.NANOSECONDS.convert(forMillis, TimeUnit.MILLISECONDS);
+ }
+
+ public PropsWatcher renew(long forMillis) {
+ watchUntilNs =
+ System.nanoTime() + TimeUnit.NANOSECONDS.convert(forMillis, TimeUnit.MILLISECONDS);
+ return this;
+ }
+
+ @Override
+ public void process(WatchedEvent event) {
+ // session events are not change events, and do not remove the watcher
+ if (Event.EventType.None.equals(event.getType())) {
+ return;
+ }
+
+ boolean expired = System.nanoTime() > watchUntilNs;
+ if (!collectionPropsObservers.containsKey(coll) && expired) {
+ // No one can be notified of the change, we can ignore it and "unset" the watch
+ log.debug("Ignoring property change for collection {}", coll);
+ return;
+ }
+
+ log.info(
+ "A collection property change: [{}] for collection [{}] has occurred - updating...",
+ event,
+ coll);
+
+ refreshAndWatch(true);
+ }
+
+ /**
+ * Refresh collection properties from ZK and leave a watch for future changes. Updates the
+ * properties in watchedCollectionProps with the results of the refresh. Optionally notifies
+ * watchers
+ */
+ void refreshAndWatch(boolean notifyWatchers) {
+ try {
+ synchronized (watchedCollectionProps) { // making decisions based on the result of a get...
+ VersionedCollectionProps vcp = fetchCollectionProperties(coll, this);
+ Map<String, String> properties = vcp.props;
+ VersionedCollectionProps existingVcp = watchedCollectionProps.get(coll);
+ if (existingVcp == null
+ || // never called before, record what we found
+ vcp.zkVersion > existingVcp.zkVersion
+ || // newer info we should update
+ vcp.zkVersion == -1) { // node was deleted start over
+ watchedCollectionProps.put(coll, vcp);
+ if (notifyWatchers) {
+ notifyPropsWatchers(coll, properties);
+ }
+ if (vcp.zkVersion == -1 && existingVcp != null) { // Collection DELETE detected
+
+ // We should not be caching a collection that has been deleted.
+ watchedCollectionProps.remove(coll);
+
+ // core ref counting not relevant here, don't need canRemove(), we just sent
+ // a notification of an empty set of properties, no reason to watch what doesn't
+ // exist.
+ collectionPropsObservers.remove(coll);
+
+ // This is the one time we know it's safe to throw this out. We just failed to set the
+ // watch due to an NoNodeException, so it isn't held by ZK and can't re-set itself due
+ // to an update.
+ collectionPropsWatchers.remove(coll);
+ }
+ }
+ }
+ } catch (KeeperException.SessionExpiredException
+ | KeeperException.ConnectionLossException e) {
+ log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK: ", e);
+ } catch (KeeperException e) {
+ log.error("Lost collection property watcher for {} due to ZK error", coll, e);
+ throw new ZooKeeperException(
+ SolrException.ErrorCode.SERVER_ERROR, "A ZK error has occurred", e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ log.error(
+ "Lost collection property watcher for {} due to the thread being interrupted", coll, e);
+ }
+ }
+ }
+
+ public void registerCollectionPropsWatcher(
+ final String collection, CollectionPropsWatcher propsWatcher) {
+ AtomicBoolean watchSet = new AtomicBoolean(false);
+ collectionPropsObservers.compute(
+ collection,
+ (k, v) -> {
+ if (v == null) {
+ v = new CollectionWatch<>();
+ watchSet.set(true);
+ }
+ v.stateWatchers.add(propsWatcher);
+ return v;
+ });
+
+ if (watchSet.get()) {
+ collectionPropsWatchers.computeIfAbsent(collection, PropsWatcher::new).refreshAndWatch(false);
+ }
+ }
+
+ protected void refreshCollectionProperties() {
+ collectionPropsObservers.forEach(
+ (k, v) -> {
+ collectionPropsWatchers.computeIfAbsent(k, PropsWatcher::new).refreshAndWatch(true);
+ });
+ }
+
+ public static String getCollectionPropsPath(final String collection) {
+ return ZkStateReader.COLLECTIONS_ZKNODE
+ + '/'
+ + collection
+ + '/'
+ + ZkStateReader.COLLECTION_PROPS_ZKNODE;
+ }
+
+ private VersionedCollectionProps fetchCollectionProperties(String collection, Watcher watcher)
+ throws KeeperException, InterruptedException {
+ final String znodePath = getCollectionPropsPath(collection);
+ // lazy init cache cleaner once we know someone is using collection properties.
+ if (collectionPropsCacheCleaner == null) {
+ synchronized (zkStateReader.getUpdateLock()) { // Double-checked locking
+ if (collectionPropsCacheCleaner == null) {
+ collectionPropsCacheCleaner = notifications.submit(new CacheCleaner());
+ }
+ }
+ }
+ while (true) {
+ try {
+ Stat stat = new Stat();
+ byte[] data = zkClient.getData(znodePath, watcher, stat, true);
+ @SuppressWarnings("unchecked")
+ Map<String, String> props = (Map<String, String>) Utils.fromJSON(data);
+ return new VersionedCollectionProps(stat.getVersion(), props);
+ } catch (ClassCastException e) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "Unable to parse collection properties for collection " + collection,
+ e);
+ } catch (KeeperException.NoNodeException e) {
+ if (watcher != null) {
+ // Leave an exists watch in place in case a collectionprops.json is created later.
+ Stat exists = zkClient.exists(znodePath, watcher, true);
+ if (exists != null) {
+ // Rare race condition, we tried to fetch the data and couldn't find it, then we found
+ // it exists. Loop and try again.
+ continue;
+ }
+ }
+ return new VersionedCollectionProps(-1, emptyMap());
+ }
+ }
+ }
+
+ private void notifyPropsWatchers(String collection, Map<String, String> properties) {
+ try {
+ collectionPropsNotifications.submit(new PropsNotification(collection, properties));
+ } catch (RejectedExecutionException e) {
+ if (!closed) {
+ log.error("Couldn't run collection properties notifications for {}", collection, e);
+ }
+ }
+ }
+
+ private class PropsNotification implements Runnable {
+
+ private final String collection;
+ private final Map<String, String> collectionProperties;
+ private final List<CollectionPropsWatcher> watchers = new ArrayList<>();
+
+ private PropsNotification(String collection, Map<String, String> collectionProperties) {
+ this.collection = collection;
+ this.collectionProperties = collectionProperties;
+ // guarantee delivery of notification regardless of what happens to collectionPropsObservers
+ // while we wait our turn in the executor by capturing the list on creation.
+ collectionPropsObservers.compute(
+ collection,
+ (k, v) -> {
+ if (v == null) return null;
+ watchers.addAll(v.stateWatchers);
+ return v;
+ });
+ }
+
+ @Override
+ public void run() {
+ for (CollectionPropsWatcher watcher : watchers) {
+ if (watcher.onStateChanged(collectionProperties)) {
+ removeCollectionPropsWatcher(collection, watcher);
+ }
+ }
+ }
+ }
+
+ private class CacheCleaner implements Runnable {
+ @Override
+ public void run() {
+ while (!Thread.interrupted()) {
+ try {
+ Thread.sleep(60000);
+ } catch (InterruptedException e) {
+ // Executor shutdown will send us an interrupt
+ break;
+ }
+ watchedCollectionProps
+ .entrySet()
+ .removeIf(
+ entry ->
+ entry.getValue().cacheUntilNs < System.nanoTime()
+ && !collectionPropsObservers.containsKey(entry.getKey()));
+ }
+ }
+ }
+
+ public void removeCollectionPropsWatcher(String collection, CollectionPropsWatcher watcher) {
+ collectionPropsObservers.compute(
+ collection,
+ (k, v) -> {
+ if (v == null) return null;
+ v.stateWatchers.remove(watcher);
+ if (v.canBeRemoved()) {
+ // don't want this to happen in middle of other blocks that might add it back.
+ synchronized (watchedCollectionProps) {
+ watchedCollectionProps.remove(collection);
+ }
+ return null;
+ }
+ return v;
+ });
+ }
+}
diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index b9882ddcc11..c2f76d62de3 100644
--- a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -37,7 +37,6 @@ import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
@@ -58,7 +57,6 @@ import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.util.CommonTestInjection;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.ObjectReleaseTracker;
-import org.apache.solr.common.util.SolrNamedThreadFactory;
import org.apache.solr.common.util.Utils;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.KeeperException.NoNodeException;
@@ -156,6 +154,7 @@ public class ZkStateReader implements SolrCloseable {
public static final String CONTAINER_PLUGINS = "plugin";
public static final String PLACEMENT_PLUGIN = "placement-plugin";
+ private final CollectionPropertiesZkStateReader collectionPropertiesZkStateReader;
/** A view of the current state of all collections. */
protected volatile ClusterState clusterState;
@@ -173,14 +172,6 @@ public class ZkStateReader implements SolrCloseable {
private final ConcurrentHashMap<String, LazyCollectionRef> lazyCollectionStates =
new ConcurrentHashMap<>();
- /** Collection properties being actively watched */
- private final ConcurrentHashMap<String, VersionedCollectionProps> watchedCollectionProps =
- new ConcurrentHashMap<>();
-
- /** Watchers of Collection properties */
- private final ConcurrentHashMap<String, PropsWatcher> collectionPropsWatchers =
- new ConcurrentHashMap<>();
-
private volatile SortedSet<String> liveNodes = emptySortedSet();
private volatile Map<String, Object> clusterProperties = Collections.emptyMap();
@@ -191,11 +182,6 @@ public class ZkStateReader implements SolrCloseable {
*/
private DocCollectionWatches collectionWatches = new DocCollectionWatches();
- // named this observers so there's less confusion between CollectionPropsWatcher map and the
- // PropsWatcher map.
- private ConcurrentHashMap<String, CollectionWatch<CollectionPropsWatcher>>
- collectionPropsObservers = new ConcurrentHashMap<>();
-
private Set<CloudCollectionsListener> cloudCollectionsListeners = ConcurrentHashMap.newKeySet();
private final ExecutorService notifications = ExecutorUtil.newMDCAwareCachedThreadPool("watches");
@@ -204,17 +190,9 @@ public class ZkStateReader implements SolrCloseable {
private Set<ClusterPropertiesListener> clusterPropertiesListeners = ConcurrentHashMap.newKeySet();
- /** Used to submit notifications to Collection Properties watchers in order */
- private final ExecutorService collectionPropsNotifications =
- ExecutorUtil.newMDCAwareSingleThreadExecutor(
- new SolrNamedThreadFactory("collectionPropsNotifications"));
-
private static final long LAZY_CACHE_TIME =
TimeUnit.NANOSECONDS.convert(STATE_UPDATE_DELAY, TimeUnit.MILLISECONDS);
- // only kept to identify if the cleaner has already been started.
- private Future<?> collectionPropsCacheCleaner;
-
/**
* Gets the ZkStateReader inside a ZK based SolrClient.
*
@@ -229,7 +207,7 @@ public class ZkStateReader implements SolrCloseable {
}
}
- private static class CollectionWatch<T> {
+ protected static class CollectionWatch<T> {
int coreRefCount = 0;
Set<T> stateWatchers = ConcurrentHashMap.newKeySet();
@@ -417,6 +395,7 @@ public class ZkStateReader implements SolrCloseable {
this.zkClient = zkClient;
this.closeClient = false;
this.securityNodeWatcher = new SecurityNodeWatcher(this, securityNodeListener);
+ collectionPropertiesZkStateReader = new CollectionPropertiesZkStateReader(this);
assert ObjectReleaseTracker.track(this);
}
@@ -454,7 +433,7 @@ public class ZkStateReader implements SolrCloseable {
this.zkClient = builder.build();
this.closeClient = true;
this.securityNodeWatcher = null;
-
+ collectionPropertiesZkStateReader = new CollectionPropertiesZkStateReader(this);
assert ObjectReleaseTracker.track(this);
}
@@ -592,11 +571,7 @@ public class ZkStateReader implements SolrCloseable {
if (securityNodeWatcher != null) {
securityNodeWatcher.register();
}
-
- collectionPropsObservers.forEach(
- (k, v) -> {
- collectionPropsWatchers.computeIfAbsent(k, PropsWatcher::new).refreshAndWatch(true);
- });
+ collectionPropertiesZkStateReader.refreshCollectionProperties();
} catch (KeeperException.NoNodeException nne) {
throw new SolrException(
ErrorCode.SERVICE_UNAVAILABLE,
@@ -896,6 +871,10 @@ public class ZkStateReader implements SolrCloseable {
return this;
}
+ public SolrZkClient getZKClient() {
+ return zkClient;
+ }
+
@Override
public void close() {
this.closed = true;
@@ -909,7 +888,7 @@ public class ZkStateReader implements SolrCloseable {
});
ExecutorUtil.shutdownAndAwaitTermination(notifications);
- ExecutorUtil.shutdownAndAwaitTermination(collectionPropsNotifications);
+ collectionPropertiesZkStateReader.close();
if (closeClient) {
zkClient.close();
}
@@ -1190,131 +1169,18 @@ public class ZkStateReader implements SolrCloseable {
}
}
- /**
- * Get collection properties for a given collection. If the collection is watched, simply return
- * it from the cache, otherwise fetch it directly from zookeeper. This is a convenience for {@code
- * getCollectionProperties(collection,0)}
- *
- * @param collection the collection for which properties are desired
- * @return a map representing the key/value properties for the collection.
- */
+ /** Get properties for a specific collection */
public Map<String, String> getCollectionProperties(final String collection) {
- return getCollectionProperties(collection, 0);
+ return collectionPropertiesZkStateReader.getCollectionProperties(collection, 0);
}
- /**
- * Get and cache collection properties for a given collection. If the collection is watched, or
- * still cached simply return it from the cache, otherwise fetch it directly from zookeeper and
- * retain the value for at least cacheForMillis milliseconds. Cached properties are watched in
- * zookeeper and updated automatically. This version of {@code getCollectionProperties} should be
- * used when properties need to be consulted frequently in the absence of an active {@link
- * CollectionPropsWatcher}.
- *
- * @param collection The collection for which properties are desired
- * @param cacheForMillis The minimum number of milliseconds to maintain a cache for the specified
- * collection's properties. Setting a {@code CollectionPropsWatcher} will override this value
- * and retain the cache for the life of the watcher. A lack of changes in zookeeper may allow
- * the caching to remain for a greater duration up to the cycle time of {@code CacheCleaner}.
- * Passing zero for this value will explicitly remove the cached copy if and only if it is due
- * to expire and no watch exists. Any positive value will extend the expiration time if
- * required.
- * @return a map representing the key/value properties for the collection.
- */
+ /** Get and cache collection properties for a given collection */
public Map<String, String> getCollectionProperties(final String collection, long cacheForMillis) {
- synchronized (watchedCollectionProps) { // making decisions based on the result of a get...
- Watcher watcher = null;
- if (cacheForMillis > 0) {
- watcher =
- collectionPropsWatchers.compute(
- collection,
- (c, w) ->
- w == null ? new PropsWatcher(c, cacheForMillis) : w.renew(cacheForMillis));
- }
- VersionedCollectionProps vprops = watchedCollectionProps.get(collection);
- boolean haveUnexpiredProps = vprops != null && vprops.cacheUntilNs > System.nanoTime();
- long untilNs =
- System.nanoTime() + TimeUnit.NANOSECONDS.convert(cacheForMillis, TimeUnit.MILLISECONDS);
- Map<String, String> properties;
- if (haveUnexpiredProps) {
- properties = vprops.props;
- vprops.cacheUntilNs = Math.max(vprops.cacheUntilNs, untilNs);
- } else {
- try {
- VersionedCollectionProps vcp = fetchCollectionProperties(collection, watcher);
- properties = vcp.props;
- if (cacheForMillis > 0) {
- vcp.cacheUntilNs = untilNs;
- watchedCollectionProps.put(collection, vcp);
- } else {
- // we're synchronized on watchedCollectionProps and we can only get here if we have
- // found an expired vprops above, so it is safe to remove the cached value and let the
- // GC free up some mem a bit sooner.
- if (!collectionPropsObservers.containsKey(collection)) {
- watchedCollectionProps.remove(collection);
- }
- }
- } catch (Exception e) {
- throw new SolrException(
- ErrorCode.SERVER_ERROR,
- "Error reading collection properties",
- SolrZkClient.checkInterrupted(e));
- }
- }
- return properties;
- }
- }
-
- private static class VersionedCollectionProps {
- int zkVersion;
- Map<String, String> props;
- long cacheUntilNs = 0;
-
- VersionedCollectionProps(int zkVersion, Map<String, String> props) {
- this.zkVersion = zkVersion;
- this.props = props;
- }
+ return collectionPropertiesZkStateReader.getCollectionProperties(collection, cacheForMillis);
}
static String getCollectionPropsPath(final String collection) {
- return COLLECTIONS_ZKNODE + '/' + collection + '/' + COLLECTION_PROPS_ZKNODE;
- }
-
- private VersionedCollectionProps fetchCollectionProperties(String collection, Watcher watcher)
- throws KeeperException, InterruptedException {
- final String znodePath = getCollectionPropsPath(collection);
- // lazy init cache cleaner once we know someone is using collection properties.
- if (collectionPropsCacheCleaner == null) {
- synchronized (this) { // There can be only one! :)
- if (collectionPropsCacheCleaner == null) {
- collectionPropsCacheCleaner = notifications.submit(new CacheCleaner());
- }
- }
- }
- while (true) {
- try {
- Stat stat = new Stat();
- byte[] data = zkClient.getData(znodePath, watcher, stat, true);
- @SuppressWarnings("unchecked")
- Map<String, String> props = (Map<String, String>) Utils.fromJSON(data);
- return new VersionedCollectionProps(stat.getVersion(), props);
- } catch (ClassCastException e) {
- throw new SolrException(
- ErrorCode.SERVER_ERROR,
- "Unable to parse collection properties for collection " + collection,
- e);
- } catch (KeeperException.NoNodeException e) {
- if (watcher != null) {
- // Leave an exists watch in place in case a collectionprops.json is created later.
- Stat exists = zkClient.exists(znodePath, watcher, true);
- if (exists != null) {
- // Rare race condition, we tried to fetch the data and couldn't find it, then we found
- // it exists. Loop and try again.
- continue;
- }
- }
- return new VersionedCollectionProps(-1, emptyMap());
- }
- }
+ return CollectionPropertiesZkStateReader.getCollectionPropsPath(collection);
}
/**
@@ -1453,101 +1319,6 @@ public class ZkStateReader implements SolrCloseable {
}
}
- /** Watches collection properties */
- class PropsWatcher implements Watcher {
- private final String coll;
- private long watchUntilNs;
-
- PropsWatcher(String coll) {
- this.coll = coll;
- watchUntilNs = 0;
- }
-
- PropsWatcher(String coll, long forMillis) {
- this.coll = coll;
- watchUntilNs =
- System.nanoTime() + TimeUnit.NANOSECONDS.convert(forMillis, TimeUnit.MILLISECONDS);
- }
-
- public PropsWatcher renew(long forMillis) {
- watchUntilNs =
- System.nanoTime() + TimeUnit.NANOSECONDS.convert(forMillis, TimeUnit.MILLISECONDS);
- return this;
- }
-
- @Override
- public void process(WatchedEvent event) {
- // session events are not change events, and do not remove the watcher
- if (EventType.None.equals(event.getType())) {
- return;
- }
-
- boolean expired = System.nanoTime() > watchUntilNs;
- if (!collectionPropsObservers.containsKey(coll) && expired) {
- // No one can be notified of the change, we can ignore it and "unset" the watch
- log.debug("Ignoring property change for collection {}", coll);
- return;
- }
-
- log.info(
- "A collection property change: [{}] for collection [{}] has occurred - updating...",
- event,
- coll);
-
- refreshAndWatch(true);
- }
-
- /**
- * Refresh collection properties from ZK and leave a watch for future changes. Updates the
- * properties in watchedCollectionProps with the results of the refresh. Optionally notifies
- * watchers
- */
- void refreshAndWatch(boolean notifyWatchers) {
- try {
- synchronized (watchedCollectionProps) { // making decisions based on the result of a get...
- VersionedCollectionProps vcp = fetchCollectionProperties(coll, this);
- Map<String, String> properties = vcp.props;
- VersionedCollectionProps existingVcp = watchedCollectionProps.get(coll);
- if (existingVcp == null
- || // never called before, record what we found
- vcp.zkVersion > existingVcp.zkVersion
- || // newer info we should update
- vcp.zkVersion == -1) { // node was deleted start over
- watchedCollectionProps.put(coll, vcp);
- if (notifyWatchers) {
- notifyPropsWatchers(coll, properties);
- }
- if (vcp.zkVersion == -1 && existingVcp != null) { // Collection DELETE detected
-
- // We should not be caching a collection that has been deleted.
- watchedCollectionProps.remove(coll);
-
- // core ref counting not relevant here, don't need canRemove(), we just sent
- // a notification of an empty set of properties, no reason to watch what doesn't
- // exist.
- collectionPropsObservers.remove(coll);
-
- // This is the one time we know it's safe to throw this out. We just failed to set the
- // watch due to an NoNodeException, so it isn't held by ZK and can't re-set itself due
- // to an update.
- collectionPropsWatchers.remove(coll);
- }
- }
- }
- } catch (KeeperException.SessionExpiredException
- | KeeperException.ConnectionLossException e) {
- log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK: ", e);
- } catch (KeeperException e) {
- log.error("Lost collection property watcher for {} due to ZK error", coll, e);
- throw new ZooKeeperException(ErrorCode.SERVER_ERROR, "A ZK error has occurred", e);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- log.error(
- "Lost collection property watcher for {} due to the thread being interrupted", coll, e);
- }
- }
- }
-
/** Watches /collections children . */
class CollectionsChildWatcher implements Watcher {
@@ -2069,38 +1840,12 @@ public class ZkStateReader implements SolrCloseable {
public void registerCollectionPropsWatcher(
final String collection, CollectionPropsWatcher propsWatcher) {
- AtomicBoolean watchSet = new AtomicBoolean(false);
- collectionPropsObservers.compute(
- collection,
- (k, v) -> {
- if (v == null) {
- v = new CollectionWatch<>();
- watchSet.set(true);
- }
- v.stateWatchers.add(propsWatcher);
- return v;
- });
- if (watchSet.get()) {
- collectionPropsWatchers.computeIfAbsent(collection, PropsWatcher::new).refreshAndWatch(false);
- }
+ collectionPropertiesZkStateReader.registerCollectionPropsWatcher(collection, propsWatcher);
}
public void removeCollectionPropsWatcher(String collection, CollectionPropsWatcher watcher) {
- collectionPropsObservers.compute(
- collection,
- (k, v) -> {
- if (v == null) return null;
- v.stateWatchers.remove(watcher);
- if (v.canBeRemoved()) {
- // don't want this to happen in middle of other blocks that might add it back.
- synchronized (watchedCollectionProps) {
- watchedCollectionProps.remove(collection);
- }
- return null;
- }
- return v;
- });
+ collectionPropertiesZkStateReader.removeCollectionPropsWatcher(collection, watcher);
}
public static class ConfigData {
@@ -2352,66 +2097,6 @@ public class ZkStateReader implements SolrCloseable {
}
}
- private void notifyPropsWatchers(String collection, Map<String, String> properties) {
- try {
- collectionPropsNotifications.submit(new PropsNotification(collection, properties));
- } catch (RejectedExecutionException e) {
- if (!closed) {
- log.error("Couldn't run collection properties notifications for {}", collection, e);
- }
- }
- }
-
- private class PropsNotification implements Runnable {
-
- private final String collection;
- private final Map<String, String> collectionProperties;
- private final List<CollectionPropsWatcher> watchers = new ArrayList<>();
-
- private PropsNotification(String collection, Map<String, String> collectionProperties) {
- this.collection = collection;
- this.collectionProperties = collectionProperties;
- // guarantee delivery of notification regardless of what happens to collectionPropsObservers
- // while we wait our turn in the executor by capturing the list on creation.
- collectionPropsObservers.compute(
- collection,
- (k, v) -> {
- if (v == null) return null;
- watchers.addAll(v.stateWatchers);
- return v;
- });
- }
-
- @Override
- public void run() {
- for (CollectionPropsWatcher watcher : watchers) {
- if (watcher.onStateChanged(collectionProperties)) {
- removeCollectionPropsWatcher(collection, watcher);
- }
- }
- }
- }
-
- private class CacheCleaner implements Runnable {
- @Override
- public void run() {
- while (!Thread.interrupted()) {
- try {
- Thread.sleep(60000);
- } catch (InterruptedException e) {
- // Executor shutdown will send us an interrupt
- break;
- }
- watchedCollectionProps
- .entrySet()
- .removeIf(
- entry ->
- entry.getValue().cacheUntilNs < System.nanoTime()
- && !collectionPropsObservers.containsKey(entry.getKey()));
- }
- }
- }
-
/**
* Helper class that acts as both a {@link DocCollectionWatcher} and a {@link LiveNodesListener}
* while wraping and delegating to a {@link CollectionStateWatcher}