You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2020/08/20 14:27:21 UTC

[lucene-solr] branch jira/solr-14749 updated: SOLR-14749: Factor out the Scheduler to a separate API. Initial attempt to implement AutoAddReplicasEventListener.

This is an automated email from the ASF dual-hosted git repository.

ab pushed a commit to branch jira/solr-14749
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git


The following commit(s) were added to refs/heads/jira/solr-14749 by this push:
     new 7297eda  SOLR-14749: Factor out the Scheduler to a separate API. Initial attempt to implement AutoAddReplicasEventListener.
7297eda is described below

commit 7297eda46b97151b4b1cf22fd63ba90d1d0a1294
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Thu Aug 20 16:26:21 2020 +0200

    SOLR-14749: Factor out the Scheduler to a separate API. Initial attempt
    to implement AutoAddReplicasEventListener.
---
 .../src/java/org/apache/solr/cloud/Overseer.java   |  37 +++++++
 .../apache/solr/cluster/events/ClusterEvent.java   |   1 -
 .../events/impl/AutoAddReplicasEventListener.java  | 102 +++++++++++++++++
 .../events/impl/ClusterEventProducerImpl.java      |  90 +--------------
 .../Schedulable.java}                              |  11 +-
 .../cluster/{events => scheduler}/Schedule.java    |   2 +-
 .../SolrScheduler.java}                            |  12 +-
 .../impl/CompiledSchedule.java                     |   4 +-
 .../cluster/scheduler/impl/SolrSchedulerImpl.java  | 121 +++++++++++++++++++++
 9 files changed, 273 insertions(+), 107 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
index bb405ad..4c5c5d2 100644
--- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java
+++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
@@ -47,6 +47,7 @@ import org.apache.solr.cloud.overseer.ReplicaMutator;
 import org.apache.solr.cloud.overseer.SliceMutator;
 import org.apache.solr.cloud.overseer.ZkStateWriter;
 import org.apache.solr.cloud.overseer.ZkWriteCommand;
+import org.apache.solr.cluster.events.ClusterSingleton;
 import org.apache.solr.common.AlreadyClosedException;
 import org.apache.solr.common.SolrCloseable;
 import org.apache.solr.common.SolrException;
@@ -66,9 +67,11 @@ import org.apache.solr.common.util.Pair;
 import org.apache.solr.common.util.Utils;
 import org.apache.solr.core.CloudConfig;
 import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.PluginBag;
 import org.apache.solr.handler.admin.CollectionsHandler;
 import org.apache.solr.handler.component.HttpShardHandler;
 import org.apache.solr.logging.MDCLoggingContext;
+import org.apache.solr.request.SolrRequestHandler;
 import org.apache.solr.update.UpdateShardHandler;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
@@ -656,6 +659,8 @@ public class Overseer implements SolrCloseable {
       }
     });
 
+    startClusterSingletons();
+
     assert ObjectReleaseTracker.track(this);
   }
 
@@ -775,6 +780,36 @@ public class Overseer implements SolrCloseable {
     }
   }
 
+  /**
+   * Start {@link ClusterSingleton} plugins when we become the leader.
+   */
+  private void startClusterSingletons() {
+    PluginBag<SolrRequestHandler> handlers = getCoreContainer().getRequestHandlers();
+    handlers.keySet().forEach(handlerName -> {
+      SolrRequestHandler handler = handlers.get(handlerName);
+      if (handler instanceof ClusterSingleton) {
+        try {
+          ((ClusterSingleton) handler).start();
+        } catch (Exception e) {
+          log.warn("Exception starting ClusterSingleton " + handler, e);
+        }
+      }
+    });
+  }
+
+  /**
+   * Stop {@link ClusterSingleton} plugins when we lose leadership.
+   */
+  private void stopClusterSingletons() {
+    PluginBag<SolrRequestHandler> handlers = getCoreContainer().getRequestHandlers();
+    handlers.keySet().forEach(handlerName -> {
+      SolrRequestHandler handler = handlers.get(handlerName);
+      if (handler instanceof ClusterSingleton) {
+        ((ClusterSingleton) handler).stop();
+      }
+    });
+  }
+
   public Stats getStats() {
     return stats;
   }
@@ -817,6 +852,8 @@ public class Overseer implements SolrCloseable {
     this.closed = true;
     doClose();
 
+    stopClusterSingletons();
+
     assert ObjectReleaseTracker.release(this);
   }
 
diff --git a/solr/core/src/java/org/apache/solr/cluster/events/ClusterEvent.java b/solr/core/src/java/org/apache/solr/cluster/events/ClusterEvent.java
index 59d1597..ee8d955 100644
--- a/solr/core/src/java/org/apache/solr/cluster/events/ClusterEvent.java
+++ b/solr/core/src/java/org/apache/solr/cluster/events/ClusterEvent.java
@@ -27,7 +27,6 @@ public interface ClusterEvent {
     NODE_DOWN,
     NODE_UP,
     REPLICA_DOWN,
-    SCHEDULED,
     // other types? eg. Overseer leader change, shard leader change,
     // node overload (eg. CPU / MEM circuit breakers tripped)?
   }
diff --git a/solr/core/src/java/org/apache/solr/cluster/events/impl/AutoAddReplicasEventListener.java b/solr/core/src/java/org/apache/solr/cluster/events/impl/AutoAddReplicasEventListener.java
new file mode 100644
index 0000000..ea15f28
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/cluster/events/impl/AutoAddReplicasEventListener.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.cluster.events.impl;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.solr.client.solrj.io.SolrClientCache;
+import org.apache.solr.cluster.events.ClusterEvent;
+import org.apache.solr.cluster.events.ClusterEventListener;
+import org.apache.solr.cluster.events.ClusterSingleton;
+import org.apache.solr.core.CoreContainer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public class AutoAddReplicasEventListener implements ClusterSingleton, ClusterEventListener {
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  private static final Set<ClusterEvent.EventType> EVENT_TYPES = new HashSet<>(
+      Arrays.asList(
+          ClusterEvent.EventType.NODE_DOWN,
+          ClusterEvent.EventType.REPLICA_DOWN
+      ));
+
+  private final CoreContainer cc;
+  private final SolrClientCache solrClientCache;
+
+  private boolean running = false;
+
+  public AutoAddReplicasEventListener(CoreContainer cc) {
+    this.cc = cc;
+    this.solrClientCache = cc.getSolrClientCache();
+  }
+
+  @Override
+  public Set<ClusterEvent.EventType> getEventTypes() {
+    return EVENT_TYPES;
+  }
+
+  @Override
+  public void onEvent(ClusterEvent event) {
+    if (!isRunning()) {
+      // ignore the event
+      return;
+    }
+    switch (event.getType()) {
+      case NODE_DOWN:
+        handleNodeDown(event);
+        break;
+      case NODE_UP:
+        // ignore?
+        break;
+      case REPLICA_DOWN:
+        handleReplicaDown(event);
+        break;
+      default:
+        log.warn("Unsupported event {}, ignoring...", event);
+    }
+  }
+
+  private void handleNodeDown(ClusterEvent event) {
+    // send MOVEREPLICA admin requests for all replicas from that node
+  }
+
+  private void handleReplicaDown(ClusterEvent event) {
+    // send ADDREPLICA admin request
+  }
+
+  @Override
+  public void start() throws Exception {
+    running = true;
+  }
+
+  @Override
+  public boolean isRunning() {
+    return running;
+  }
+
+  @Override
+  public void stop() {
+    running = false;
+  }
+}
diff --git a/solr/core/src/java/org/apache/solr/cluster/events/impl/ClusterEventProducerImpl.java b/solr/core/src/java/org/apache/solr/cluster/events/impl/ClusterEventProducerImpl.java
index 0bcff77..53389d4 100644
--- a/solr/core/src/java/org/apache/solr/cluster/events/impl/ClusterEventProducerImpl.java
+++ b/solr/core/src/java/org/apache/solr/cluster/events/impl/ClusterEventProducerImpl.java
@@ -20,15 +20,10 @@ import java.io.Closeable;
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
 import java.time.Instant;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
 
 import org.apache.solr.cloud.ZkController;
 import org.apache.solr.cluster.events.ClusterEvent;
@@ -37,11 +32,7 @@ import org.apache.solr.cluster.events.ClusterEventProducer;
 import org.apache.solr.cluster.events.ClusterSingleton;
 import org.apache.solr.cluster.events.NodeDownEvent;
 import org.apache.solr.cluster.events.NodeUpEvent;
-import org.apache.solr.cluster.events.Schedule;
-import org.apache.solr.cluster.events.ScheduledEvent;
-import org.apache.solr.cluster.events.ScheduledEventListener;
 import org.apache.solr.common.cloud.LiveNodesListener;
-import org.apache.solr.common.util.SolrNamedThreadFactory;
 import org.apache.solr.core.CoreContainer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -52,29 +43,20 @@ import org.slf4j.LoggerFactory;
  * <p>For each cluster event relevant listeners are always invoked sequentially
  * (not in parallel) and in arbitrary order. This means that if any listener blocks the
  * processing other listeners may be invoked much later or not at all.</p>
- * <p>Scheduled events are triggered at most with {@link #SCHEDULE_INTERVAL_SEC} interval. See also above note
- * on the sequential processing. If the total time of execution exceeds any schedule
- * interval such events will be silently missed and they will be invoked only when the
- * next event will be generated.</p>
  */
 public class ClusterEventProducerImpl implements ClusterEventProducer, ClusterSingleton, Closeable {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
-  public static final int SCHEDULE_INTERVAL_SEC = 10;
-
   private final Map<ClusterEvent.EventType, Set<ClusterEventListener>> listeners = new HashMap<>();
-  private final Map<String, CompiledSchedule> schedules = new ConcurrentHashMap<>();
   private final CoreContainer cc;
   private LiveNodesListener liveNodesListener;
   private ZkController zkController;
-  private ScheduledExecutorService scheduler;
   private boolean running;
 
   private final Set<ClusterEvent.EventType> supportedEvents =
       new HashSet<>() {{
         add(ClusterEvent.EventType.NODE_DOWN);
         add(ClusterEvent.EventType.NODE_UP);
-        add(ClusterEvent.EventType.SCHEDULED);
       }};
 
   private volatile boolean isClosed = false;
@@ -86,10 +68,9 @@ public class ClusterEventProducerImpl implements ClusterEventProducer, ClusterSi
 
   // ClusterSingleton lifecycle methods
   @Override
-  public void start() throws Exception {
+  public void start() {
     if (zkController == null) {
       liveNodesListener = null;
-      scheduler = null;
       return;
     }
 
@@ -143,11 +124,6 @@ public class ClusterEventProducerImpl implements ClusterEventProducer, ClusterSi
 
     // XXX register collection state listener?
     // XXX not sure how to efficiently monitor for REPLICA_DOWN events
-
-    // create scheduler
-    scheduler = Executors.newSingleThreadScheduledExecutor(new SolrNamedThreadFactory("cluster-event-scheduler"));
-    scheduler.schedule(() -> maybeFireScheduledEvent(), SCHEDULE_INTERVAL_SEC, TimeUnit.SECONDS);
-    running = true;
   }
 
   @Override
@@ -165,16 +141,7 @@ public class ClusterEventProducerImpl implements ClusterEventProducer, ClusterSi
     if (liveNodesListener != null) {
       zkController.zkStateReader.removeLiveNodesListener(liveNodesListener);
     }
-    if (scheduler != null) {
-      scheduler.shutdownNow();
-      try {
-        scheduler.awaitTermination(60, TimeUnit.SECONDS);
-      } catch (InterruptedException e) {
-        log.warn("Interrupted while waiting for the scheduler to shut down, ignoring");
-      }
-    }
     liveNodesListener = null;
-    scheduler = null;
   }
 
   private void ensureRunning() {
@@ -183,39 +150,6 @@ public class ClusterEventProducerImpl implements ClusterEventProducer, ClusterSi
     }
   }
 
-  private void maybeFireScheduledEvent() {
-    ensureRunning();
-    Set<ClusterEventListener> scheduledListeners = listeners.getOrDefault(ClusterEvent.EventType.SCHEDULED, Collections.emptySet());
-    if (scheduledListeners.isEmpty()) {
-      return;
-    }
-    scheduledListeners.forEach(listener -> {
-      ScheduledEventListener scheduledEventListener = (ScheduledEventListener) listener;
-      Schedule schedule = scheduledEventListener.getSchedule();
-      CompiledSchedule compiledSchedule = schedules.get(schedule.getName());
-      if (compiledSchedule == null) { // ???
-        return;
-      }
-      if (compiledSchedule.shouldRun()) {
-        Instant now = Instant.now();
-        ClusterEvent event = new ScheduledEvent() {
-          @Override
-          public Schedule getSchedule() {
-            return schedule;
-          }
-
-          @Override
-          public Instant getTimestamp() {
-            return now;
-          }
-        };
-        listener.onEvent(event);
-        compiledSchedule.setLastRunAt(now);
-      }
-    });
-  }
-
-
   @Override
   public void registerListener(ClusterEventListener listener) throws Exception {
     ensureRunning();
@@ -228,35 +162,13 @@ public class ClusterEventProducerImpl implements ClusterEventProducer, ClusterSi
     } catch (Throwable e) {
       throw new Exception(e);
     }
-    CompiledSchedule compiledSchedule = null;
-    if (listener.getEventTypes().contains(ClusterEvent.EventType.SCHEDULED)) {
-      if (!(listener instanceof ScheduledEventListener)) {
-        throw new Exception("listener " + listener + " wants to process " +
-            ClusterEvent.EventType.SCHEDULED + " events but is not a " +
-            ScheduledEventListener.class.getSimpleName());
-      } else {
-        compiledSchedule = new CompiledSchedule(((ScheduledEventListener) listener).getSchedule());
-      }
-    }
     ClusterEventProducer.super.registerListener(listener);
-    if (compiledSchedule != null) {
-      schedules.put(compiledSchedule.name, compiledSchedule);
-    }
-  }
-
-  @Override
-  public void unregisterListener(ClusterEventListener listener) {
-    if (listener.getEventTypes().contains(ClusterEvent.EventType.SCHEDULED)) {
-      schedules.remove(((ScheduledEventListener) listener).getSchedule().getName());
-    }
-    ClusterEventProducer.super.unregisterListener(listener);
   }
 
   @Override
   public void close() throws IOException {
     stop();
     isClosed = true;
-    schedules.clear();
     listeners.clear();
   }
 
diff --git a/solr/core/src/java/org/apache/solr/cluster/events/ScheduledEvent.java b/solr/core/src/java/org/apache/solr/cluster/scheduler/Schedulable.java
similarity index 82%
rename from solr/core/src/java/org/apache/solr/cluster/events/ScheduledEvent.java
rename to solr/core/src/java/org/apache/solr/cluster/scheduler/Schedulable.java
index 47b6dfd..f907245 100644
--- a/solr/core/src/java/org/apache/solr/cluster/events/ScheduledEvent.java
+++ b/solr/core/src/java/org/apache/solr/cluster/scheduler/Schedulable.java
@@ -14,17 +14,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.solr.cluster.events;
+package org.apache.solr.cluster.scheduler;
 
 /**
  *
  */
-public interface ScheduledEvent extends ClusterEvent {
-
-  @Override
-  default EventType getType() {
-    return EventType.SCHEDULED;
-  }
+public interface Schedulable {
 
   Schedule getSchedule();
+
+  void run();
 }
diff --git a/solr/core/src/java/org/apache/solr/cluster/events/Schedule.java b/solr/core/src/java/org/apache/solr/cluster/scheduler/Schedule.java
similarity index 96%
rename from solr/core/src/java/org/apache/solr/cluster/events/Schedule.java
rename to solr/core/src/java/org/apache/solr/cluster/scheduler/Schedule.java
index d9afc5e..31210bc 100644
--- a/solr/core/src/java/org/apache/solr/cluster/events/Schedule.java
+++ b/solr/core/src/java/org/apache/solr/cluster/scheduler/Schedule.java
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.solr.cluster.events;
+package org.apache.solr.cluster.scheduler;
 
 /**
  * Simple schedule for generating periodically occurring events.
diff --git a/solr/core/src/java/org/apache/solr/cluster/events/ScheduledEventListener.java b/solr/core/src/java/org/apache/solr/cluster/scheduler/SolrScheduler.java
similarity index 74%
rename from solr/core/src/java/org/apache/solr/cluster/events/ScheduledEventListener.java
rename to solr/core/src/java/org/apache/solr/cluster/scheduler/SolrScheduler.java
index 5f0457a..7ea76b4 100644
--- a/solr/core/src/java/org/apache/solr/cluster/events/ScheduledEventListener.java
+++ b/solr/core/src/java/org/apache/solr/cluster/scheduler/SolrScheduler.java
@@ -14,16 +14,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.solr.cluster.events;
+package org.apache.solr.cluster.scheduler;
 
 /**
  *
  */
-public interface ScheduledEventListener extends ClusterEventListener {
+public interface SolrScheduler {
 
-  /**
-   * Return the schedule that this listener needs. This is used when registering the
-   * listener to properly configure the scheduler.
-   */
-  Schedule getSchedule();
+  void registerSchedulable(Schedulable schedulable);
+
+  void unregisterSchedulable(Schedulable schedulable);
 }
diff --git a/solr/core/src/java/org/apache/solr/cluster/events/impl/CompiledSchedule.java b/solr/core/src/java/org/apache/solr/cluster/scheduler/impl/CompiledSchedule.java
similarity index 97%
rename from solr/core/src/java/org/apache/solr/cluster/events/impl/CompiledSchedule.java
rename to solr/core/src/java/org/apache/solr/cluster/scheduler/impl/CompiledSchedule.java
index 37e26f2..fefee45 100644
--- a/solr/core/src/java/org/apache/solr/cluster/events/impl/CompiledSchedule.java
+++ b/solr/core/src/java/org/apache/solr/cluster/scheduler/impl/CompiledSchedule.java
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.solr.cluster.events.impl;
+package org.apache.solr.cluster.scheduler.impl;
 
 import java.lang.invoke.MethodHandles;
 import java.text.ParseException;
@@ -26,7 +26,7 @@ import java.util.Date;
 import java.util.Locale;
 import java.util.TimeZone;
 
-import org.apache.solr.cluster.events.Schedule;
+import org.apache.solr.cluster.scheduler.Schedule;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.util.DateMathParser;
 import org.apache.solr.util.TimeZoneUtils;
diff --git a/solr/core/src/java/org/apache/solr/cluster/scheduler/impl/SolrSchedulerImpl.java b/solr/core/src/java/org/apache/solr/cluster/scheduler/impl/SolrSchedulerImpl.java
new file mode 100644
index 0000000..ee87da8
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/cluster/scheduler/impl/SolrSchedulerImpl.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.cluster.scheduler.impl;
+
+import java.lang.invoke.MethodHandles;
+import java.time.Instant;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.solr.cluster.events.ClusterSingleton;
+import org.apache.solr.cluster.scheduler.Schedule;
+import org.apache.solr.cluster.scheduler.Schedulable;
+import org.apache.solr.cluster.scheduler.SolrScheduler;
+import org.apache.solr.common.util.SolrNamedThreadFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * <p>Scheduled executions are triggered at most with {@link #SCHEDULE_INTERVAL_SEC} interval.
+ * Each registered {@link Schedulable} is processed sequentially and if its next execution time
+ * is in the past its {@link Schedulable#run()} method will be invoked.
+ * <p>NOTE: If the total time of execution of all registered Schedulable-s exceeds any schedule
+ * interval then exact execution times will be silently missed.</p>
+ */
+public class SolrSchedulerImpl implements SolrScheduler, ClusterSingleton {
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  public static final int SCHEDULE_INTERVAL_SEC = 10;
+
+  private final Set<Schedulable> schedulables = ConcurrentHashMap.newKeySet();
+  private final Map<String, CompiledSchedule> compiledSchedules = new ConcurrentHashMap<>();
+
+  private ScheduledExecutorService scheduler;
+  private boolean running = false;
+
+  @Override
+  public void registerSchedulable(Schedulable schedulable) {
+    try {
+      CompiledSchedule compiledSchedule = new CompiledSchedule(schedulable.getSchedule());
+      if (compiledSchedules.containsKey(compiledSchedule.name)) {
+        throw new Exception("Schedule defined by " + schedulable +
+            " already exists under name " + compiledSchedule.name);
+      }
+      compiledSchedules.put(compiledSchedule.name, compiledSchedule);
+      schedulables.add(schedulable);
+    } catch (Exception e) {
+      log.warn("Invalid schedule for {}, skipping: {}", schedulable, e);
+    }
+  }
+
+  @Override
+  public void unregisterSchedulable(Schedulable schedulable) {
+    compiledSchedules.remove(schedulable.getSchedule().getName());
+    schedulables.remove(schedulable);
+  }
+
+  @Override
+  public void start() {
+    // create scheduler
+    scheduler = Executors.newSingleThreadScheduledExecutor(new SolrNamedThreadFactory("cluster-event-scheduler"));
+    scheduler.schedule(() -> maybeFireScheduledEvent(), SCHEDULE_INTERVAL_SEC, TimeUnit.SECONDS);
+    running = true;
+  }
+
+  private void maybeFireScheduledEvent() {
+    if (!running) {
+      return;
+    }
+    if (compiledSchedules.isEmpty()) {
+      return;
+    }
+    schedulables.forEach(schedulable -> {
+      Schedule schedule = schedulable.getSchedule();
+      CompiledSchedule compiledSchedule = compiledSchedules.get(schedule.getName());
+      if (compiledSchedule == null) { // ???
+        return;
+      }
+      if (compiledSchedule.shouldRun()) {
+        Instant now = Instant.now();
+        schedulable.run();
+        compiledSchedule.setLastRunAt(now);
+      }
+    });
+  }
+
+  @Override
+  public boolean isRunning() {
+    return running;
+  }
+
+  @Override
+  public void stop() {
+    running = false;
+    if (scheduler != null) {
+      scheduler.shutdownNow();
+      try {
+        scheduler.awaitTermination(60, TimeUnit.SECONDS);
+      } catch (InterruptedException e) {
+        log.warn("Interrupted while waiting for the scheduler to shut down, ignoring");
+      }
+    }
+  }
+}