You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by rp...@apache.org on 2025/04/18 10:07:44 UTC
(ignite-3) 01/01: Implement graceful WatchProcessor stop
This is an automated email from the ASF dual-hosted git repository.
rpuch pushed a commit to branch fighting-critical-failures
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
commit 12432eef1b99f72fb5fbb8ec5f883675a1377c1d
Author: Roman Puchkovskiy <ro...@gmail.com>
AuthorDate: Fri Apr 18 14:04:48 2025 +0400
Implement graceful WatchProcessor stop
---
.../metastorage/server/WatchProcessor.java | 62 ++++++++++++++++++----
1 file changed, 53 insertions(+), 9 deletions(-)
diff --git a/modules/metastorage/src/main/java/org/apache/ignite/internal/metastorage/server/WatchProcessor.java b/modules/metastorage/src/main/java/org/apache/ignite/internal/metastorage/server/WatchProcessor.java
index 034bc9e0077..030ca822cf9 100644
--- a/modules/metastorage/src/main/java/org/apache/ignite/internal/metastorage/server/WatchProcessor.java
+++ b/modules/metastorage/src/main/java/org/apache/ignite/internal/metastorage/server/WatchProcessor.java
@@ -20,6 +20,8 @@ package org.apache.ignite.internal.metastorage.server;
import static java.util.Collections.emptyList;
import static java.util.concurrent.CompletableFuture.allOf;
import static java.util.concurrent.CompletableFuture.failedFuture;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.stream.Collectors.toList;
import static org.apache.ignite.internal.failure.FailureType.CRITICAL_ERROR;
import static org.apache.ignite.internal.metastorage.server.raft.MetaStorageWriteHandler.IDEMPOTENT_COMMAND_PREFIX_BYTES;
@@ -35,9 +37,13 @@ import java.util.OptionalLong;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.ThreadPoolExecutor.DiscardPolicy;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Supplier;
import java.util.stream.Collectors;
import org.apache.ignite.internal.close.ManuallyCloseable;
import org.apache.ignite.internal.failure.FailureContext;
@@ -53,6 +59,7 @@ import org.apache.ignite.internal.metastorage.RevisionUpdateListener;
import org.apache.ignite.internal.metastorage.WatchEvent;
import org.apache.ignite.internal.metastorage.WatchListener;
import org.apache.ignite.internal.thread.IgniteThreadFactory;
+import org.apache.ignite.internal.util.IgniteSpinBusyLock;
import org.apache.ignite.internal.util.IgniteUtils;
/**
@@ -114,6 +121,9 @@ public class WatchProcessor implements ManuallyCloseable {
*/
private final AtomicBoolean firedFailureOnChain = new AtomicBoolean(false);
+ private final IgniteSpinBusyLock busyLock = new IgniteSpinBusyLock();
+ private final AtomicBoolean stopped = new AtomicBoolean(false);
+
/**
* Creates a new instance.
*
@@ -122,9 +132,17 @@ public class WatchProcessor implements ManuallyCloseable {
public WatchProcessor(String nodeName, EntryReader entryReader, FailureProcessor failureProcessor) {
this.entryReader = entryReader;
- this.watchExecutor = Executors.newFixedThreadPool(
+ ThreadFactory threadFactory = IgniteThreadFactory.create(nodeName, "metastorage-watch-executor", LOG, NOTHING_ALLOWED);
+ this.watchExecutor = new ThreadPoolExecutor(
+ 4,
4,
- IgniteThreadFactory.create(nodeName, "metastorage-watch-executor", LOG, NOTHING_ALLOWED)
+ 0L,
+ MILLISECONDS,
+ new LinkedBlockingQueue<>(),
+ threadFactory,
+ // This executor gets shut down during node stop, so we don't care about its tasks being discarded; we would have to
+ // filter those RejectedExecutionExeptions by hand anyway.
+ new DiscardPolicy()
);
this.failureProcessor = failureProcessor;
@@ -156,6 +174,18 @@ public class WatchProcessor implements ManuallyCloseable {
this.watchEventHandlingCallback = callback;
}
+ private <T> CompletableFuture<T> inBusyLockAsync(Supplier<CompletableFuture<T>> fn) {
+ return IgniteUtils.inBusyLockAsync(busyLock, fn);
+ }
+
+ private void inBusyLock(Runnable fn) {
+ IgniteUtils.inBusyLock(busyLock, fn);
+ }
+
+ private void inBusyLockSafe(Runnable fn) {
+ IgniteUtils.inBusyLockSafe(busyLock, fn);
+ }
+
/**
* Queues the following set of actions that will be executed after the previous invocation of this method completes:
*
@@ -173,10 +203,14 @@ public class WatchProcessor implements ManuallyCloseable {
* @return Future that gets completed when all registered watches have been notified of the given event.
*/
public CompletableFuture<Void> notifyWatches(long newRevision, List<Entry> updatedEntries, HybridTimestamp time) {
+ return inBusyLockAsync(() -> notifyWatchesInternal(newRevision, updatedEntries, time));
+ }
+
+ private CompletableFuture<Void> notifyWatchesInternal(long newRevision, List<Entry> updatedEntries, HybridTimestamp time) {
assert time != null;
CompletableFuture<Void> newFuture = notificationFuture
- .thenComposeAsync(v -> {
+ .thenComposeAsync(v -> inBusyLockAsync(() -> {
List<Entry> filteredUpdatedEntries = updatedEntries.isEmpty() ? emptyList() : updatedEntries.stream()
.filter(WatchProcessor::isNotIdempotentCacheCommand)
.collect(toList());
@@ -191,12 +225,12 @@ public class WatchProcessor implements ManuallyCloseable {
CompletableFuture<Void> notifyUpdateRevisionFuture = notifyUpdateRevisionListeners(newRevision);
CompletableFuture<Void> notificationFuture = allOf(notifyWatchesFuture, notifyUpdateRevisionFuture)
- .thenRunAsync(() -> invokeOnRevisionCallback(newRevision, time), watchExecutor);
+ .thenRunAsync(() -> inBusyLock(() -> invokeOnRevisionCallback(newRevision, time)), watchExecutor);
notificationFuture.whenComplete((unused, e) -> maybeLogLongProcessing(filteredUpdatedEntries, startTimeNanos));
return notificationFuture;
- }, watchExecutor)
+ }), watchExecutor)
.whenComplete((unused, e) -> {
if (e != null) {
notifyFailureHandlerOnFirstFailureInNotificationChain(e);
@@ -307,15 +341,19 @@ public class WatchProcessor implements ManuallyCloseable {
* @param time Timestamp value for advancing.
*/
public void advanceSafeTime(Runnable callback, HybridTimestamp time) {
+ inBusyLockSafe(() -> advanceSafeTimeInternal(callback, time));
+ }
+
+ private void advanceSafeTimeInternal(Runnable callback, HybridTimestamp time) {
assert time != null;
//noinspection NonAtomicOperationOnVolatileField
notificationFuture = notificationFuture
- .thenRunAsync(() -> {
+ .thenRunAsync(() -> inBusyLockSafe(() -> {
callback.run();
watchEventHandlingCallback.onSafeTimeAdvanced(time);
- }, watchExecutor)
+ }), watchExecutor)
.whenComplete((ignored, e) -> {
if (e != null) {
notifyFailureHandlerOnFirstFailureInNotificationChain(e);
@@ -341,9 +379,15 @@ public class WatchProcessor implements ManuallyCloseable {
@Override
public void close() {
+ if (!stopped.compareAndSet(false, true)) {
+ return;
+ }
+
+ busyLock.block();
+
notificationFuture.completeExceptionally(new NodeStoppingException());
- IgniteUtils.shutdownAndAwaitTermination(watchExecutor, 10, TimeUnit.SECONDS);
+ IgniteUtils.shutdownAndAwaitTermination(watchExecutor, 10, SECONDS);
}
/** Registers a Meta Storage revision update listener. */