You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hugegraph.apache.org by ji...@apache.org on 2026/10/15 16:42:14 UTC
(hugegraph) branch master updated: fix(store): serialize scan stream requests (#3354)
This is an automated email from the ASF dual-hosted git repository.
imbajin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hugegraph.git
The following commit(s) were added to refs/heads/master by this push:
new 9adfaf295 fix(store): serialize scan stream requests (#3354)
9adfaf295 is described below
commit 9adfaf295eb7f00ae5c2c848d8862b59b06756cc
Author: imbajin <ji...@apache.org>
AuthorDate: Fri Oct 16 00:42:09 2026 +0800
fix(store): serialize scan stream requests (#3354)
- serialize request sends and terminal observer calls
- suppress requests and repeated terminals after completion
- preserve final pages, original errors and channel retirement
- cover completion and cancellation during in-flight sends
---
.../hugegraph/store/client/grpc/KvPageScanner.java | 51 ++++---
.../store/client/grpc/KvPageScannerTest.java | 151 +++++++++++++++++++++
2 files changed, 183 insertions(+), 19 deletions(-)
diff --git a/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvPageScanner.java b/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvPageScanner.java
index 15bb53e8b..3ab1d8c84 100644
--- a/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvPageScanner.java
+++ b/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvPageScanner.java
@@ -69,6 +69,8 @@ class KvPageScanner implements KvCloseableIterator<Kv>, HgPageSize, HgSeekAble {
StreamObserver<ScanStreamReq>> streamFactory;
private final boolean orderedScan;
private final AtomicBoolean completed = new AtomicBoolean(false);
+ // Buffer publication and channel retirement must stay outside this lock.
+ private final Object requestLock = new Object();
private final SelectParam.Builder selectBuilder = SelectParam.newBuilder();
private final BlockingQueue<ScanStreamReq> reqQueue = new LinkedBlockingQueue<>();
private int pageSize = clientConfig.getNetKvScannerPageSize();
@@ -190,19 +192,38 @@ class KvPageScanner implements KvCloseableIterator<Kv>, HgPageSize, HgSeekAble {
}
private void send(ScanStreamReq req) {
- if (!this.completed.get()) {
- try {
- this.observer.onNext(req);
- } catch (IllegalStateException | IllegalArgumentException e) {
+ synchronized (this.requestLock) {
+ if (!this.completed.get()) {
+ try {
+ this.observer.onNext(req);
+ } catch (IllegalStateException | IllegalArgumentException e) {
+ } catch (Exception e) {
+ throw e;
+ }
+ }
+ }
+ }
+
+ private void completeRequest() {
+ synchronized (this.requestLock) {
+ if (!this.completed.compareAndSet(false, true)) {
+ return;
+ }
+ try {
+ this.observer.onCompleted();
} catch (Exception e) {
- throw e;
+ log.warn("failed to invoke requestObserver.onCompleted(), reason: {}", e.getMessage());
}
}
}
private void clientError(String msg) {
- this.observer.onError(GrpcUtil.toErr(msg));
+ synchronized (this.requestLock) {
+ if (this.completed.compareAndSet(false, true)) {
+ this.observer.onError(GrpcUtil.toErr(msg));
+ }
+ }
}
/*** Iterator ***/
@@ -304,12 +325,7 @@ class KvPageScanner implements KvCloseableIterator<Kv>, HgPageSize, HgSeekAble {
return;
}
if (value.getOver()) {
- completed.set(true);
- try {
- observer.onCompleted();
- } catch (Exception e) {
- log.warn("failed to invoke requestObserver.onCompleted(), reason: {}", e.getMessage());
- }
+ completeRequest();
}
proxy.send(value.getDataList());
if (completed.get()) {
@@ -319,12 +335,7 @@ class KvPageScanner implements KvCloseableIterator<Kv>, HgPageSize, HgSeekAble {
@Override
public void onError(Throwable t) {
- completed.set(true);
- try {
- observer.onCompleted();
- } catch (Exception e) {
- log.warn("failed to invoke requestObserver.onCompleted(), reason:", e.getMessage());
- }
+ completeRequest();
// Retire the failed pool before publishing the original iterator error,
// so immediate retries cannot reuse it. Cancellation or protocol errors
// must not evict healthy nodes, and stale notices preserve replacements.
@@ -339,7 +350,9 @@ class KvPageScanner implements KvCloseableIterator<Kv>, HgPageSize, HgSeekAble {
@Override
public void onCompleted() {
- completed.set(true);
+ synchronized (requestLock) {
+ completed.set(true);
+ }
proxy.close();
}
}
diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/grpc/KvPageScannerTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/grpc/KvPageScannerTest.java
index 5c5625550..a99aab957 100644
--- a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/grpc/KvPageScannerTest.java
+++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/grpc/KvPageScannerTest.java
@@ -34,6 +34,8 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Function;
@@ -51,6 +53,7 @@ import org.apache.hugegraph.store.grpc.stream.KvPageRes;
import org.apache.hugegraph.store.grpc.stream.ScanStreamReq;
import org.junit.Assert;
import org.junit.Test;
+import org.mockito.stubbing.Answer;
import com.google.protobuf.ByteString;
@@ -279,6 +282,154 @@ public class KvPageScannerTest {
}
}
+ @Test
+ public void testRequestTerminalsWaitForRequestSend() throws Exception {
+ var clientError = KvPageScanner.class.getDeclaredMethod("clientError", String.class);
+ clientError.setAccessible(true);
+ for (int terminal = 0; terminal < 3; terminal++) {
+ boolean failed = terminal != 0;
+ boolean cancelled = terminal == 2;
+ CountDownLatch sending = new CountDownLatch(1);
+ CountDownLatch releaseSend = new CountDownLatch(1);
+ AtomicBoolean overlap = new AtomicBoolean();
+ AtomicBoolean inOnNext = new AtomicBoolean();
+ AtomicInteger terminals = new AtomicInteger();
+ AtomicReference<StreamObserver<KvPageRes>> response = new AtomicReference<>();
+ AtomicReference<Thread> callbackThread = new AtomicReference<>();
+ StreamObserver<ScanStreamReq> requestObserver = mock(StreamObserver.class);
+ doAnswer(invocation -> {
+ inOnNext.set(true);
+ sending.countDown();
+ try {
+ Assert.assertTrue("Request send gate timed out",
+ releaseSend.await(5, TimeUnit.SECONDS));
+ return null;
+ } finally {
+ inOnNext.set(false);
+ }
+ }).when(requestObserver).onNext(any());
+ Answer<Void> terminalAnswer = invocation -> {
+ if (inOnNext.get()) {
+ overlap.set(true);
+ }
+ terminals.incrementAndGet();
+ return null;
+ };
+ doAnswer(terminalAnswer).when(requestObserver).onCompleted();
+ doAnswer(terminalAnswer).when(requestObserver).onError(any());
+ KvPageScanner scanner = new KvPageScanner(session(),
+ ScanStreamReq.newBuilder().setMethod(ScanMethod.ALL).setTable("table"),
+ observer -> {
+ response.set(observer);
+ return requestObserver;
+ });
+ var error = (cancelled ? Status.CANCELLED : Status.INVALID_ARGUMENT)
+ .withDescription("scan failed").asRuntimeException();
+ ExecutorService executor = Executors.newFixedThreadPool(2);
+ try {
+ Future<Boolean> receiver = executor.submit(scanner::hasNext);
+ Assert.assertTrue(sending.await(5, TimeUnit.SECONDS));
+ Future<?> callback = executor.submit(() -> {
+ callbackThread.set(Thread.currentThread());
+ if (cancelled) {
+ clientError.invoke(scanner, "scan timed out");
+ response.get().onError(error);
+ } else if (failed) {
+ response.get().onError(error);
+ } else {
+ response.get().onNext(page(true, 7));
+ }
+ return null;
+ });
+ awaitRequestTerminalBlocked(callbackThread, callback, terminals);
+ Assert.assertFalse("Request terminal overlapped onNext", overlap.get());
+ releaseSend.countDown();
+ callback.get(5, TimeUnit.SECONDS);
+ Assert.assertFalse("Request terminal overlapped onNext", overlap.get());
+ if (failed) {
+ try {
+ receiver.get(5, TimeUnit.SECONDS);
+ Assert.fail("The original scan error must reach the caller");
+ } catch (java.util.concurrent.ExecutionException e) {
+ Assert.assertSame(error, e.getCause().getCause());
+ }
+ } else {
+ Assert.assertTrue(receiver.get(5, TimeUnit.SECONDS));
+ Assert.assertEquals(7, scanner.next().getKey().byteAt(0) & 0xff);
+ Assert.assertFalse(scanner.hasNext());
+ }
+ response.get().onCompleted();
+ scanner.close();
+ Assert.assertEquals(1, terminals.get());
+ verify(requestObserver).onNext(any());
+ if (cancelled) {
+ verify(requestObserver).onError(any());
+ verify(requestObserver, never()).onCompleted();
+ } else {
+ verify(requestObserver).onCompleted();
+ verify(requestObserver, never()).onError(any());
+ }
+ } finally {
+ releaseSend.countDown();
+ executor.shutdownNow();
+ Assert.assertTrue("Scan workers failed to stop",
+ executor.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+ }
+
+ @Test
+ public void testTimeoutCancellationPreservesLateErrorWithoutFurtherRequests() throws Exception {
+ try (FailureFixture fixture = new FailureFixture()) {
+ AtomicReference<StreamObserver<KvPageRes>> response = new AtomicReference<>();
+ StreamObserver<ScanStreamReq> requestObserver = mock(StreamObserver.class);
+ KvPageScanner scanner = new KvPageScanner(fixture.session,
+ ScanStreamReq.newBuilder().setMethod(ScanMethod.ALL).setTable("table"),
+ observer -> {
+ response.set(observer);
+ return requestObserver;
+ });
+ // Exercise receive-timeout cancellation without waiting for the configured timeout.
+ var clientError = KvPageScanner.class.getDeclaredMethod("clientError", String.class);
+ clientError.setAccessible(true);
+ clientError.invoke(scanner, "scan timed out");
+ clientError.invoke(scanner, "late timeout");
+ scanner.close();
+ var error = Status.UNAVAILABLE.withDescription("late transport failure")
+ .asRuntimeException();
+ response.get().onError(error);
+ response.get().onCompleted();
+ verify(requestObserver).onError(any());
+ verify(requestObserver, never()).onNext(any());
+ verify(requestObserver, never()).onCompleted();
+ Assert.assertNull(fixture.manager.getStoreNode(fixture.nodeId));
+ Assert.assertFalse(AbstractGrpcClient.channels.containsKey(fixture.target));
+ try {
+ scanner.hasNext();
+ Assert.fail("The original transport error must remain visible");
+ } catch (HgStoreClientException e) {
+ Assert.assertSame(error, e.getCause());
+ }
+ }
+ }
+
+ private static void awaitRequestTerminalBlocked(AtomicReference<Thread> callbackThread,
+ Future<?> callback,
+ AtomicInteger terminals) throws Exception {
+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
+ while (System.nanoTime() < deadline) {
+ Thread thread = callbackThread.get();
+ if (terminals.get() > 0 || callback.isDone() ||
+ thread != null && thread.getState() == Thread.State.BLOCKED) {
+ break;
+ }
+ Thread.sleep(1);
+ }
+ Assert.assertEquals("Request terminal must wait until onNext returns", 0, terminals.get());
+ Assert.assertNotNull(callbackThread.get());
+ Assert.assertEquals(Thread.State.BLOCKED, callbackThread.get().getState());
+ }
+
private static void awaitBlockedReceiver(Thread receiver, Future<?> result) throws Exception {
long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
while (receiver.getState() != Thread.State.TIMED_WAITING && System.nanoTime() < deadline) {