You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2026/09/30 12:10:24 UTC
[S] Change in asterixdb[master]: [NO ISSUE][HYR] Code a failure to reach a node
From Michael Blow <mb...@apache.org>:
Michael Blow has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21950?usp=email )
Change subject: [NO ISSUE][HYR] Code a failure to reach a node
......................................................................
[NO ISSUE][HYR] Code a failure to reach a node
- user model changes: no
- storage format changes: no
- interface changes: no
Details: A node that has gone away stays registered at the CC until its
heartbeats lapse, so jobs are still scheduled on it in the meantime.
Starting tasks on it, or cleaning a job up on it, then fails with a bare
IPCException that carries no error code, and the client is answered with
an internal error that names nothing it can act on.
Report both as the new NODE_UNREACHABLE, naming the node.
Ext-ref: MB-74318
Generated-by: Claude Opus 5.5 <no...@anthropic.com>
Change-Id: I202d4aa1f3fad8998c2d485a1062347a920f38f1
---
M hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
M hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/executor/JobExecutor.java
M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java
M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/test/java/org/apache/hyracks/control/cc/job/JobManagerTest.java
5 files changed, 42 insertions(+), 4 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/50/21950/1
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
index 8690e1a..df297f5 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
@@ -166,6 +166,7 @@
VECTOR_INDEX_BUILD_FAILED(136),
MALFORMED_VECTOR_INDEX(137),
UNEXPECTED_VECTOR_VALUE(138),
+ NODE_UNREACHABLE(139),
// Compilation error codes.
RULECOLLECTION_NOT_INSTANCE_OF_LIST(10000),
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
index 1bcd824..b2d3e44 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
@@ -156,6 +156,7 @@
136 = Failed to build the vector index. %1$s
137 = Malformed vector index. %1$s
138 = Encountered unexpected vector. %1$s
+139 = Node %1$s is unreachable
10000 = The given rule collection %1$s is not an instance of the List class.
10001 = Cannot compose partition constraint %1$s with %2$s
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/executor/JobExecutor.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/executor/JobExecutor.java
index d7dd590..feedb02 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/executor/JobExecutor.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/executor/JobExecutor.java
@@ -68,6 +68,7 @@
import org.apache.hyracks.control.common.job.TaskAttemptDescriptor;
import org.apache.hyracks.control.common.work.IResultCallback;
import org.apache.hyracks.control.common.work.NoOpCallback;
+import org.apache.hyracks.ipc.exceptions.IPCException;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -519,10 +520,15 @@
}
JobKind jobKind = (JobKind) jobRun.getJobSpecification().getProperty(HyracksJobProperty.JOB_KIND);
byte[] jagBytes = changed ? acgBytes : null;
- node.getNodeController().startTasks(deploymentId, jobId, jagBytes, taskDescriptors,
- connectorPolicies, jobRun.getFlags(),
- ccs.createOrGetJobParameterByteStore(jobId).getParameterMap(), deployedJobSpecId,
- jobRun.getStartTime(), jobRun.getStartTimeZoneId(), jobKind);
+ try {
+ node.getNodeController().startTasks(deploymentId, jobId, jagBytes, taskDescriptors,
+ connectorPolicies, jobRun.getFlags(),
+ ccs.createOrGetJobParameterByteStore(jobId).getParameterMap(), deployedJobSpecId,
+ jobRun.getStartTime(), jobRun.getStartTimeZoneId(), jobKind);
+ } catch (IPCException e) {
+ // the node is still registered until its heartbeats lapse, so jobs keep landing on it
+ throw HyracksException.create(ErrorCode.NODE_UNREACHABLE, e, nodeId);
+ }
}
}
} catch (Exception e) {
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java
index c9667d7..de2e111 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/job/JobManager.java
@@ -57,6 +57,7 @@
import org.apache.hyracks.control.common.controllers.CCConfig;
import org.apache.hyracks.control.common.work.IResultCallback;
import org.apache.hyracks.control.common.work.NoOpCallback;
+import org.apache.hyracks.ipc.exceptions.IPCException;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -239,6 +240,10 @@
} else {
try {
ncs.getNodeController().cleanUpJoblet(jobId, status);
+ } catch (IPCException e) {
+ LOGGER.error("Exception cleaning up joblet {} on node {}", jobId, n, e);
+ caughtException = ExceptionUtils.suppress(caughtException,
+ HyracksException.create(ErrorCode.NODE_UNREACHABLE, e, n));
} catch (Exception e) {
LOGGER.error("Exception cleaning up joblet {} on node {}", jobId, n, e);
caughtException = ExceptionUtils.suppress(caughtException, e);
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/test/java/org/apache/hyracks/control/cc/job/JobManagerTest.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/test/java/org/apache/hyracks/control/cc/job/JobManagerTest.java
index 7076e4d..db708ad 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/test/java/org/apache/hyracks/control/cc/job/JobManagerTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/test/java/org/apache/hyracks/control/cc/job/JobManagerTest.java
@@ -53,6 +53,7 @@
import org.apache.hyracks.control.common.ipc.NodeControllerRemoteProxy;
import org.apache.hyracks.control.common.logs.LogFile;
import org.apache.hyracks.control.common.work.NoOpCallback;
+import org.apache.hyracks.ipc.exceptions.IPCException;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -354,6 +355,30 @@
verify(ccs.getContext(), never()).notifyJobSubmissionFailed(any(), any());
}
+ /**
+ * A node that has gone away stays registered until its heartbeats lapse, so cleaning up a job on it fails
+ * with a bare IPC failure. Uncoded, that reaches the client as an internal error rather than as the node
+ * being unreachable.
+ */
+ @Test
+ public void unreachableNodeFailsCleanupAsNodeUnreachable() throws Exception {
+ IJobCapacityController jobCapacityController = mock(IJobCapacityController.class);
+ ClusterControllerService ccs = mockClusterControllerService();
+ IJobManager jobManager = new JobManager(ccConfig, ccs, jobCapacityController);
+ NodeControllerRemoteProxy nodeController =
+ ccs.getNodeManager().getNodeControllerState("node1").getNodeController();
+ doThrow(new IPCException(new IOException("Connection failed"))).when(nodeController).cleanUpJoblet(any(),
+ any());
+
+ JobRun run = mockJobRun(1);
+ try {
+ jobManager.prepareComplete(run, JobStatus.FAILURE, Collections.emptyList());
+ Assert.fail("cleanup on an unreachable node should fail");
+ } catch (HyracksException e) {
+ Assert.assertTrue(String.valueOf(e), e.matches(ErrorCode.NODE_UNREACHABLE));
+ }
+ }
+
private JobRun mockJobRun(long id) {
JobRun run = mock(JobRun.class, Mockito.RETURNS_DEEP_STUBS);
when(run.getExceptions()).thenReturn(Collections.emptyList());
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21950?usp=email
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email
Gerrit-MessageType: newchange
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I202d4aa1f3fad8998c2d485a1062347a920f38f1
Gerrit-Change-Number: 21950
Gerrit-PatchSet: 1
Gerrit-Owner: Michael Blow <mb...@apache.org>