You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2026/10/05 21:48:52 UTC
[M] Change in asterixdb[master]: [NO ISSUE][TEST] Drop injected cloud failures from Unstable test logs
From Ian Maxon <im...@apache.org>:
Ian Maxon has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21998?usp=email )
Change subject: [NO ISSUE][TEST] Drop injected cloud failures from Unstable test logs
......................................................................
[NO ISSUE][TEST] Drop injected cloud failures from Unstable test logs
Every failure UnstableCloudClient injects is logged with its full stack
trace by each layer it passes through (retries, task and job failures,
LSM flushes, the tests' own retry loop), drowning anything of interest
in the Cloud*Unstable test logs. Filter out events caused by the
simulated failure for the duration of those tests; matching on the
cause rather than on logger names catches every layer while leaving
unrelated errors visible.
Generated-by: Claude Opus 5.5 <no...@anthropic.com>
Change-Id: Ie1b597c9de5140c0810bd39b2730d3e5b64f3523
---
M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageUnstableTest.java
M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/GCSCloudStorageUnstableTest.java
M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/RetryingQueryTranslator.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/SimulatedCloudFailureLogFilter.java
M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/UnstableCloudClient.java
5 files changed, 139 insertions(+), 2 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/98/21998/1
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageUnstableTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageUnstableTest.java
index 3b4657c..5eba0f2 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageUnstableTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageUnstableTest.java
@@ -77,6 +77,7 @@
@BeforeClass
public static void setUp() throws Exception {
System.setProperty(CloudRetryableRequestUtil.CLOUD_UNSTABLE_MODE, "true");
+ SimulatedCloudFailureLogFilter.install();
S3MockContainer s3Mock = LocalCloudUtilAdobeMock.startS3CloudEnvironment(true, true);
fillConfigTemplate(MOCK_SERVER_HOSTNAME_FRAGMENT + s3Mock.getHttpServerPort(), CONFIG_FILE_TEMPLATE,
CONFIG_FILE_NAME);
@@ -95,6 +96,7 @@
public static void tearDown() throws Exception {
System.clearProperty(CloudRetryableRequestUtil.CLOUD_UNSTABLE_MODE);
LangExecutionUtil.tearDown();
+ SimulatedCloudFailureLogFilter.uninstall();
LocalCloudUtilAdobeMock.shutdownSilently();
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/GCSCloudStorageUnstableTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/GCSCloudStorageUnstableTest.java
index 5ae96f2..bcd14da 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/GCSCloudStorageUnstableTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/GCSCloudStorageUnstableTest.java
@@ -88,6 +88,7 @@
@BeforeClass
public static void setUp() throws Exception {
System.setProperty(CloudRetryableRequestUtil.CLOUD_UNSTABLE_MODE, "true");
+ SimulatedCloudFailureLogFilter.install();
Storage storage = StorageOptions.newBuilder().setHost(MOCK_SERVER_HOSTNAME)
.setCredentials(NoCredentials.getInstance()).setProjectId(MOCK_SERVER_PROJECT_ID).build().getService();
cleanup(storage);
@@ -104,6 +105,7 @@
public static void tearDown() throws Exception {
System.clearProperty(CloudRetryableRequestUtil.CLOUD_UNSTABLE_MODE);
LangExecutionUtil.tearDown();
+ SimulatedCloudFailureLogFilter.uninstall();
}
@Parameterized.Parameters(name = "GCSCloudStorageUnstableTest {index}: {0}")
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/RetryingQueryTranslator.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/RetryingQueryTranslator.java
index feff749..6639bbf 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/RetryingQueryTranslator.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/RetryingQueryTranslator.java
@@ -20,6 +20,7 @@
package org.apache.asterix.test.cloud_storage;
import static org.apache.asterix.cloud.clients.UnstableCloudClient.ERROR_RATE;
+import static org.apache.asterix.cloud.clients.UnstableCloudClient.SIMULATED_ERROR_MESSAGE;
import java.util.List;
import java.util.concurrent.ExecutorService;
@@ -149,7 +150,7 @@
private boolean retryOnFailure(Exception e) {
if (e instanceof HyracksDataException) {
return ((HyracksDataException) e).getErrorCode() == ErrorCode.FAILED_IO_OPERATION.intValue()
- && ExceptionUtils.getRootCause(e).getMessage().contains("Simulated error");
+ && ExceptionUtils.getRootCause(e).getMessage().contains(SIMULATED_ERROR_MESSAGE);
}
return false;
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/SimulatedCloudFailureLogFilter.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/SimulatedCloudFailureLogFilter.java
new file mode 100644
index 0000000..8bccad8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/SimulatedCloudFailureLogFilter.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.cloud_storage;
+
+import static org.apache.asterix.cloud.clients.UnstableCloudClient.SIMULATED_ERROR_MESSAGE;
+
+import java.util.Collections;
+import java.util.IdentityHashMap;
+import java.util.Set;
+
+import org.apache.asterix.cloud.clients.UnstableCloudClient;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.Marker;
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.Logger;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.filter.AbstractFilter;
+import org.apache.logging.log4j.message.Message;
+
+/**
+ * Drops log events caused by the failures {@link UnstableCloudClient} injects. Every retried request and every
+ * aborted job logs the simulated failure with its stack trace, burying anything of interest in the unstable cloud
+ * tests' logs. Installed as a configuration-wide filter so events are dropped before any logger or appender sees
+ * them; events without a simulated failure are left to the regular configuration.
+ */
+public final class SimulatedCloudFailureLogFilter extends AbstractFilter {
+
+ private static final SimulatedCloudFailureLogFilter INSTANCE = new SimulatedCloudFailureLogFilter();
+
+ private SimulatedCloudFailureLogFilter() {
+ super(Result.DENY, Result.NEUTRAL);
+ }
+
+ public static void install() {
+ LoggerContext.getContext(false).getConfiguration().addFilter(INSTANCE);
+ }
+
+ public static void uninstall() {
+ LoggerContext.getContext(false).getConfiguration().removeFilter(INSTANCE);
+ }
+
+ @Override
+ public Result filter(LogEvent event) {
+ return result(isSimulated(event.getThrown()) || isSimulated(event.getMessage()));
+ }
+
+ @Override
+ public Result filter(Logger logger, Level level, Marker marker, Message msg, Throwable t) {
+ return result(isSimulated(t) || isSimulated(msg));
+ }
+
+ @Override
+ public Result filter(Logger logger, Level level, Marker marker, Object msg, Throwable t) {
+ return result(isSimulated(t) || (msg instanceof String && ((String) msg).contains(SIMULATED_ERROR_MESSAGE)));
+ }
+
+ @Override
+ public Result filter(Logger logger, Level level, Marker marker, String msg) {
+ return result(msg != null && msg.contains(SIMULATED_ERROR_MESSAGE));
+ }
+
+ @Override
+ public Result filter(Logger logger, Level level, Marker marker, String msg, Object... params) {
+ // parameterized calls pass the throwable as the trailing parameter rather than separately
+ return result((msg != null && msg.contains(SIMULATED_ERROR_MESSAGE)) || anySimulated(params));
+ }
+
+ private Result result(boolean simulated) {
+ return simulated ? onMatch : onMismatch;
+ }
+
+ private static boolean isSimulated(Message msg) {
+ if (msg == null) {
+ return false;
+ }
+ if (isSimulated(msg.getThrowable()) || anySimulated(msg.getParameters())) {
+ return true;
+ }
+ String formatted = msg.getFormattedMessage();
+ return formatted != null && formatted.contains(SIMULATED_ERROR_MESSAGE);
+ }
+
+ private static boolean anySimulated(Object[] params) {
+ if (params != null) {
+ for (Object param : params) {
+ if (param instanceof Throwable && isSimulated((Throwable) param)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ private static boolean isSimulated(Throwable t) {
+ return t != null && isSimulated(t, Collections.newSetFromMap(new IdentityHashMap<>()));
+ }
+
+ // the simulated failure may be wrapped arbitrarily deep, or rethrown only as a message once it crosses a job
+ // boundary, so the whole cause and suppressed graph is searched
+ private static boolean isSimulated(Throwable t, Set<Throwable> visited) {
+ if (t == null || !visited.add(t)) {
+ return false;
+ }
+ String message = t.getMessage();
+ if (message != null && message.contains(SIMULATED_ERROR_MESSAGE)) {
+ return true;
+ }
+ for (Throwable suppressed : t.getSuppressed()) {
+ if (isSimulated(suppressed, visited)) {
+ return true;
+ }
+ }
+ return isSimulated(t.getCause(), visited);
+ }
+}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/UnstableCloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/UnstableCloudClient.java
index 6e0e3b8..f917331 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/UnstableCloudClient.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/UnstableCloudClient.java
@@ -45,6 +45,7 @@
import com.fasterxml.jackson.databind.ObjectMapper;
public class UnstableCloudClient implements ICloudClient {
+ public static final String SIMULATED_ERROR_MESSAGE = "Simulated error";
// 10% error rate
public static final AtomicReference<Double> ERROR_RATE = new AtomicReference<>(0.11d);
private static final Random RANDOM = new Random(0);
@@ -164,7 +165,7 @@
private static void fail() throws HyracksDataException {
double prob = RANDOM.nextInt(100) / 100.0d;
if (prob < ERROR_RATE.get()) {
- throw HyracksDataException.create(ErrorCode.FAILED_IO_OPERATION, new IOException("Simulated error"));
+ throw HyracksDataException.create(ErrorCode.FAILED_IO_OPERATION, new IOException(SIMULATED_ERROR_MESSAGE));
}
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21998?usp=email
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email
Gerrit-MessageType: newchange
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Ie1b597c9de5140c0810bd39b2730d3e5b64f3523
Gerrit-Change-Number: 21998
Gerrit-PatchSet: 1
Gerrit-Owner: Ian Maxon <im...@apache.org>