You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2025/10/16 00:42:55 UTC

(pinot) branch master updated: Ensure proper exception is returned when query is cancelled/killed (#17009)

This is an automated email from the ASF dual-hosted git repository.

jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new 01912c4aa88  Ensure proper exception is returned when query is cancelled/killed (#17009)
01912c4aa88 is described below

commit 01912c4aa88bfe045c4f101dbf6a15e654fa6657
Author: Xiaotian (Jackie) Jiang <17...@users.noreply.github.com>
AuthorDate: Wed Oct 15 17:42:48 2025 -0700

     Ensure proper exception is returned when query is cancelled/killed (#17009)
---
 .../StreamingInstanceResponseOperator.java         |  14 +-
 .../core/query/reduce/BrokerReduceService.java     |  20 +-
 .../pinot/core/query/scheduler/QueryScheduler.java |  56 ++--
 .../core/transport/InstanceRequestHandler.java     |   2 +-
 .../tests/BaseQueryKillingIntegrationTest.java     | 209 +++++++++++++
 .../CpuBasedBrokerQueryKillingIntegrationTest.java | 123 ++++++++
 .../CpuBasedServerQueryKillingIntegrationTest.java | 120 ++++++++
 ...moryBasedServerQueryKillingIntegrationTest.java | 119 ++++++++
 .../MultiNodesOfflineClusterIntegrationTest.java   |  27 ++
 .../tests/OOMProtectionEnabledIntegrationTest.java | 107 -------
 ...flineClusterCpuBasedServerQueryKillingTest.java | 239 ---------------
 ...flineClusterMemBasedBrokerQueryKillingTest.java | 241 ---------------
 ...flineClusterMemBasedServerQueryKillingTest.java | 331 ---------------------
 ...rMemBasedServerResourceUsageAccountantTest.java | 302 -------------------
 .../OfflineClusterResourceUsageAccountantTest.java | 272 -----------------
 .../tests/OfflineGRPCServerIntegrationTest.java    |  30 ++
 ...OfflineGRPCServerMultiStageIntegrationTest.java |   7 +-
 ...lineGRPCServerOOMAccountingIntegrationTest.java |  56 ----
 .../OfflineSecureGRPCServerIntegrationTest.java    |   2 +
 .../query/mailbox/InMemorySendingMailbox.java      |  18 +-
 .../pinot/query/mailbox/ReceivingMailbox.java      | 129 ++++----
 .../mailbox/channel/MailboxContentObserver.java    |  17 +-
 .../apache/pinot/query/runtime/QueryRunner.java    |   2 +-
 .../runtime/executor/OpChainSchedulerService.java  |  18 +-
 .../runtime/operator/MailboxSendOperator.java      |  36 ++-
 .../utils/BlockingMultiStreamConsumer.java         |   6 +-
 .../pinot/query/mailbox/MailboxServiceTest.java    |  54 ++--
 .../pinot/query/mailbox/ReceivingMailboxTest.java  |  54 ++--
 28 files changed, 846 insertions(+), 1765 deletions(-)

diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/streaming/StreamingInstanceResponseOperator.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/streaming/StreamingInstanceResponseOperator.java
index 03b0d93f9bf..857c853bcd0 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/streaming/StreamingInstanceResponseOperator.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/streaming/StreamingInstanceResponseOperator.java
@@ -32,7 +32,6 @@ import org.apache.pinot.segment.spi.SegmentContext;
 import org.apache.pinot.spi.exception.QueryErrorCode;
 import org.apache.pinot.spi.exception.QueryErrorMessage;
 import org.apache.pinot.spi.exception.QueryException;
-import org.apache.pinot.spi.exception.TerminationException;
 import org.apache.pinot.spi.query.QueryThreadContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -91,11 +90,14 @@ public class StreamingInstanceResponseOperator extends InstanceResponseOperator
         return buildInstanceResponseBlock(resultsBlock).toMetadataOnlyResponseBlock();
       }
     } catch (Throwable t) {
-      TerminationException terminateException = QueryThreadContext.getTerminateException();
-      if (terminateException != null) {
-        return new InstanceResponseBlock(new ExceptionResultsBlock(terminateException));
-      } else if (t instanceof QueryException) {
-        return new InstanceResponseBlock(new ExceptionResultsBlock((QueryException) t));
+      // First check terminate exception and use it as the results block if exists. We want to return the termination
+      // reason when query is explicitly terminated.
+      QueryException queryException = QueryThreadContext.getTerminateException();
+      if (queryException == null && t instanceof QueryException) {
+        queryException = (QueryException) t;
+      }
+      if (queryException != null) {
+        return new InstanceResponseBlock(new ExceptionResultsBlock(queryException));
       } else {
         LOGGER.error("Caught exception while streaming results (query: {})", _queryContext, t);
         return new InstanceResponseBlock(new ExceptionResultsBlock(QueryErrorMessage.safeMsg(QueryErrorCode.INTERNAL,
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/BrokerReduceService.java b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/BrokerReduceService.java
index dbdb805ae6c..d938deee1ec 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/BrokerReduceService.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/BrokerReduceService.java
@@ -40,7 +40,6 @@ import org.apache.pinot.spi.env.PinotConfiguration;
 import org.apache.pinot.spi.exception.BadQueryRequestException;
 import org.apache.pinot.spi.exception.QueryErrorCode;
 import org.apache.pinot.spi.exception.QueryException;
-import org.apache.pinot.spi.exception.TerminationException;
 import org.apache.pinot.spi.query.QueryThreadContext;
 import org.apache.pinot.spi.utils.CommonConstants;
 import org.apache.pinot.spi.utils.builder.TableNameBuilder;
@@ -173,17 +172,16 @@ public class BrokerReduceService extends BaseReduceService {
     } catch (RuntimeException e) {
       // First check terminate exception and use it as the results block if exists. We want to return the termination
       // reason when query is explicitly terminated.
-      TerminationException terminateException = QueryThreadContext.getTerminateException();
-      if (terminateException != null) {
-        brokerResponseNative.addException(QueryProcessingException.fromQueryException(terminateException));
+      QueryException queryException = QueryThreadContext.getTerminateException();
+      if (queryException == null && e instanceof QueryException) {
+        queryException = (QueryException) e;
+      }
+      if (queryException != null) {
+        brokerResponseNative.addException(QueryProcessingException.fromQueryException(queryException));
       } else {
-        if (e instanceof QueryException) {
-          brokerResponseNative.addException(QueryProcessingException.fromQueryException((QueryException) e));
-        } else {
-          LOGGER.error("Caught exception while reducing data tables (query: {})", serverQueryContext, e);
-          brokerResponseNative.addException(new QueryProcessingException(QueryErrorCode.MERGE_RESPONSE,
-              "Caught exception while reducing data tables: " + e.getMessage()));
-        }
+        LOGGER.error("Caught exception while reducing data tables (query: {})", serverQueryContext, e);
+        brokerResponseNative.addException(new QueryProcessingException(QueryErrorCode.MERGE_RESPONSE,
+            "Caught exception while reducing data tables: " + e.getMessage()));
       }
     }
     QueryContext queryContext;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/scheduler/QueryScheduler.java b/pinot-core/src/main/java/org/apache/pinot/core/query/scheduler/QueryScheduler.java
index 9e75cf2e372..f123de77cf8 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/scheduler/QueryScheduler.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/scheduler/QueryScheduler.java
@@ -25,7 +25,6 @@ import java.util.Map;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.atomic.LongAccumulator;
 import javax.annotation.Nullable;
-import org.apache.commons.lang3.StringUtils;
 import org.apache.pinot.common.datatable.DataTable.MetadataKey;
 import org.apache.pinot.common.metrics.ServerMeter;
 import org.apache.pinot.common.metrics.ServerMetrics;
@@ -40,9 +39,7 @@ import org.apache.pinot.core.query.request.context.TimerContext;
 import org.apache.pinot.core.query.scheduler.resources.ResourceManager;
 import org.apache.pinot.spi.accounting.ThreadAccountant;
 import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.exception.EarlyTerminationException;
 import org.apache.pinot.spi.exception.QueryErrorCode;
-import org.apache.pinot.spi.exception.QueryErrorMessage;
 import org.apache.pinot.spi.exception.QueryException;
 import org.apache.pinot.spi.query.QueryExecutionContext;
 import org.apache.pinot.spi.query.QueryThreadContext;
@@ -197,34 +194,49 @@ public abstract class QueryScheduler {
     TimerContext timerContext = queryRequest.getTimerContext();
     TimerContext.Timer responseSerializationTimer =
         timerContext.startNewPhaseTimer(ServerQueryPhase.RESPONSE_SERIALIZATION);
+    long requestId = queryRequest.getRequestId();
+    String brokerId = queryRequest.getBrokerId();
 
-    byte[] responseByte = null;
+    byte[] responseBytes = null;
     try {
-      responseByte = instanceResponse.toDataTable().toBytes();
-    } catch (EarlyTerminationException e) {
-      QueryException terminateException = QueryThreadContext.getTerminateException();
-      String userMsg = "Cancelled while building data table" + (terminateException != null ? ": " + terminateException
-          : StringUtils.EMPTY);
-      LOGGER.error(userMsg);
-      QueryErrorMessage errMsg = QueryErrorMessage.safeMsg(QueryErrorCode.QUERY_CANCELLATION, userMsg);
-      Map<String, String> queryOptions = queryRequest.getQueryContext().getQueryOptions();
-      String workloadName = QueryOptionsUtils.getWorkloadName(queryOptions);
-      instanceResponse = new InstanceResponseBlock(new ExceptionResultsBlock(errMsg));
-      instanceResponse.addMetadata(MetadataKey.REQUEST_ID.getName(), Long.toString(queryRequest.getRequestId()));
-      instanceResponse.addMetadata(MetadataKey.QUERY_ID.getName(), queryRequest.getCid());
-      instanceResponse.addMetadata(MetadataKey.WORKLOAD_NAME.getName(), workloadName);
-      return serializeResponse(queryRequest, instanceResponse);
+      responseBytes = instanceResponse.toDataTable().toBytes();
     } catch (Exception e) {
-      _serverMetrics.addMeteredGlobalValue(ServerMeter.RESPONSE_SERIALIZATION_EXCEPTIONS, 1);
-      LOGGER.error("Caught exception while serializing response for requestId: {}, brokerId: {}",
-          queryRequest.getRequestId(), queryRequest.getBrokerId(), e);
+      // First check terminate exception and use it as the response if exists. We want to return the termination reason
+      // when query is explicitly terminated.
+      QueryException queryException = QueryThreadContext.getTerminateException();
+      // Do not log exception when query is explicitly terminated
+      if (queryException == null) {
+        if (e instanceof QueryException) {
+          queryException = (QueryException) e;
+          // TODO: Revisit if we should log exception here
+          LOGGER.warn("Caught QueryException while serializing response from response for requestId: {}, brokerId: {}",
+              requestId, brokerId, queryException);
+        } else {
+          _serverMetrics.addMeteredGlobalValue(ServerMeter.RESPONSE_SERIALIZATION_EXCEPTIONS, 1);
+          LOGGER.error("Caught exception while serializing response from response for requestId: {}, brokerId: {}",
+              requestId, brokerId, e);
+          queryException = QueryErrorCode.INTERNAL.asException("Error serializing response", e);
+        }
+      }
+      try {
+        InstanceResponseBlock errorResponse = new InstanceResponseBlock(new ExceptionResultsBlock(queryException));
+        errorResponse.addMetadata(MetadataKey.REQUEST_ID.getName(), Long.toString(requestId));
+        errorResponse.addMetadata(MetadataKey.QUERY_ID.getName(), queryRequest.getCid());
+        String workloadName = QueryOptionsUtils.getWorkloadName(queryRequest.getQueryContext().getQueryOptions());
+        errorResponse.addMetadata(MetadataKey.WORKLOAD_NAME.getName(), workloadName);
+        responseBytes = errorResponse.toDataTable().toBytes();
+      } catch (Exception e1) {
+        _serverMetrics.addMeteredGlobalValue(ServerMeter.RESPONSE_SERIALIZATION_EXCEPTIONS, 1);
+        LOGGER.error("Caught exception while constructing error response for requestId: {}, brokerId: {}",
+            requestId, brokerId, e1);
+      }
     }
 
     responseSerializationTimer.stopAndRecord();
     timerContext.startNewPhaseTimer(ServerQueryPhase.TOTAL_QUERY_TIME, timerContext.getQueryArrivalTimeMs())
         .stopAndRecord();
 
-    return responseByte;
+    return responseBytes;
   }
 
   /**
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/transport/InstanceRequestHandler.java b/pinot-core/src/main/java/org/apache/pinot/core/transport/InstanceRequestHandler.java
index 1753fc914d3..f0e3d728809 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/transport/InstanceRequestHandler.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/transport/InstanceRequestHandler.java
@@ -246,7 +246,7 @@ public class InstanceRequestHandler extends SimpleChannelInboundHandler<ByteBuf>
     if (executionContext == null) {
       return false;
     }
-    boolean cancelled = executionContext.terminate(QueryErrorCode.QUERY_CANCELLATION, "Cancelled by user");
+    boolean cancelled = executionContext.terminate(QueryErrorCode.QUERY_CANCELLATION, "Cancelled on: " + _instanceName);
     if (LOGGER.isDebugEnabled()) {
       if (cancelled) {
         LOGGER.debug("Cancelled query: {}", queryId);
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseQueryKillingIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseQueryKillingIntegrationTest.java
new file mode 100644
index 00000000000..871d909f82c
--- /dev/null
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseQueryKillingIntegrationTest.java
@@ -0,0 +1,209 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import org.apache.avro.Schema.Field;
+import org.apache.avro.Schema.Type;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.commons.io.FileUtils;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.env.PinotConfiguration;
+import org.apache.pinot.spi.utils.CommonConstants.Broker;
+import org.apache.pinot.spi.utils.CommonConstants.Server;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+import org.apache.pinot.util.TestUtils;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+
+/// Base class for query killing integration tests.
+public abstract class BaseQueryKillingIntegrationTest extends BaseClusterIntegrationTest {
+  protected static final String STRING_DIM_SV1 = "stringDimSV1";
+  protected static final String STRING_DIM_SV2 = "stringDimSV2";
+  protected static final String INT_DIM_SV1 = "intDimSV1";
+  protected static final String LONG_DIM_SV1 = "longDimSV1";
+  protected static final String DOUBLE_DIM_SV1 = "doubleDimSV1";
+  protected static final String BOOLEAN_DIM_SV1 = "booleanDimSV1";
+  protected static final int NUM_SEGMENTS = 3;
+  protected static final int NUM_DOCS_PER_SEGMENT = 3_000_000;
+
+  protected static final String LARGE_SELECT_STAR_QUERY = "SELECT * FROM mytable LIMIT 9000000";
+
+  protected static final String LARGE_DISTINCT_QUERY =
+      "SELECT DISTINCT stringDimSV2 FROM mytable ORDER BY stringDimSV2 LIMIT 3000000";
+
+  protected static final String LARGE_GROUP_BY_QUERY =
+      "SELECT DISTINCT_COUNT_HLL(intDimSV1, 14), stringDimSV2 FROM mytable GROUP BY 2 ORDER BY 1 LIMIT 3000000";
+
+  /// SafeTrim sort-aggregation case, pair-wise combine
+  protected static final String LARGE_GROUP_BY_PAIRWISE_COMBINE_QUERY =
+      "SET sortAggregateSingleThreadedNumSegmentsThreshold=1; SET sortAggregateLimitThreshold=3000001; "
+          + "SELECT DISTINCT_COUNT_HLL(intDimSV1, 14), stringDimSV2 FROM mytable GROUP BY 2 ORDER BY 2 LIMIT 3000000";
+
+  /// SafeTrim sort-aggregation case, sequential combine
+  protected static final String LARGE_GROUP_BY_SEQUENTIAL_COMBINE_QUERY =
+      "SET sortAggregateSingleThreadedNumSegmentsThreshold=10000; SET sortAggregateLimitThreshold=3000001; "
+          + "SELECT DISTINCT_COUNT_HLL(intDimSV1, 14), stringDimSV2 FROM mytable GROUP BY 2 ORDER BY 2 LIMIT 3000000";
+
+  protected static final String AGGREGATE_QUERY = "SELECT DISTINCT_COUNT_HLL(intDimSV1, 14) FROM mytable";
+  protected static final String SELECT_STAR_QUERY = "SELECT * FROM mytable LIMIT 5";
+
+  @BeforeClass
+  public void setUp()
+      throws Exception {
+    TestUtils.ensureDirectoriesExistAndEmpty(_tempDir, _segmentDir, _tarDir);
+
+    // Start the Pinot cluster
+    startZk();
+    startController();
+    startBroker();
+    startServer();
+
+    // Create and upload the schema and table config
+    Schema schema = new Schema.SchemaBuilder().setSchemaName(DEFAULT_TABLE_NAME)
+        .addSingleValueDimension(STRING_DIM_SV1, FieldSpec.DataType.STRING)
+        .addSingleValueDimension(STRING_DIM_SV2, FieldSpec.DataType.STRING)
+        .addSingleValueDimension(INT_DIM_SV1, FieldSpec.DataType.INT)
+        .addSingleValueDimension(LONG_DIM_SV1, FieldSpec.DataType.LONG)
+        .addSingleValueDimension(DOUBLE_DIM_SV1, FieldSpec.DataType.DOUBLE)
+        .addSingleValueDimension(BOOLEAN_DIM_SV1, FieldSpec.DataType.BOOLEAN)
+        .build();
+    addSchema(schema);
+    TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(DEFAULT_TABLE_NAME).build();
+    addTableConfig(tableConfig);
+
+    List<File> avroFiles = createAvroFiles();
+    ClusterIntegrationTestUtils.buildSegmentsFromAvro(avroFiles, tableConfig, schema, 0, _segmentDir, _tarDir);
+    uploadSegments(DEFAULT_TABLE_NAME, _tarDir);
+
+    // Wait for all documents loaded
+    waitForAllDocsLoaded(600_000L);
+  }
+
+  @AfterClass
+  public void tearDown()
+      throws IOException {
+    stopServer();
+    stopBroker();
+    stopController();
+    stopZk();
+    FileUtils.deleteDirectory(_tempDir);
+  }
+
+  @Override
+  protected void overrideBrokerConf(PinotConfiguration brokerConf) {
+    brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_CPU_TIME_MEASUREMENT, true);
+    brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
+  }
+
+  @Override
+  protected void overrideServerConf(PinotConfiguration serverConf) {
+    serverConf.setProperty(Server.CONFIG_OF_ENABLE_THREAD_CPU_TIME_MEASUREMENT, true);
+    serverConf.setProperty(Server.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
+    serverConf.setProperty(Server.CONFIG_OF_QUERY_EXECUTOR_NUM_GROUPS_LIMIT, Integer.MAX_VALUE);
+  }
+
+  @Override
+  protected long getCountStarResult() {
+    return NUM_SEGMENTS * NUM_DOCS_PER_SEGMENT;
+  }
+
+  @Test
+  public void testResourceUsageStats()
+      throws Exception {
+    JsonNode queryResponse = postQuery(SELECT_STAR_QUERY);
+    long offlineThreadMemAllocatedBytes = queryResponse.get("offlineThreadMemAllocatedBytes").asLong();
+    long offlineResponseSerMemAllocatedBytes = queryResponse.get("offlineResponseSerMemAllocatedBytes").asLong();
+    long offlineTotalMemAllocatedBytes = queryResponse.get("offlineTotalMemAllocatedBytes").asLong();
+
+    assertTrue(offlineThreadMemAllocatedBytes > 0);
+    assertTrue(offlineResponseSerMemAllocatedBytes > 0);
+    assertEquals(offlineThreadMemAllocatedBytes + offlineResponseSerMemAllocatedBytes, offlineTotalMemAllocatedBytes);
+
+    long offlineThreadCpuTimeNs = queryResponse.get("offlineThreadCpuTimeNs").asLong();
+    long offlineSystemActivitiesCpuTimeNs = queryResponse.get("offlineSystemActivitiesCpuTimeNs").asLong();
+    long offlineResponseSerializationCpuTimeNs = queryResponse.get("offlineResponseSerializationCpuTimeNs").asLong();
+    long offlineTotalCpuTimeNs = queryResponse.get("offlineTotalCpuTimeNs").asLong();
+    assertTrue(offlineThreadCpuTimeNs > 0);
+    assertTrue(offlineSystemActivitiesCpuTimeNs > 0);
+    assertTrue(offlineResponseSerializationCpuTimeNs > 0);
+    assertEquals(offlineThreadCpuTimeNs + offlineSystemActivitiesCpuTimeNs + offlineResponseSerializationCpuTimeNs,
+        offlineTotalCpuTimeNs);
+  }
+
+  @DataProvider
+  public String[] expensiveQueries() {
+    return new String[]{
+        LARGE_SELECT_STAR_QUERY, LARGE_DISTINCT_QUERY, LARGE_GROUP_BY_QUERY, LARGE_GROUP_BY_PAIRWISE_COMBINE_QUERY,
+        LARGE_GROUP_BY_SEQUENTIAL_COMBINE_QUERY
+    };
+  }
+
+  protected List<File> createAvroFiles()
+      throws IOException {
+    // Create Avro schema
+    org.apache.avro.Schema avroSchema = org.apache.avro.Schema.createRecord("myRecord", null, null, false);
+    avroSchema.setFields(List.of(new Field(STRING_DIM_SV1, org.apache.avro.Schema.create(Type.STRING), null, null),
+        new Field(STRING_DIM_SV2, org.apache.avro.Schema.create(Type.STRING), null, null),
+        new Field(INT_DIM_SV1, org.apache.avro.Schema.create(Type.INT), null, null),
+        new Field(LONG_DIM_SV1, org.apache.avro.Schema.create(Type.LONG), null, null),
+        new Field(DOUBLE_DIM_SV1, org.apache.avro.Schema.create(Type.DOUBLE), null, null),
+        new Field(BOOLEAN_DIM_SV1, org.apache.avro.Schema.create(Type.BOOLEAN), null, null)));
+
+    // Create Avro files
+    List<File> ret = new ArrayList<>();
+    for (int segmentId = 0; segmentId < NUM_SEGMENTS; segmentId++) {
+      File avroFile = new File(_tempDir, "data_" + segmentId + ".avro");
+      try (DataFileWriter<GenericData.Record> fileWriter = new DataFileWriter<>(new GenericDatumWriter<>(avroSchema))) {
+        fileWriter.create(avroSchema, avroFile);
+
+        int randBound = NUM_DOCS_PER_SEGMENT / 2;
+        Random random = new Random(0);
+        for (int docId = 0; docId < NUM_DOCS_PER_SEGMENT; docId++) {
+          GenericData.Record record = new GenericData.Record(avroSchema);
+          record.put(STRING_DIM_SV1, "test query killing");
+          record.put(STRING_DIM_SV2, "test query killing" + docId);
+          record.put(INT_DIM_SV1, random.nextInt(randBound));
+          record.put(LONG_DIM_SV1, random.nextLong());
+          record.put(DOUBLE_DIM_SV1, random.nextDouble());
+          record.put(BOOLEAN_DIM_SV1, true);
+          fileWriter.append(record);
+        }
+        ret.add(avroFile);
+      }
+    }
+    return ret;
+  }
+}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/CpuBasedBrokerQueryKillingIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/CpuBasedBrokerQueryKillingIntegrationTest.java
new file mode 100644
index 00000000000..8a633505bb4
--- /dev/null
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/CpuBasedBrokerQueryKillingIntegrationTest.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.pinot.core.accounting.ResourceUsageAccountantFactory;
+import org.apache.pinot.spi.env.PinotConfiguration;
+import org.apache.pinot.spi.exception.QueryErrorCode;
+import org.apache.pinot.spi.utils.CommonConstants;
+import org.apache.pinot.spi.utils.CommonConstants.Accounting;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+
+public class CpuBasedBrokerQueryKillingIntegrationTest extends BaseQueryKillingIntegrationTest {
+
+  @Override
+  protected void overrideBrokerConf(PinotConfiguration brokerConf) {
+    super.overrideBrokerConf(brokerConf);
+
+    String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
+    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME, ResourceUsageAccountantFactory.class.getName());
+    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_CPU_SAMPLING, true);
+    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_CPU_TIME_BASED_KILLING_ENABLED, true);
+    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_CPU_TIME_BASED_KILLING_THRESHOLD_MS, 500);
+    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_CRITICAL_LEVEL_HEAP_USAGE_RATIO, 1.1f);
+    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_PANIC_LEVEL_HEAP_USAGE_RATIO, 1.1f);
+  }
+
+  @Test
+  public void testCpuTimeKill()
+      throws Exception {
+    // Trigger broker side CPU kill with a large streaming query
+    // Do not run other queries because they can cause OOM on server first
+    setUseMultiStageQueryEngine(true);
+    verifyCpuTimeKill(LARGE_SELECT_STAR_QUERY, postQuery(LARGE_SELECT_STAR_QUERY));
+  }
+
+  @Test
+  public void testCpuTimeKillMultipleQueries()
+      throws Exception {
+    // Trigger broker side CPU kill with a large streaming query
+    // Do not run other queries because they can cause OOM on server first
+    setUseMultiStageQueryEngine(true);
+    AtomicReference<JsonNode> queryResponse1 = new AtomicReference<>();
+    AtomicReference<JsonNode> queryResponse2 = new AtomicReference<>();
+    AtomicReference<JsonNode> queryResponse3 = new AtomicReference<>();
+    CountDownLatch countDownLatch = new CountDownLatch(3);
+    ExecutorService executor = Executors.newFixedThreadPool(3);
+    executor.submit(() -> {
+      queryResponse1.set(postQuery(LARGE_SELECT_STAR_QUERY));
+      countDownLatch.countDown();
+      return null;
+    });
+    executor.submit(() -> {
+      queryResponse2.set(postQuery(AGGREGATE_QUERY));
+      countDownLatch.countDown();
+      return null;
+    });
+    executor.submit(() -> {
+      queryResponse3.set(postQuery(SELECT_STAR_QUERY));
+      countDownLatch.countDown();
+      return null;
+    });
+    executor.shutdown();
+    countDownLatch.await();
+
+    JsonNode response = queryResponse1.get();
+    verifyCpuTimeKill(LARGE_SELECT_STAR_QUERY, response);
+
+    response = queryResponse2.get();
+    JsonNode exceptionsNode = response.get("exceptions");
+    assertNotNull(exceptionsNode, "Missing exceptions for query: " + AGGREGATE_QUERY);
+    assertTrue(exceptionsNode.isEmpty(),
+        "Expected no exceptions for query: " + AGGREGATE_QUERY + ", but got: " + exceptionsNode);
+
+    response = queryResponse3.get();
+    exceptionsNode = response.get("exceptions");
+    assertNotNull(exceptionsNode, "Missing exceptions for query: " + SELECT_STAR_QUERY);
+    assertTrue(exceptionsNode.isEmpty(),
+        "Expected no exceptions for query: " + SELECT_STAR_QUERY + ", but got: " + exceptionsNode);
+  }
+
+  private void verifyCpuTimeKill(String query, JsonNode response) {
+    JsonNode exceptionsNode = response.get("exceptions");
+    assertNotNull(exceptionsNode, "Missing exceptions for query: " + query);
+    assertEquals(exceptionsNode.size(), 1, "Expected 1 exception for query: " + query + ", but got: " + exceptionsNode);
+    JsonNode exceptionNode = exceptionsNode.get(0);
+    JsonNode errorCodeNode = exceptionNode.get("errorCode");
+    assertNotNull(errorCodeNode, "Missing errorCode from exception: " + exceptionNode);
+    int errorCode = errorCodeNode.asInt();
+    assertEquals(errorCode, QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED.getId(),
+        "Unexpected error code: " + errorCode + " from exception: " + exceptionNode);
+    JsonNode messageNode = exceptionNode.get("message");
+    assertNotNull(messageNode, "Missing message from exception: " + exceptionNode);
+    String message = messageNode.asText();
+    assertTrue(message.contains("CPU time based killed on BROKER"),
+        "Unexpected exception message: " + message + " from exception: " + exceptionNode);
+  }
+}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/CpuBasedServerQueryKillingIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/CpuBasedServerQueryKillingIntegrationTest.java
new file mode 100644
index 00000000000..17b51bd77a1
--- /dev/null
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/CpuBasedServerQueryKillingIntegrationTest.java
@@ -0,0 +1,120 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.pinot.core.accounting.ResourceUsageAccountantFactory;
+import org.apache.pinot.spi.env.PinotConfiguration;
+import org.apache.pinot.spi.exception.QueryErrorCode;
+import org.apache.pinot.spi.utils.CommonConstants;
+import org.apache.pinot.spi.utils.CommonConstants.Accounting;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+
+public class CpuBasedServerQueryKillingIntegrationTest extends BaseQueryKillingIntegrationTest {
+
+  @Override
+  protected void overrideServerConf(PinotConfiguration serverConf) {
+    super.overrideServerConf(serverConf);
+
+    String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
+    serverConf.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME, ResourceUsageAccountantFactory.class.getName());
+    serverConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_CPU_SAMPLING, true);
+    serverConf.setProperty(prefix + Accounting.CONFIG_OF_CPU_TIME_BASED_KILLING_ENABLED, true);
+    serverConf.setProperty(prefix + Accounting.CONFIG_OF_CPU_TIME_BASED_KILLING_THRESHOLD_MS, 500);
+    serverConf.setProperty(prefix + Accounting.CONFIG_OF_CRITICAL_LEVEL_HEAP_USAGE_RATIO, 1.1f);
+    serverConf.setProperty(prefix + Accounting.CONFIG_OF_PANIC_LEVEL_HEAP_USAGE_RATIO, 1.1f);
+  }
+
+  @Test(dataProvider = "expensiveQueries")
+  public void testCpuTimeKill(String query)
+      throws Exception {
+    verifyCpuTimeKill(query, postQuery(query));
+    setUseMultiStageQueryEngine(true);
+    verifyCpuTimeKill(query, postQuery(query));
+  }
+
+  @Test(dataProvider = "useBothQueryEngines")
+  public void testCpuTimeKillMultipleQueries(boolean useMultiStageQueryEngine)
+      throws Exception {
+    setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+    AtomicReference<JsonNode> queryResponse1 = new AtomicReference<>();
+    AtomicReference<JsonNode> queryResponse2 = new AtomicReference<>();
+    AtomicReference<JsonNode> queryResponse3 = new AtomicReference<>();
+    CountDownLatch countDownLatch = new CountDownLatch(3);
+    ExecutorService executor = Executors.newFixedThreadPool(3);
+    executor.submit(() -> {
+      queryResponse1.set(postQuery(LARGE_DISTINCT_QUERY));
+      countDownLatch.countDown();
+      return null;
+    });
+    executor.submit(() -> {
+      queryResponse2.set(postQuery(AGGREGATE_QUERY));
+      countDownLatch.countDown();
+      return null;
+    });
+    executor.submit(() -> {
+      queryResponse3.set(postQuery(SELECT_STAR_QUERY));
+      countDownLatch.countDown();
+      return null;
+    });
+    executor.shutdown();
+    countDownLatch.await();
+
+    JsonNode response = queryResponse1.get();
+    verifyCpuTimeKill(LARGE_DISTINCT_QUERY, response);
+
+    response = queryResponse2.get();
+    JsonNode exceptionsNode = response.get("exceptions");
+    assertNotNull(exceptionsNode, "Missing exceptions for query: " + AGGREGATE_QUERY);
+    assertTrue(exceptionsNode.isEmpty(),
+        "Expected no exceptions for query: " + AGGREGATE_QUERY + ", but got: " + exceptionsNode);
+
+    response = queryResponse3.get();
+    exceptionsNode = response.get("exceptions");
+    assertNotNull(exceptionsNode, "Missing exceptions for query: " + SELECT_STAR_QUERY);
+    assertTrue(exceptionsNode.isEmpty(),
+        "Expected no exceptions for query: " + SELECT_STAR_QUERY + ", but got: " + exceptionsNode);
+  }
+
+  private void verifyCpuTimeKill(String query, JsonNode response) {
+    JsonNode exceptionsNode = response.get("exceptions");
+    assertNotNull(exceptionsNode, "Missing exceptions for query: " + query);
+    assertEquals(exceptionsNode.size(), 1, "Expected 1 exception for query: " + query + ", but got: " + exceptionsNode);
+    JsonNode exceptionNode = exceptionsNode.get(0);
+    JsonNode errorCodeNode = exceptionNode.get("errorCode");
+    assertNotNull(errorCodeNode, "Missing errorCode from exception: " + exceptionNode);
+    int errorCode = errorCodeNode.asInt();
+    assertEquals(errorCode, QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED.getId(),
+        "Unexpected error code: " + errorCode + " from exception: " + exceptionNode);
+    JsonNode messageNode = exceptionNode.get("message");
+    assertNotNull(messageNode, "Missing message from exception: " + exceptionNode);
+    String message = messageNode.asText();
+    assertTrue(message.contains("CPU time based killed on SERVER"),
+        "Unexpected exception message: " + message + " from exception: " + exceptionNode);
+  }
+}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MemoryBasedServerQueryKillingIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MemoryBasedServerQueryKillingIntegrationTest.java
new file mode 100644
index 00000000000..eb99471023c
--- /dev/null
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MemoryBasedServerQueryKillingIntegrationTest.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.pinot.core.accounting.ResourceUsageAccountantFactory;
+import org.apache.pinot.spi.env.PinotConfiguration;
+import org.apache.pinot.spi.exception.QueryErrorCode;
+import org.apache.pinot.spi.utils.CommonConstants;
+import org.apache.pinot.spi.utils.CommonConstants.Accounting;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+
+public class MemoryBasedServerQueryKillingIntegrationTest extends BaseQueryKillingIntegrationTest {
+
+  @Override
+  protected void overrideServerConf(PinotConfiguration serverConf) {
+    super.overrideServerConf(serverConf);
+
+    String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
+    serverConf.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME, ResourceUsageAccountantFactory.class.getName());
+    serverConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
+    serverConf.setProperty(prefix + Accounting.CONFIG_OF_OOM_PROTECTION_KILLING_QUERY, true);
+    serverConf.setProperty(prefix + Accounting.CONFIG_OF_ALARMING_LEVEL_HEAP_USAGE_RATIO, 0f);
+    serverConf.setProperty(prefix + Accounting.CONFIG_OF_CRITICAL_LEVEL_HEAP_USAGE_RATIO, 0.15f);
+  }
+
+  @Test(dataProvider = "expensiveQueries")
+  public void testOomKill(String query)
+      throws Exception {
+    verifyOomKill(query, postQuery(query));
+    setUseMultiStageQueryEngine(true);
+    verifyOomKill(query, postQuery(query));
+  }
+
+  @Test(dataProvider = "useBothQueryEngines")
+  public void testOomKillMultipleQueries(boolean useMultiStageQueryEngine)
+      throws Exception {
+    setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+    AtomicReference<JsonNode> queryResponse1 = new AtomicReference<>();
+    AtomicReference<JsonNode> queryResponse2 = new AtomicReference<>();
+    AtomicReference<JsonNode> queryResponse3 = new AtomicReference<>();
+    CountDownLatch countDownLatch = new CountDownLatch(3);
+    ExecutorService executor = Executors.newFixedThreadPool(3);
+    executor.submit(() -> {
+      queryResponse1.set(postQuery(LARGE_DISTINCT_QUERY));
+      countDownLatch.countDown();
+      return null;
+    });
+    executor.submit(() -> {
+      queryResponse2.set(postQuery(AGGREGATE_QUERY));
+      countDownLatch.countDown();
+      return null;
+    });
+    executor.submit(() -> {
+      queryResponse3.set(postQuery(SELECT_STAR_QUERY));
+      countDownLatch.countDown();
+      return null;
+    });
+    executor.shutdown();
+    countDownLatch.await();
+
+    JsonNode response = queryResponse1.get();
+    verifyOomKill(LARGE_DISTINCT_QUERY, response);
+
+    response = queryResponse2.get();
+    JsonNode exceptionsNode = response.get("exceptions");
+    assertNotNull(exceptionsNode, "Missing exceptions for query: " + AGGREGATE_QUERY);
+    assertTrue(exceptionsNode.isEmpty(),
+        "Expected no exceptions for query: " + AGGREGATE_QUERY + ", but got: " + exceptionsNode);
+
+    response = queryResponse3.get();
+    exceptionsNode = response.get("exceptions");
+    assertNotNull(exceptionsNode, "Missing exceptions for query: " + SELECT_STAR_QUERY);
+    assertTrue(exceptionsNode.isEmpty(),
+        "Expected no exceptions for query: " + SELECT_STAR_QUERY + ", but got: " + exceptionsNode);
+  }
+
+  private void verifyOomKill(String query, JsonNode response) {
+    JsonNode exceptionsNode = response.get("exceptions");
+    assertNotNull(exceptionsNode, "Missing exceptions for query: " + query);
+    assertEquals(exceptionsNode.size(), 1, "Expected 1 exception for query: " + query + ", but got: " + exceptionsNode);
+    JsonNode exceptionNode = exceptionsNode.get(0);
+    JsonNode errorCodeNode = exceptionNode.get("errorCode");
+    assertNotNull(errorCodeNode, "Missing errorCode from exception: " + exceptionNode);
+    int errorCode = errorCodeNode.asInt();
+    assertEquals(errorCode, QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED.getId(),
+        "Unexpected error code: " + errorCode + " from exception: " + exceptionNode);
+    JsonNode messageNode = exceptionNode.get("message");
+    assertNotNull(messageNode, "Missing message from exception: " + exceptionNode);
+    String message = messageNode.asText();
+    assertTrue(message.contains("OOM killed on SERVER"),
+        "Unexpected exception message: " + message + " from exception: " + exceptionNode);
+  }
+}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiNodesOfflineClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiNodesOfflineClusterIntegrationTest.java
index 851e3de140c..fc269a0c648 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiNodesOfflineClusterIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiNodesOfflineClusterIntegrationTest.java
@@ -24,13 +24,16 @@ import java.util.Map;
 import org.apache.helix.model.ExternalView;
 import org.apache.helix.model.IdealState;
 import org.apache.pinot.broker.broker.helix.BaseBrokerStarter;
+import org.apache.pinot.core.accounting.ResourceUsageAccountantFactory;
 import org.apache.pinot.server.starter.helix.BaseServerStarter;
 import org.apache.pinot.spi.env.PinotConfiguration;
 import org.apache.pinot.spi.exception.QueryErrorCode;
+import org.apache.pinot.spi.utils.CommonConstants;
 import org.apache.pinot.spi.utils.CommonConstants.Broker;
 import org.apache.pinot.spi.utils.CommonConstants.Broker.FailureDetector;
 import org.apache.pinot.spi.utils.CommonConstants.Helix;
 import org.apache.pinot.spi.utils.CommonConstants.Helix.StateModel.BrokerResourceStateModel;
+import org.apache.pinot.spi.utils.CommonConstants.Server;
 import org.apache.pinot.util.TestUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -68,8 +71,32 @@ public class MultiNodesOfflineClusterIntegrationTest extends OfflineClusterInteg
   @Override
   protected void overrideBrokerConf(PinotConfiguration brokerConf) {
     super.overrideBrokerConf(brokerConf);
+
     brokerConf.setProperty(FailureDetector.CONFIG_OF_TYPE, FailureDetector.Type.CONNECTION.name());
     brokerConf.setProperty(Broker.CONFIG_OF_USE_LEAF_SERVER_FOR_INTERMEDIATE_STAGE, true);
+
+    // Enable thread CPU/memory tracking but not killing queries
+    brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_CPU_TIME_MEASUREMENT, true);
+    brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
+    String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
+    brokerConf.setProperty(prefix + CommonConstants.Accounting.CONFIG_OF_FACTORY_NAME,
+        ResourceUsageAccountantFactory.class.getName());
+    brokerConf.setProperty(prefix + CommonConstants.Accounting.CONFIG_OF_ENABLE_THREAD_CPU_SAMPLING, true);
+    brokerConf.setProperty(prefix + CommonConstants.Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
+  }
+
+  @Override
+  protected void overrideServerConf(PinotConfiguration serverConf) {
+    super.overrideServerConf(serverConf);
+
+    // Enable thread CPU/memory tracking but not killing queries
+    serverConf.setProperty(Server.CONFIG_OF_ENABLE_THREAD_CPU_TIME_MEASUREMENT, true);
+    serverConf.setProperty(Server.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
+    String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
+    serverConf.setProperty(prefix + CommonConstants.Accounting.CONFIG_OF_FACTORY_NAME,
+        ResourceUsageAccountantFactory.class.getName());
+    serverConf.setProperty(prefix + CommonConstants.Accounting.CONFIG_OF_ENABLE_THREAD_CPU_SAMPLING, true);
+    serverConf.setProperty(prefix + CommonConstants.Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
   }
 
   @Test
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OOMProtectionEnabledIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OOMProtectionEnabledIntegrationTest.java
deleted file mode 100644
index cf88aa41429..00000000000
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OOMProtectionEnabledIntegrationTest.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.integration.tests;
-
-import java.io.File;
-import java.util.List;
-import org.apache.pinot.core.accounting.PerQueryCPUMemAccountantFactory;
-import org.apache.pinot.spi.config.table.TableConfig;
-import org.apache.pinot.spi.data.Schema;
-import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.utils.CommonConstants;
-import org.apache.pinot.spi.utils.CommonConstants.Accounting;
-import org.apache.pinot.spi.utils.CommonConstants.Broker;
-import org.apache.pinot.spi.utils.CommonConstants.Server;
-import org.apache.pinot.util.TestUtils;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-
-public class OOMProtectionEnabledIntegrationTest extends BaseClusterIntegrationTestSet {
-  private static final int NUM_OFFLINE_SEGMENTS = 8;
-  private static final int NUM_REALTIME_SEGMENTS = 6;
-
-  @Override
-  protected void overrideBrokerConf(PinotConfiguration configuration) {
-    configuration.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
-
-    String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
-    configuration.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
-    configuration.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME,
-        PerQueryCPUMemAccountantFactory.class.getName());
-  }
-
-  @Override
-  protected void overrideServerConf(PinotConfiguration configuration) {
-    configuration.setProperty(Server.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
-
-    String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
-    configuration.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
-    configuration.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME,
-        PerQueryCPUMemAccountantFactory.class.getName());
-  }
-
-  @BeforeClass
-  public void setUp()
-      throws Exception {
-    TestUtils.ensureDirectoriesExistAndEmpty(_tempDir, _segmentDir, _tarDir);
-
-    // Start Zk, Kafka and Pinot
-    startZk();
-    startController();
-    startBroker();
-    startServer();
-    startKafka();
-
-    List<File> avroFiles = getAllAvroFiles();
-    List<File> offlineAvroFiles = getOfflineAvroFiles(avroFiles, NUM_OFFLINE_SEGMENTS);
-    List<File> realtimeAvroFiles = getRealtimeAvroFiles(avroFiles, NUM_REALTIME_SEGMENTS);
-
-    // Create and upload the schema and table config
-    Schema schema = createSchema();
-    addSchema(schema);
-    TableConfig offlineTableConfig = createOfflineTableConfig();
-    addTableConfig(offlineTableConfig);
-    addTableConfig(createRealtimeTableConfig(realtimeAvroFiles.get(0)));
-
-    // Create and upload segments
-    ClusterIntegrationTestUtils.buildSegmentsFromAvro(offlineAvroFiles, offlineTableConfig, schema, 0, _segmentDir,
-        _tarDir);
-    uploadSegments(getTableName(), _tarDir);
-
-    // Push data into Kafka
-    pushAvroIntoKafka(realtimeAvroFiles);
-
-    // Set up the H2 connection
-    setUpH2Connection(avroFiles);
-
-    // Initialize the query generator
-    setUpQueryGenerator(avroFiles);
-
-    // Wait for all documents loaded
-    waitForAllDocsLoaded(100_000L);
-  }
-
-  @Test(dataProvider = "useBothQueryEngines")
-  public void testHardcodedQueries(boolean useMultiStageEngine)
-      throws Exception {
-    setUseMultiStageQueryEngine(useMultiStageEngine);
-    super.testHardcodedQueries();
-  }
-}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterCpuBasedServerQueryKillingTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterCpuBasedServerQueryKillingTest.java
deleted file mode 100644
index dffffcda82b..00000000000
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterCpuBasedServerQueryKillingTest.java
+++ /dev/null
@@ -1,239 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.integration.tests;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.collect.ImmutableList;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.IntStream;
-import org.apache.avro.file.DataFileWriter;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericDatumWriter;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.pinot.core.accounting.PerQueryCPUMemAccountantFactory;
-import org.apache.pinot.spi.config.table.TableConfig;
-import org.apache.pinot.spi.config.table.TableType;
-import org.apache.pinot.spi.data.FieldSpec;
-import org.apache.pinot.spi.data.Schema;
-import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.utils.CommonConstants;
-import org.apache.pinot.spi.utils.CommonConstants.Accounting;
-import org.apache.pinot.spi.utils.CommonConstants.Broker;
-import org.apache.pinot.spi.utils.CommonConstants.Server;
-import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
-import org.apache.pinot.util.TestUtils;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertTrue;
-
-
-/**
- * Integration test for heap size based server query killing, this works only for xmx4G
- * <p>
- * Query killing isn't currently supported in the v2 multi-stage query engine so these tests only run on the v1 engine.
- */
-public class OfflineClusterCpuBasedServerQueryKillingTest extends BaseClusterIntegrationTestSet {
-  public static final String STRING_DIM_SV1 = "stringDimSV1";
-  public static final String STRING_DIM_SV2 = "stringDimSV2";
-  public static final String INT_DIM_SV1 = "intDimSV1";
-  public static final String LONG_DIM_SV1 = "longDimSV1";
-  public static final String DOUBLE_DIM_SV1 = "doubleDimSV1";
-  public static final String BOOLEAN_DIM_SV1 = "booleanDimSV1";
-  private static final String OOM_QUERY =
-      "SELECT PERCENTILETDigest(doubleDimSV1, 50) AS digest, intDimSV1 FROM mytable GROUP BY intDimSV1"
-          + " ORDER BY digest LIMIT 15000";
-
-  private static final String DIGEST_QUERY_1 =
-      "SELECT PERCENTILETDigest(doubleDimSV1, 50) AS digest FROM mytable";
-  private static final String COUNT_STAR_QUERY =
-      "SELECT * FROM mytable LIMIT 5";
-
-  private static final ExecutorService EXECUTOR_SERVICE = Executors.newFixedThreadPool(20, r -> {
-    Thread thread = new Thread(r);
-    thread.setDaemon(false);
-    return thread;
-  });
-
-  @BeforeClass
-  public void setUp()
-      throws Exception {
-    TestUtils.ensureDirectoriesExistAndEmpty(_tempDir, _segmentDir, _tarDir);
-
-    // Start the Pinot cluster
-    startZk();
-    startController();
-    startBroker();
-    startServer();
-
-    // Create and upload the schema and table config
-    Schema schema = new Schema.SchemaBuilder().setSchemaName(DEFAULT_SCHEMA_NAME)
-        .addSingleValueDimension(STRING_DIM_SV1, FieldSpec.DataType.STRING)
-        .addSingleValueDimension(STRING_DIM_SV2, FieldSpec.DataType.STRING)
-        .addSingleValueDimension(INT_DIM_SV1, FieldSpec.DataType.INT)
-        .addSingleValueDimension(LONG_DIM_SV1, FieldSpec.DataType.LONG)
-        .addSingleValueDimension(DOUBLE_DIM_SV1, FieldSpec.DataType.DOUBLE)
-        .addSingleValueDimension(BOOLEAN_DIM_SV1, FieldSpec.DataType.BOOLEAN)
-        .build();
-    addSchema(schema);
-    TableConfig tableConfig = createOfflineTableConfig();
-    addTableConfig(tableConfig);
-
-    List<File> avroFiles = createAvroFile();
-    ClusterIntegrationTestUtils.buildSegmentsFromAvro(avroFiles, tableConfig, schema, 0, _segmentDir, _tarDir);
-    uploadSegments(DEFAULT_TABLE_NAME, _tarDir);
-
-    //Wait for all documents loaded
-    waitForAllDocsLoaded(10_000L);
-  }
-
-  protected void overrideBrokerConf(PinotConfiguration brokerConf) {
-    brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_CPU_TIME_MEASUREMENT, true);
-  }
-
-  protected void overrideServerConf(PinotConfiguration serverConf) {
-    serverConf.setProperty(Server.CONFIG_OF_ENABLE_THREAD_CPU_TIME_MEASUREMENT, true);
-
-    String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME, PerQueryCPUMemAccountantFactory.class.getName());
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_CPU_SAMPLING, true);
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_CPU_TIME_BASED_KILLING_ENABLED, true);
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_CPU_TIME_BASED_KILLING_THRESHOLD_MS, 1_000);
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_CRITICAL_LEVEL_HEAP_USAGE_RATIO, 1.1f);
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_PANIC_LEVEL_HEAP_USAGE_RATIO, 1.1f);
-  }
-
-  protected long getCountStarResult() {
-    return 3_000_000;
-  }
-
-  protected String getTimeColumnName() {
-    return null;
-  }
-
-  protected TableConfig createOfflineTableConfig() {
-    return new TableConfigBuilder(TableType.OFFLINE).setTableName(getTableName())
-        .setTimeColumnName(getTimeColumnName()).setFieldConfigList(getFieldConfigs()).setNumReplicas(getNumReplicas())
-        .setSegmentVersion(getSegmentVersion()).setLoadMode(getLoadMode()).setTaskConfig(getTaskConfig())
-        .setBrokerTenant(getBrokerTenant()).setServerTenant(getServerTenant()).setIngestionConfig(getIngestionConfig())
-        .setQueryConfig(getQueryConfig()).setNullHandlingEnabled(getNullHandlingEnabled())
-        .setSegmentPartitionConfig(getSegmentPartitionConfig())
-        .build();
-  }
-
-  @Test(dataProvider = "useBothQueryEngines")
-  public void testDigestTimeoutMultipleQueries(boolean useMultiStageQueryEngine)
-      throws Exception {
-    setUseMultiStageQueryEngine(useMultiStageQueryEngine);
-    notSupportedInV2();
-    AtomicReference<JsonNode> queryResponse1 = new AtomicReference<>();
-    AtomicReference<JsonNode> queryResponse2 = new AtomicReference<>();
-    AtomicReference<JsonNode> queryResponse3 = new AtomicReference<>();
-
-    CountDownLatch countDownLatch = new CountDownLatch(3);
-
-    EXECUTOR_SERVICE.submit(
-        () -> {
-          try {
-            queryResponse1.set(postQuery(OOM_QUERY));
-            countDownLatch.countDown();
-          } catch (Exception ignored) {
-          }
-        }
-    );
-    EXECUTOR_SERVICE.submit(
-        () -> {
-          try {
-            queryResponse2.set(postQuery(DIGEST_QUERY_1));
-            countDownLatch.countDown();
-          } catch (Exception ignored) {
-          }
-        }
-    );
-    EXECUTOR_SERVICE.submit(
-        () -> {
-          try {
-            queryResponse3.set(postQuery(COUNT_STAR_QUERY));
-            countDownLatch.countDown();
-          } catch (Exception ignored) {
-          }
-        }
-    );
-    countDownLatch.await();
-    String exceptions = queryResponse1.get().get("exceptions").toString();
-    assertTrue(exceptions.contains("CPU time based killed on SERVER"));
-    assertFalse(StringUtils.isEmpty(queryResponse2.get().get("exceptions").toString()));
-    assertFalse(StringUtils.isEmpty(queryResponse3.get().get("exceptions").toString()));
-  }
-
-  private List<File> createAvroFile()
-      throws IOException {
-
-    // create avro schema
-    org.apache.avro.Schema avroSchema = org.apache.avro.Schema.createRecord("myRecord", null, null, false);
-    avroSchema.setFields(ImmutableList.of(
-        new org.apache.avro.Schema.Field(STRING_DIM_SV1,
-            org.apache.avro.Schema.create(org.apache.avro.Schema.Type.STRING), null, null),
-        new org.apache.avro.Schema.Field(STRING_DIM_SV2,
-            org.apache.avro.Schema.create(org.apache.avro.Schema.Type.STRING), null, null),
-        new org.apache.avro.Schema.Field(INT_DIM_SV1,
-            org.apache.avro.Schema.create(org.apache.avro.Schema.Type.INT), null, null),
-        new org.apache.avro.Schema.Field(LONG_DIM_SV1,
-            org.apache.avro.Schema.create(org.apache.avro.Schema.Type.LONG), null, null),
-        new org.apache.avro.Schema.Field(DOUBLE_DIM_SV1,
-            org.apache.avro.Schema.create(org.apache.avro.Schema.Type.DOUBLE), null, null),
-        new org.apache.avro.Schema.Field(BOOLEAN_DIM_SV1,
-            org.apache.avro.Schema.create(org.apache.avro.Schema.Type.BOOLEAN), null, null)));
-
-    List<File> ret = new ArrayList<>();
-    for (int file = 0; file < 3; file++) {
-      // create avro file
-      File avroFile = new File(_tempDir, "data_" + file + ".avro");
-      try (DataFileWriter<GenericData.Record> fileWriter = new DataFileWriter<>(new GenericDatumWriter<>(avroSchema))) {
-        fileWriter.create(avroSchema, avroFile);
-
-        int numDocs = 1_000_000;
-        int randBound = numDocs / 2;
-        Random random = new Random(0);
-        IntStream randomInt = random.ints(0, 100_000);
-        for (int docId = 0; docId < numDocs; docId++) {
-          GenericData.Record record = new GenericData.Record(avroSchema);
-          record.put(STRING_DIM_SV1, "test query killing");
-          record.put(STRING_DIM_SV2, "test query killing");
-          record.put(INT_DIM_SV1, random.nextInt(randBound));
-          record.put(LONG_DIM_SV1, random.nextLong());
-          record.put(DOUBLE_DIM_SV1, random.nextDouble());
-          record.put(BOOLEAN_DIM_SV1, true);
-          fileWriter.append(record);
-        }
-        ret.add(avroFile);
-      }
-    }
-    return ret;
-  }
-}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedBrokerQueryKillingTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedBrokerQueryKillingTest.java
deleted file mode 100644
index 4b05492afd1..00000000000
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedBrokerQueryKillingTest.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.integration.tests;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.collect.ImmutableList;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.IntStream;
-import org.apache.avro.file.DataFileWriter;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericDatumWriter;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.pinot.core.accounting.PerQueryCPUMemAccountantFactory;
-import org.apache.pinot.spi.config.table.TableConfig;
-import org.apache.pinot.spi.config.table.TableType;
-import org.apache.pinot.spi.data.FieldSpec;
-import org.apache.pinot.spi.data.Schema;
-import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.exception.QueryErrorCode;
-import org.apache.pinot.spi.utils.CommonConstants;
-import org.apache.pinot.spi.utils.CommonConstants.Accounting;
-import org.apache.pinot.spi.utils.CommonConstants.Broker;
-import org.apache.pinot.spi.utils.CommonConstants.Server;
-import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
-import org.apache.pinot.util.TestUtils;
-import org.testng.annotations.BeforeClass;
-
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertTrue;
-
-
-/**
- * Integration test for heap size based broker query killing, this works only for xmx4G
- * TODO: re-enable or remove this after we resolve https://github.com/apache/pinot/issues/11099
- */
-public class OfflineClusterMemBasedBrokerQueryKillingTest extends BaseClusterIntegrationTestSet {
-
-  public static final String STRING_DIM_SV1 = "stringDimSV1";
-  public static final String STRING_DIM_SV2 = "stringDimSV2";
-  public static final String INT_DIM_SV1 = "intDimSV1";
-  public static final String LONG_DIM_SV1 = "longDimSV1";
-  public static final String DOUBLE_DIM_SV1 = "doubleDimSV1";
-  public static final String BOOLEAN_DIM_SV1 = "booleanDimSV1";
-  private static final int NUM_SERVERS = 3;
-  private static final String OOM_QUERY =
-      "SELECT PERCENTILETDigest(doubleDimSV1, 50) AS digest, intDimSV1 FROM mytable WHERE intDimSV1 > 450000 GROUP BY"
-          + " intDimSV1 ORDER BY digest LIMIT 15000";
-
-  private static final String DIGEST_QUERY_1 =
-      "SELECT PERCENTILETDigest(doubleDimSV1, 50) AS digest FROM mytable";
-  private static final String COUNT_STAR_QUERY =
-      "SELECT * FROM mytable LIMIT 5";
-
-  private static final ExecutorService EXECUTOR_SERVICE = Executors.newFixedThreadPool(20, r -> {
-    Thread thread = new Thread(r);
-    thread.setDaemon(false);
-    return thread;
-  });
-
-  @BeforeClass
-  public void setUp()
-      throws Exception {
-    TestUtils.ensureDirectoriesExistAndEmpty(_tempDir, _segmentDir, _tarDir);
-
-    // Start the Pinot cluster
-    startZk();
-    startController();
-    startBroker();
-    startServers(NUM_SERVERS);
-
-    // Create and upload the schema and table config
-    Schema schema = new Schema.SchemaBuilder().setSchemaName(DEFAULT_SCHEMA_NAME)
-        .addSingleValueDimension(STRING_DIM_SV1, FieldSpec.DataType.STRING)
-        .addSingleValueDimension(STRING_DIM_SV2, FieldSpec.DataType.STRING)
-        .addSingleValueDimension(INT_DIM_SV1, FieldSpec.DataType.INT)
-        .addSingleValueDimension(LONG_DIM_SV1, FieldSpec.DataType.LONG)
-        .addSingleValueDimension(DOUBLE_DIM_SV1, FieldSpec.DataType.DOUBLE)
-        .addSingleValueDimension(BOOLEAN_DIM_SV1, FieldSpec.DataType.BOOLEAN)
-        .build();
-    addSchema(schema);
-    TableConfig tableConfig = createOfflineTableConfig();
-    addTableConfig(tableConfig);
-
-    List<File> avroFiles = createAvroFile();
-    ClusterIntegrationTestUtils.buildSegmentsFromAvro(avroFiles, tableConfig, schema, 0, _segmentDir, _tarDir);
-    uploadSegments(DEFAULT_TABLE_NAME, _tarDir);
-
-    //Wait for all documents loaded
-    waitForAllDocsLoaded(10_000L);
-  }
-
-  protected void overrideBrokerConf(PinotConfiguration brokerConf) {
-    brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
-
-    String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
-    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME, PerQueryCPUMemAccountantFactory.class.getName());
-    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
-    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_OOM_PROTECTION_KILLING_QUERY, true);
-    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_ALARMING_LEVEL_HEAP_USAGE_RATIO, 0f);
-    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_CRITICAL_LEVEL_HEAP_USAGE_RATIO, 0.4f);
-    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_PANIC_LEVEL_HEAP_USAGE_RATIO, 1.1f);
-    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_MIN_MEMORY_FOOTPRINT_TO_KILL_RATIO, 0.0025f);
-  }
-
-  @Override
-  protected void overrideServerConf(PinotConfiguration serverConf) {
-    serverConf.setProperty(Server.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
-  }
-
-  protected long getCountStarResult() {
-    return 3_000_000;
-  }
-
-  protected String getTimeColumnName() {
-    return null;
-  }
-
-  protected TableConfig createOfflineTableConfig() {
-    return new TableConfigBuilder(TableType.OFFLINE).setTableName(getTableName())
-        .setTimeColumnName(getTimeColumnName()).setFieldConfigList(getFieldConfigs()).setNumReplicas(getNumReplicas())
-        .setSegmentVersion(getSegmentVersion()).setLoadMode(getLoadMode()).setTaskConfig(getTaskConfig())
-        .setBrokerTenant(getBrokerTenant()).setServerTenant(getServerTenant()).setIngestionConfig(getIngestionConfig())
-        .setQueryConfig(getQueryConfig()).setNullHandlingEnabled(getNullHandlingEnabled())
-        .setSegmentPartitionConfig(getSegmentPartitionConfig())
-        .build();
-  }
-
-  public void testDigestOOMMultipleQueries()
-      throws Exception {
-    AtomicReference<JsonNode> queryResponse1 = new AtomicReference<>();
-    AtomicReference<JsonNode> queryResponse2 = new AtomicReference<>();
-    AtomicReference<JsonNode> queryResponse3 = new AtomicReference<>();
-
-    CountDownLatch countDownLatch = new CountDownLatch(3);
-
-    EXECUTOR_SERVICE.submit(
-        () -> {
-          try {
-            queryResponse1.set(postQuery(OOM_QUERY));
-            countDownLatch.countDown();
-          } catch (Exception ignored) {
-          }
-        }
-    );
-    EXECUTOR_SERVICE.submit(
-        () -> {
-          try {
-            queryResponse2.set(postQuery(DIGEST_QUERY_1));
-            countDownLatch.countDown();
-          } catch (Exception ignored) {
-          }
-        }
-    );
-    EXECUTOR_SERVICE.submit(
-        () -> {
-          try {
-            queryResponse3.set(postQuery(COUNT_STAR_QUERY));
-            countDownLatch.countDown();
-          } catch (Exception ignored) {
-          }
-        }
-    );
-    countDownLatch.await();
-    String exceptions = queryResponse1.get().get("exceptions").toString();
-    assertTrue(exceptions.contains("Interrupted in broker reduce phase"));
-    assertTrue(exceptions.contains("\"errorCode\":" + QueryErrorCode.QUERY_CANCELLATION.getId()));
-    assertTrue(exceptions.contains("got killed on BROKER"));
-    assertFalse(StringUtils.isEmpty(queryResponse2.get().get("exceptions").toString()));
-    assertFalse(StringUtils.isEmpty(queryResponse3.get().get("exceptions").toString()));
-  }
-
-  private List<File> createAvroFile()
-      throws IOException {
-
-    // create avro schema
-    org.apache.avro.Schema avroSchema = org.apache.avro.Schema.createRecord("myRecord", null, null, false);
-    avroSchema.setFields(ImmutableList.of(
-        new org.apache.avro.Schema.Field(STRING_DIM_SV1,
-            org.apache.avro.Schema.create(org.apache.avro.Schema.Type.STRING), null, null),
-        new org.apache.avro.Schema.Field(STRING_DIM_SV2,
-            org.apache.avro.Schema.create(org.apache.avro.Schema.Type.STRING), null, null),
-        new org.apache.avro.Schema.Field(INT_DIM_SV1,
-            org.apache.avro.Schema.create(org.apache.avro.Schema.Type.INT), null, null),
-        new org.apache.avro.Schema.Field(LONG_DIM_SV1,
-            org.apache.avro.Schema.create(org.apache.avro.Schema.Type.LONG), null, null),
-        new org.apache.avro.Schema.Field(DOUBLE_DIM_SV1,
-            org.apache.avro.Schema.create(org.apache.avro.Schema.Type.DOUBLE), null, null),
-        new org.apache.avro.Schema.Field(BOOLEAN_DIM_SV1,
-            org.apache.avro.Schema.create(org.apache.avro.Schema.Type.BOOLEAN), null, null)));
-
-    List<File> ret = new ArrayList<>();
-    for (int file = 0; file < 3; file++) {
-      // create avro file
-      File avroFile = new File(_tempDir, "data_" + file + ".avro");
-      try (DataFileWriter<GenericData.Record> fileWriter = new DataFileWriter<>(new GenericDatumWriter<>(avroSchema))) {
-        fileWriter.create(avroSchema, avroFile);
-
-        int numDocs = 1_000_000;
-        int randBound = numDocs / 2;
-        Random random = new Random(0);
-        IntStream randomInt = random.ints(0, 100_000);
-        for (int docId = 0; docId < numDocs; docId++) {
-          GenericData.Record record = new GenericData.Record(avroSchema);
-          record.put(STRING_DIM_SV1, "test query killing");
-          record.put(STRING_DIM_SV2, "test query killing");
-          record.put(INT_DIM_SV1, random.nextInt(randBound));
-          record.put(LONG_DIM_SV1, random.nextLong());
-          record.put(DOUBLE_DIM_SV1, random.nextDouble());
-          record.put(BOOLEAN_DIM_SV1, true);
-          fileWriter.append(record);
-        }
-        ret.add(avroFile);
-      }
-    }
-    return ret;
-  }
-}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedServerQueryKillingTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedServerQueryKillingTest.java
deleted file mode 100644
index 311d93361bc..00000000000
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedServerQueryKillingTest.java
+++ /dev/null
@@ -1,331 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.integration.tests;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.atomic.AtomicReference;
-import org.apache.avro.Schema.Field;
-import org.apache.avro.Schema.Type;
-import org.apache.avro.file.DataFileWriter;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericDatumWriter;
-import org.apache.pinot.core.accounting.PerQueryCPUMemAccountantFactory.PerQueryCPUMemResourceUsageAccountant;
-import org.apache.pinot.spi.accounting.QueryResourceTracker;
-import org.apache.pinot.spi.accounting.ThreadAccountantFactory;
-import org.apache.pinot.spi.config.instance.InstanceType;
-import org.apache.pinot.spi.config.table.TableConfig;
-import org.apache.pinot.spi.config.table.TableType;
-import org.apache.pinot.spi.data.FieldSpec;
-import org.apache.pinot.spi.data.Schema;
-import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.exception.QueryErrorCode;
-import org.apache.pinot.spi.utils.CommonConstants;
-import org.apache.pinot.spi.utils.CommonConstants.Accounting;
-import org.apache.pinot.spi.utils.CommonConstants.Broker;
-import org.apache.pinot.spi.utils.CommonConstants.Server;
-import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
-import org.apache.pinot.util.TestUtils;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.DataProvider;
-import org.testng.annotations.Test;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertTrue;
-
-
-/**
- * Integration test for heap size based server query killing, this works only for xmx4G.
- */
-public class OfflineClusterMemBasedServerQueryKillingTest extends BaseClusterIntegrationTestSet {
-  private static final String STRING_DIM_SV1 = "stringDimSV1";
-  private static final String STRING_DIM_SV2 = "stringDimSV2";
-  private static final String INT_DIM_SV1 = "intDimSV1";
-  private static final String LONG_DIM_SV1 = "longDimSV1";
-  private static final String DOUBLE_DIM_SV1 = "doubleDimSV1";
-  private static final String BOOLEAN_DIM_SV1 = "booleanDimSV1";
-  private static final int NUM_SEGMENTS = 3;
-  private static final int NUM_DOCS_PER_SEGMENT = 3_000_000;
-
-  private static final String OOM_QUERY =
-      "SELECT DISTINCT stringDimSV2 FROM mytable ORDER BY stringDimSV2 LIMIT 3000000";
-
-  private static final String OOM_QUERY_2 =
-      "SELECT DISTINCT_COUNT_HLL(intDimSV1, 14), stringDimSV2 FROM mytable GROUP BY 2 ORDER BY 1 LIMIT 3000000";
-
-  /// SafeTrim sort-aggregation case, pair-wise combine
-  private static final String OOM_QUERY_3 =
-      "SET sortAggregateSingleThreadedNumSegmentsThreshold=1; SET sortAggregateLimitThreshold=3000001; "
-          + "SELECT DISTINCT_COUNT_HLL(intDimSV1, 14), stringDimSV2 FROM mytable GROUP BY 2 ORDER BY 2 LIMIT 3000000";
-
-  /// SafeTrim sort-aggregation case, sequential combine
-  private static final String OOM_QUERY_4 =
-      "SET sortAggregateSingleThreadedNumSegmentsThreshold=10000; SET sortAggregateLimitThreshold=3000001; "
-          + "SELECT DISTINCT_COUNT_HLL(intDimSV1, 14), stringDimSV2 FROM mytable GROUP BY 2 ORDER BY 2 LIMIT 3000000";
-
-  private static final String AGGREGATE_QUERY = "SELECT DISTINCT_COUNT_HLL(intDimSV1, 14) FROM mytable";
-  private static final String SELECT_STAR_QUERY = "SELECT * FROM mytable LIMIT 5";
-  private static final String OOM_SELECT_STAR_QUERY = "SELECT * FROM mytable LIMIT 3000000";
-
-  /**
-   * Keeps track of metadata of queries that have been terminated due to OOM.
-   * This is used to verify that the query was killed and the method used to kill the query.
-   */
-  public static class TestAccountantFactory implements ThreadAccountantFactory {
-
-    @Override
-    public TestAccountant init(PinotConfiguration config, String instanceId, InstanceType instanceType) {
-      return new TestAccountant(config, instanceId, instanceType);
-    }
-
-    public static class TestAccountant extends PerQueryCPUMemResourceUsageAccountant {
-      private QueryResourceTracker _queryResourceTracker = null;
-
-      public TestAccountant(PinotConfiguration config, String instanceId, InstanceType instanceType) {
-        super(config, instanceId, instanceType);
-      }
-
-      @Override
-      public void logTerminatedQuery(QueryResourceTracker queryTracker, long totalHeapUsage) {
-        super.logTerminatedQuery(queryTracker, totalHeapUsage);
-        _queryResourceTracker = queryTracker;
-      }
-
-      public void reset() {
-        _queryResourceTracker = null;
-      }
-
-      public QueryResourceTracker getQueryResourceTracker() {
-        return _queryResourceTracker;
-      }
-    }
-  }
-
-  private TestAccountantFactory.TestAccountant _testAccountant;
-
-  @BeforeClass
-  public void setUp()
-      throws Exception {
-    TestUtils.ensureDirectoriesExistAndEmpty(_tempDir, _segmentDir, _tarDir);
-
-    // Start the Pinot cluster
-    startZk();
-    startController();
-    startBroker();
-    startServer();
-    _testAccountant =
-        (TestAccountantFactory.TestAccountant) _serverStarters.get(0).getServerInstance().getThreadAccountant();
-
-    // Create and upload the schema and table config
-    Schema schema = new Schema.SchemaBuilder().setSchemaName(DEFAULT_TABLE_NAME)
-        .addSingleValueDimension(STRING_DIM_SV1, FieldSpec.DataType.STRING)
-        .addSingleValueDimension(STRING_DIM_SV2, FieldSpec.DataType.STRING)
-        .addSingleValueDimension(INT_DIM_SV1, FieldSpec.DataType.INT)
-        .addSingleValueDimension(LONG_DIM_SV1, FieldSpec.DataType.LONG)
-        .addSingleValueDimension(DOUBLE_DIM_SV1, FieldSpec.DataType.DOUBLE)
-        .addSingleValueDimension(BOOLEAN_DIM_SV1, FieldSpec.DataType.BOOLEAN)
-        .build();
-    addSchema(schema);
-    TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(DEFAULT_TABLE_NAME).build();
-    addTableConfig(tableConfig);
-
-    List<File> avroFiles = createAvroFiles();
-    ClusterIntegrationTestUtils.buildSegmentsFromAvro(avroFiles, tableConfig, schema, 0, _segmentDir, _tarDir);
-    uploadSegments(DEFAULT_TABLE_NAME, _tarDir);
-
-    // Wait for all documents loaded
-    waitForAllDocsLoaded(600_000L);
-  }
-
-  @AfterMethod
-  public void resetTestAccountant() {
-    _testAccountant.reset();
-  }
-
-  protected void overrideBrokerConf(PinotConfiguration brokerConf) {
-    brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
-  }
-
-  protected void overrideServerConf(PinotConfiguration serverConf) {
-    serverConf.setProperty(Server.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
-    serverConf.setProperty(Server.CONFIG_OF_QUERY_EXECUTOR_NUM_GROUPS_LIMIT, Integer.MAX_VALUE);
-
-    String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME, TestAccountantFactory.class.getName());
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_OOM_PROTECTION_KILLING_QUERY, true);
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_ALARMING_LEVEL_HEAP_USAGE_RATIO, 0f);
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_CRITICAL_LEVEL_HEAP_USAGE_RATIO, 0.15f);
-  }
-
-  protected long getCountStarResult() {
-    return NUM_SEGMENTS * NUM_DOCS_PER_SEGMENT;
-  }
-
-  @Test
-  public void testMemoryAllocationStats()
-      throws Exception {
-    JsonNode queryResponse = postQuery(SELECT_STAR_QUERY);
-    long offlineThreadMemAllocatedBytes = queryResponse.get("offlineThreadMemAllocatedBytes").asLong();
-    long offlineResponseSerMemAllocatedBytes = queryResponse.get("offlineResponseSerMemAllocatedBytes").asLong();
-    long offlineTotalMemAllocatedBytes = queryResponse.get("offlineTotalMemAllocatedBytes").asLong();
-
-    assertTrue(offlineThreadMemAllocatedBytes > 0);
-    assertTrue(offlineResponseSerMemAllocatedBytes > 0);
-    assertEquals(offlineThreadMemAllocatedBytes + offlineResponseSerMemAllocatedBytes, offlineTotalMemAllocatedBytes);
-  }
-
-  @DataProvider
-  public String[] oomQueries() {
-    return new String[]{OOM_QUERY, OOM_QUERY_2, OOM_QUERY_3, OOM_QUERY_4, OOM_SELECT_STAR_QUERY};
-  }
-
-  @Test(dataProvider = "oomQueries")
-  public void testOomSse(String query)
-      throws Exception {
-    testOom(query);
-    setUseMultiStageQueryEngine(true);
-    testOom(query);
-  }
-
-  private void testOom(String query)
-      throws Exception {
-    JsonNode queryResponse = postQuery(query);
-    JsonNode exceptionsNode = queryResponse.get("exceptions");
-    assertNotNull(exceptionsNode, "Missing exceptions for query: " + query);
-    assertEquals(exceptionsNode.size(), 1, "Expected 1 exception for query: " + query + ", but got: " + exceptionsNode);
-    JsonNode exceptionNode = exceptionsNode.get(0);
-    JsonNode errorCodeNode = exceptionNode.get("errorCode");
-    assertNotNull(errorCodeNode, "Missing errorCode from exception: " + exceptionNode);
-    int errorCode = errorCodeNode.asInt();
-    assertEquals(errorCode, QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED.getId(),
-        "Unexpected error code: " + errorCode + " from exception: " + exceptionNode);
-    JsonNode messageNode = exceptionNode.get("message");
-    assertNotNull(messageNode, "Missing message from exception: " + exceptionNode);
-    String message = messageNode.asText();
-    assertTrue(message.contains("OOM killed on SERVER"),
-        "Unexpected exception message: " + message + " from exception: " + exceptionNode);
-    assertEquals(queryResponse.get("requestId").asLong(),
-        _testAccountant.getQueryResourceTracker().getExecutionContext().getRequestId());
-  }
-
-  @Test(dataProvider = "useBothQueryEngines")
-  public void testOomMultipleQueries(boolean useMultiStageQueryEngine)
-      throws Exception {
-    setUseMultiStageQueryEngine(useMultiStageQueryEngine);
-    AtomicReference<JsonNode> queryResponse1 = new AtomicReference<>();
-    AtomicReference<JsonNode> queryResponse2 = new AtomicReference<>();
-    AtomicReference<JsonNode> queryResponse3 = new AtomicReference<>();
-    CountDownLatch countDownLatch = new CountDownLatch(3);
-    ExecutorService executor = Executors.newFixedThreadPool(3);
-    executor.submit(() -> {
-      queryResponse1.set(postQuery(OOM_QUERY));
-      countDownLatch.countDown();
-      return null;
-    });
-    executor.submit(() -> {
-      queryResponse2.set(postQuery(AGGREGATE_QUERY));
-      countDownLatch.countDown();
-      return null;
-    });
-    executor.submit(() -> {
-      queryResponse3.set(postQuery(SELECT_STAR_QUERY));
-      countDownLatch.countDown();
-      return null;
-    });
-    executor.shutdown();
-    countDownLatch.await();
-
-    JsonNode queryResponse = queryResponse1.get();
-    JsonNode exceptionsNode = queryResponse.get("exceptions");
-    assertNotNull(exceptionsNode, "Missing exceptions for query: " + OOM_QUERY);
-    assertEquals(exceptionsNode.size(), 1,
-        "Expected 1 exception for query: " + OOM_QUERY + ", but got: " + exceptionsNode);
-    JsonNode exceptionNode = exceptionsNode.get(0);
-    JsonNode errorCodeNode = exceptionNode.get("errorCode");
-    assertNotNull(errorCodeNode, "Missing errorCode from exception: " + exceptionNode);
-    int errorCode = errorCodeNode.asInt();
-    assertEquals(errorCode, QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED.getId(),
-        "Unexpected error code: " + errorCode + " from exception: " + exceptionNode);
-    JsonNode messageNode = exceptionNode.get("message");
-    assertNotNull(messageNode, "Missing message from exception: " + exceptionNode);
-    String message = messageNode.asText();
-    assertTrue(message.contains("OOM killed on SERVER"),
-        "Unexpected exception message: " + message + " from exception: " + exceptionNode);
-    assertEquals(queryResponse.get("requestId").asLong(),
-        _testAccountant.getQueryResourceTracker().getExecutionContext().getRequestId());
-
-    queryResponse = queryResponse2.get();
-    exceptionsNode = queryResponse.get("exceptions");
-    assertNotNull(exceptionsNode, "Missing exceptions for query: " + AGGREGATE_QUERY);
-    assertTrue(exceptionsNode.isEmpty(),
-        "Expected no exceptions for query: " + AGGREGATE_QUERY + ", but got: " + exceptionsNode);
-
-    queryResponse = queryResponse3.get();
-    exceptionsNode = queryResponse.get("exceptions");
-    assertNotNull(exceptionsNode, "Missing exceptions for query: " + SELECT_STAR_QUERY);
-    assertTrue(exceptionsNode.isEmpty(),
-        "Expected no exceptions for query: " + SELECT_STAR_QUERY + ", but got: " + exceptionsNode);
-  }
-
-  private List<File> createAvroFiles()
-      throws IOException {
-    // Create Avro schema
-    org.apache.avro.Schema avroSchema = org.apache.avro.Schema.createRecord("myRecord", null, null, false);
-    avroSchema.setFields(List.of(new Field(STRING_DIM_SV1, org.apache.avro.Schema.create(Type.STRING), null, null),
-        new Field(STRING_DIM_SV2, org.apache.avro.Schema.create(Type.STRING), null, null),
-        new Field(INT_DIM_SV1, org.apache.avro.Schema.create(Type.INT), null, null),
-        new Field(LONG_DIM_SV1, org.apache.avro.Schema.create(Type.LONG), null, null),
-        new Field(DOUBLE_DIM_SV1, org.apache.avro.Schema.create(Type.DOUBLE), null, null),
-        new Field(BOOLEAN_DIM_SV1, org.apache.avro.Schema.create(Type.BOOLEAN), null, null)));
-
-    // Create Avro files
-    List<File> ret = new ArrayList<>();
-    for (int segmentId = 0; segmentId < NUM_SEGMENTS; segmentId++) {
-      File avroFile = new File(_tempDir, "data_" + segmentId + ".avro");
-      try (DataFileWriter<GenericData.Record> fileWriter = new DataFileWriter<>(new GenericDatumWriter<>(avroSchema))) {
-        fileWriter.create(avroSchema, avroFile);
-
-        int randBound = NUM_DOCS_PER_SEGMENT / 2;
-        Random random = new Random(0);
-        for (int docId = 0; docId < NUM_DOCS_PER_SEGMENT; docId++) {
-          GenericData.Record record = new GenericData.Record(avroSchema);
-          record.put(STRING_DIM_SV1, "test query killing");
-          record.put(STRING_DIM_SV2, "test query killing" + docId);
-          record.put(INT_DIM_SV1, random.nextInt(randBound));
-          record.put(LONG_DIM_SV1, random.nextLong());
-          record.put(DOUBLE_DIM_SV1, random.nextDouble());
-          record.put(BOOLEAN_DIM_SV1, true);
-          fileWriter.append(record);
-        }
-        ret.add(avroFile);
-      }
-    }
-    return ret;
-  }
-}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedServerResourceUsageAccountantTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedServerResourceUsageAccountantTest.java
deleted file mode 100644
index 631bf4bc029..00000000000
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedServerResourceUsageAccountantTest.java
+++ /dev/null
@@ -1,302 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.integration.tests;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.collect.ImmutableList;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.IntStream;
-import org.apache.avro.file.DataFileWriter;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericDatumWriter;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.log4j.Level;
-import org.apache.log4j.LogManager;
-import org.apache.pinot.core.accounting.QueryResourceAggregator;
-import org.apache.pinot.core.accounting.ResourceUsageAccountantFactory;
-import org.apache.pinot.spi.accounting.ThreadResourceUsageProvider;
-import org.apache.pinot.spi.config.table.TableConfig;
-import org.apache.pinot.spi.config.table.TableType;
-import org.apache.pinot.spi.data.FieldSpec;
-import org.apache.pinot.spi.data.Schema;
-import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.exception.QueryErrorCode;
-import org.apache.pinot.spi.trace.Tracing;
-import org.apache.pinot.spi.utils.CommonConstants;
-import org.apache.pinot.spi.utils.CommonConstants.Accounting;
-import org.apache.pinot.spi.utils.CommonConstants.Broker;
-import org.apache.pinot.spi.utils.CommonConstants.Server;
-import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
-import org.apache.pinot.util.TestUtils;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertTrue;
-
-
-/**
- * Integration test for heap size based server query killing, this works only for xmx4G.
- * <p>
- * Query killing isn't currently supported in the v2 multi-stage query engine so these tests only run on the v1 engine.
- */
-public class OfflineClusterMemBasedServerResourceUsageAccountantTest extends BaseClusterIntegrationTestSet {
-  public static final String STRING_DIM_SV1 = "stringDimSV1";
-  public static final String STRING_DIM_SV2 = "stringDimSV2";
-  public static final String INT_DIM_SV1 = "intDimSV1";
-  public static final String LONG_DIM_SV1 = "longDimSV1";
-  public static final String DOUBLE_DIM_SV1 = "doubleDimSV1";
-  public static final String BOOLEAN_DIM_SV1 = "booleanDimSV1";
-  private static final int NUM_DOCS = 3_000_000;
-
-  private static final String OOM_QUERY =
-      "SELECT PERCENTILETDigest(doubleDimSV1, 50) AS digest, intDimSV1 FROM mytable GROUP BY intDimSV1"
-          + " ORDER BY digest LIMIT 30000";
-
-  private static final String OOM_QUERY_2 =
-      "SELECT stringDimSV2 FROM mytable GROUP BY stringDimSV2"
-          + " ORDER BY stringDimSV2 LIMIT 3000000";
-
-  private static final String DIGEST_QUERY_1 =
-      "SELECT PERCENTILETDigest(doubleDimSV1, 50) AS digest FROM mytable";
-  private static final String COUNT_STAR_QUERY =
-      "SELECT * FROM mytable LIMIT 5";
-  private static final String OOM_QUERY_SELECTION_ONLY = "SELECT * FROM mytable LIMIT 9000000";
-
-  private static final ExecutorService EXECUTOR_SERVICE = Executors.newFixedThreadPool(20, r -> {
-    Thread thread = new Thread(r);
-    thread.setDaemon(false);
-    return thread;
-  });
-
-  @BeforeClass
-  public void setUp()
-      throws Exception {
-    TestUtils.ensureDirectoriesExistAndEmpty(_tempDir, _segmentDir, _tarDir);
-
-    // Start the Pinot cluster
-    startZk();
-    startController();
-    startBroker();
-    startServer();
-
-    // Create and upload the schema and table config
-    Schema schema = new Schema.SchemaBuilder().setSchemaName(DEFAULT_SCHEMA_NAME)
-        .addSingleValueDimension(STRING_DIM_SV1, FieldSpec.DataType.STRING)
-        .addSingleValueDimension(STRING_DIM_SV2, FieldSpec.DataType.STRING)
-        .addSingleValueDimension(INT_DIM_SV1, FieldSpec.DataType.INT)
-        .addSingleValueDimension(LONG_DIM_SV1, FieldSpec.DataType.LONG)
-        .addSingleValueDimension(DOUBLE_DIM_SV1, FieldSpec.DataType.DOUBLE)
-        .addSingleValueDimension(BOOLEAN_DIM_SV1, FieldSpec.DataType.BOOLEAN)
-        .build();
-    addSchema(schema);
-    TableConfig tableConfig = createOfflineTableConfig();
-    addTableConfig(tableConfig);
-
-    List<File> avroFiles = createAvroFile();
-    ClusterIntegrationTestUtils.buildSegmentsFromAvro(avroFiles, tableConfig, schema, 0, _segmentDir, _tarDir);
-    uploadSegments(DEFAULT_TABLE_NAME, _tarDir);
-
-    //Wait for all documents loaded
-    waitForAllDocsLoaded(10_000L);
-
-    // Setup logging and resource accounting
-    LogManager.getLogger(OfflineClusterMemBasedServerQueryKillingTest.class).setLevel(Level.INFO);
-    LogManager.getLogger(ResourceUsageAccountantFactory.ResourceUsageAccountant.class)
-        .setLevel(Level.INFO);
-    LogManager.getLogger(ThreadResourceUsageProvider.class).setLevel(Level.INFO);
-    LogManager.getLogger(Tracing.class).setLevel(Level.INFO);
-    LogManager.getLogger(QueryResourceAggregator.class).setLevel(Level.INFO);
-  }
-
-  protected void overrideBrokerConf(PinotConfiguration brokerConf) {
-    brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
-  }
-
-  protected void overrideServerConf(PinotConfiguration serverConf) {
-    serverConf.setProperty(Server.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
-
-    String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME, ResourceUsageAccountantFactory.class.getName());
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_OOM_PROTECTION_KILLING_QUERY, true);
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_ALARMING_LEVEL_HEAP_USAGE_RATIO, 0f);
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_CRITICAL_LEVEL_HEAP_USAGE_RATIO, 0.15f);
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_WORKLOAD_ENABLE_COST_COLLECTION, true);
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_WORKLOAD_ENABLE_COST_ENFORCEMENT, true);
-  }
-
-  protected long getCountStarResult() {
-    return NUM_DOCS * 3;
-  }
-
-  protected String getTimeColumnName() {
-    return null;
-  }
-
-  protected TableConfig createOfflineTableConfig() {
-    return new TableConfigBuilder(TableType.OFFLINE).setTableName(getTableName())
-        .setTimeColumnName(getTimeColumnName()).setFieldConfigList(getFieldConfigs()).setNumReplicas(getNumReplicas())
-        .setSegmentVersion(getSegmentVersion()).setLoadMode(getLoadMode()).setTaskConfig(getTaskConfig())
-        .setBrokerTenant(getBrokerTenant()).setServerTenant(getServerTenant()).setIngestionConfig(getIngestionConfig())
-        .setQueryConfig(getQueryConfig()).setNullHandlingEnabled(getNullHandlingEnabled())
-        .setSegmentPartitionConfig(getSegmentPartitionConfig())
-        .build();
-  }
-
-  @Test(dataProvider = "useBothQueryEngines")
-  public void testDigestOOM(boolean useMultiStageQueryEngine)
-      throws Exception {
-    setUseMultiStageQueryEngine(useMultiStageQueryEngine);
-    notSupportedInV2();
-    JsonNode queryResponse = postQuery(OOM_QUERY);
-    String exceptions = queryResponse.get("exceptions").toString();
-    assertTrue(exceptions.contains("\"errorCode\":" + QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED.getId()),
-        exceptions);
-    assertTrue(exceptions.contains("OOM killed on SERVER"), exceptions);
-  }
-
-  @Test(dataProvider = "useBothQueryEngines")
-  public void testSelectionOnlyOOM(boolean useMultiStageQueryEngine)
-      throws Exception {
-    setUseMultiStageQueryEngine(useMultiStageQueryEngine);
-    notSupportedInV2();
-    JsonNode queryResponse = postQuery(OOM_QUERY_SELECTION_ONLY);
-
-    String exceptions = queryResponse.get("exceptions").toString();
-    assertTrue(exceptions.contains("\"errorCode\":" + QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED.getId()),
-        exceptions);
-    assertTrue(exceptions.contains("OOM killed on SERVER"), exceptions);
-  }
-
-  @Test(dataProvider = "useBothQueryEngines")
-  public void testDigestOOM2(boolean useMultiStageQueryEngine)
-      throws Exception {
-    setUseMultiStageQueryEngine(useMultiStageQueryEngine);
-    notSupportedInV2();
-    JsonNode queryResponse = postQuery(OOM_QUERY_2);
-    String exceptions = queryResponse.get("exceptions").toString();
-    assertTrue(exceptions.contains("\"errorCode\":" + QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED.getId()),
-        exceptions);
-    assertTrue(exceptions.contains("OOM killed on SERVER"), exceptions);
-  }
-
-  @Test(dataProvider = "useBothQueryEngines")
-  public void testDigestOOMMultipleQueries(boolean useMultiStageQueryEngine)
-      throws Exception {
-    setUseMultiStageQueryEngine(useMultiStageQueryEngine);
-    notSupportedInV2();
-    AtomicReference<JsonNode> queryResponse1 = new AtomicReference<>();
-    AtomicReference<JsonNode> queryResponse2 = new AtomicReference<>();
-    AtomicReference<JsonNode> queryResponse3 = new AtomicReference<>();
-
-    CountDownLatch countDownLatch = new CountDownLatch(3);
-
-    EXECUTOR_SERVICE.submit(
-        () -> {
-          try {
-            queryResponse1.set(postQuery(OOM_QUERY));
-            countDownLatch.countDown();
-          } catch (Exception ignored) {
-          }
-        }
-    );
-    EXECUTOR_SERVICE.submit(
-        () -> {
-          try {
-            queryResponse2.set(postQuery(DIGEST_QUERY_1));
-            countDownLatch.countDown();
-          } catch (Exception ignored) {
-          }
-        }
-    );
-    EXECUTOR_SERVICE.submit(
-        () -> {
-          try {
-            queryResponse3.set(postQuery(COUNT_STAR_QUERY));
-            countDownLatch.countDown();
-          } catch (Exception ignored) {
-          }
-        }
-    );
-    countDownLatch.await();
-    String exceptions = queryResponse1.get().get("exceptions").toString();
-    assertTrue(exceptions.contains("\"errorCode\":" + QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED.getId()),
-        exceptions);
-    assertTrue(exceptions.contains("OOM killed on SERVER"), exceptions);
-    assertFalse(StringUtils.isEmpty(queryResponse2.get().get("exceptions").toString()));
-    assertFalse(StringUtils.isEmpty(queryResponse3.get().get("exceptions").toString()));
-  }
-
-  // TODO: Add integation test after workload configs PR is merged.
-
-  private List<File> createAvroFile()
-      throws IOException {
-
-    // create avro schema
-    org.apache.avro.Schema avroSchema = org.apache.avro.Schema.createRecord("myRecord", null, null, false);
-    avroSchema.setFields(ImmutableList.of(
-        new org.apache.avro.Schema.Field(STRING_DIM_SV1,
-            org.apache.avro.Schema.create(org.apache.avro.Schema.Type.STRING), null, null),
-        new org.apache.avro.Schema.Field(STRING_DIM_SV2,
-            org.apache.avro.Schema.create(org.apache.avro.Schema.Type.STRING), null, null),
-        new org.apache.avro.Schema.Field(INT_DIM_SV1,
-            org.apache.avro.Schema.create(org.apache.avro.Schema.Type.INT), null, null),
-        new org.apache.avro.Schema.Field(LONG_DIM_SV1,
-            org.apache.avro.Schema.create(org.apache.avro.Schema.Type.LONG), null, null),
-        new org.apache.avro.Schema.Field(DOUBLE_DIM_SV1,
-            org.apache.avro.Schema.create(org.apache.avro.Schema.Type.DOUBLE), null, null),
-        new org.apache.avro.Schema.Field(BOOLEAN_DIM_SV1,
-            org.apache.avro.Schema.create(org.apache.avro.Schema.Type.BOOLEAN), null, null)));
-
-    List<File> ret = new ArrayList<>();
-    for (int file = 0; file < 3; file++) {
-      // create avro file
-      File avroFile = new File(_tempDir, "data_" + file + ".avro");
-      try (DataFileWriter<GenericData.Record> fileWriter = new DataFileWriter<>(new GenericDatumWriter<>(avroSchema))) {
-        fileWriter.create(avroSchema, avroFile);
-
-        int numDocs = NUM_DOCS;
-        int randBound = numDocs / 2;
-        Random random = new Random(0);
-        IntStream randomInt = random.ints(0, 100_000);
-        for (int docId = 0; docId < numDocs; docId++) {
-          GenericData.Record record = new GenericData.Record(avroSchema);
-          record.put(STRING_DIM_SV1, "test query killing");
-          record.put(STRING_DIM_SV2, "test query killing" + docId);
-          record.put(INT_DIM_SV1, random.nextInt(randBound));
-          record.put(LONG_DIM_SV1, random.nextLong());
-          record.put(DOUBLE_DIM_SV1, random.nextDouble());
-          record.put(BOOLEAN_DIM_SV1, true);
-          fileWriter.append(record);
-        }
-        ret.add(avroFile);
-      }
-    }
-    return ret;
-  }
-}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterResourceUsageAccountantTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterResourceUsageAccountantTest.java
deleted file mode 100644
index 66af8cd3154..00000000000
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterResourceUsageAccountantTest.java
+++ /dev/null
@@ -1,272 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.integration.tests;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.collect.ImmutableList;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.IntStream;
-import org.apache.avro.file.DataFileWriter;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericDatumWriter;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.log4j.Level;
-import org.apache.log4j.LogManager;
-import org.apache.pinot.core.accounting.QueryResourceAggregator;
-import org.apache.pinot.core.accounting.ResourceUsageAccountantFactory;
-import org.apache.pinot.spi.accounting.ThreadResourceUsageProvider;
-import org.apache.pinot.spi.config.table.TableConfig;
-import org.apache.pinot.spi.config.table.TableType;
-import org.apache.pinot.spi.data.FieldSpec;
-import org.apache.pinot.spi.data.Schema;
-import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.exception.QueryErrorCode;
-import org.apache.pinot.spi.trace.Tracing;
-import org.apache.pinot.spi.utils.CommonConstants;
-import org.apache.pinot.spi.utils.CommonConstants.Accounting;
-import org.apache.pinot.spi.utils.CommonConstants.Broker;
-import org.apache.pinot.spi.utils.CommonConstants.Server;
-import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
-import org.apache.pinot.util.TestUtils;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import static org.testng.Assert.assertTrue;
-
-
-public class OfflineClusterResourceUsageAccountantTest extends BaseClusterIntegrationTestSet {
-  public static final String STRING_DIM_SV1 = "stringDimSV1";
-  public static final String STRING_DIM_SV2 = "stringDimSV2";
-  public static final String INT_DIM_SV1 = "intDimSV1";
-  public static final String LONG_DIM_SV1 = "longDimSV1";
-  public static final String DOUBLE_DIM_SV1 = "doubleDimSV1";
-  public static final String BOOLEAN_DIM_SV1 = "booleanDimSV1";
-  private static final String OOM_QUERY =
-      "SELECT PERCENTILETDigest(doubleDimSV1, 50) AS digest, intDimSV1 FROM mytable GROUP BY intDimSV1"
-          + " ORDER BY digest LIMIT 15000";
-
-  private static final String DIGEST_QUERY_1 =
-      "SELECT PERCENTILETDigest(doubleDimSV1, 50) AS digest FROM mytable";
-  private static final String COUNT_STAR_QUERY =
-      "SELECT * FROM mytable LIMIT 5";
-
-  private static final ExecutorService EXECUTOR_SERVICE = Executors.newFixedThreadPool(20, r -> {
-    Thread thread = new Thread(r);
-    thread.setDaemon(false);
-    return thread;
-  });
-
-  @BeforeClass
-  public void setUp()
-      throws Exception {
-    TestUtils.ensureDirectoriesExistAndEmpty(_tempDir, _segmentDir, _tarDir);
-
-    // Start the Pinot cluster
-    startZk();
-    startController();
-    startBroker();
-    startServer();
-
-    // Create and upload the schema and table config
-    Schema schema = new Schema.SchemaBuilder().setSchemaName(DEFAULT_SCHEMA_NAME)
-        .addSingleValueDimension(STRING_DIM_SV1, FieldSpec.DataType.STRING)
-        .addSingleValueDimension(STRING_DIM_SV2, FieldSpec.DataType.STRING)
-        .addSingleValueDimension(INT_DIM_SV1, FieldSpec.DataType.INT)
-        .addSingleValueDimension(LONG_DIM_SV1, FieldSpec.DataType.LONG)
-        .addSingleValueDimension(DOUBLE_DIM_SV1, FieldSpec.DataType.DOUBLE)
-        .addSingleValueDimension(BOOLEAN_DIM_SV1, FieldSpec.DataType.BOOLEAN)
-        .build();
-    addSchema(schema);
-    TableConfig tableConfig = createOfflineTableConfig();
-    addTableConfig(tableConfig);
-
-    List<File> avroFiles = createAvroFile();
-    ClusterIntegrationTestUtils.buildSegmentsFromAvro(avroFiles, tableConfig, schema, 0, _segmentDir, _tarDir);
-    uploadSegments(DEFAULT_TABLE_NAME, _tarDir);
-
-    //Wait for all documents loaded
-    waitForAllDocsLoaded(10_000L);
-
-    // Setup logging and resource accounting
-    LogManager.getLogger(OfflineClusterCpuBasedServerQueryKillingTest.class).setLevel(Level.INFO);
-    LogManager.getLogger(ResourceUsageAccountantFactory.ResourceUsageAccountant.class)
-        .setLevel(Level.INFO);
-    LogManager.getLogger(ThreadResourceUsageProvider.class).setLevel(Level.INFO);
-    LogManager.getLogger(Tracing.class).setLevel(Level.INFO);
-    LogManager.getLogger(QueryResourceAggregator.class).setLevel(Level.INFO);
-  }
-
-  protected void overrideBrokerConf(PinotConfiguration brokerConf) {
-    brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_CPU_TIME_MEASUREMENT, true);
-    brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
-
-    String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
-    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME, ResourceUsageAccountantFactory.class.getName());
-    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_ALARMING_LEVEL_HEAP_USAGE_RATIO, 0.2f);
-    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_CRITICAL_LEVEL_HEAP_USAGE_RATIO, 1.1f);
-    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_PANIC_LEVEL_HEAP_USAGE_RATIO, 1.1f);
-    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
-    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_CPU_SAMPLING, true);
-    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_OOM_PROTECTION_KILLING_QUERY, true);
-    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_CPU_TIME_BASED_KILLING_ENABLED, true);
-    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_WORKLOAD_ENABLE_COST_COLLECTION, true);
-    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_WORKLOAD_ENABLE_COST_ENFORCEMENT, true);
-  }
-
-  protected void overrideServerConf(PinotConfiguration serverConf) {
-    serverConf.setProperty(Server.CONFIG_OF_ENABLE_THREAD_CPU_TIME_MEASUREMENT, true);
-    serverConf.setProperty(Server.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
-
-    String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_ALARMING_LEVEL_HEAP_USAGE_RATIO, 0.2f);
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_CRITICAL_LEVEL_HEAP_USAGE_RATIO, 1.1f);
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME, ResourceUsageAccountantFactory.class.getName());
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_PANIC_LEVEL_HEAP_USAGE_RATIO, 1.1f);
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_CPU_SAMPLING, true);
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_OOM_PROTECTION_KILLING_QUERY, true);
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_CPU_TIME_BASED_KILLING_ENABLED, true);
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_CPU_TIME_BASED_KILLING_THRESHOLD_MS, 1_000);
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_WORKLOAD_ENABLE_COST_COLLECTION, true);
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_WORKLOAD_ENABLE_COST_ENFORCEMENT, true);
-  }
-
-  protected long getCountStarResult() {
-    return 3_000_000;
-  }
-
-  protected String getTimeColumnName() {
-    return null;
-  }
-
-  protected TableConfig createOfflineTableConfig() {
-    return new TableConfigBuilder(TableType.OFFLINE).setTableName(getTableName())
-        .setTimeColumnName(getTimeColumnName()).setFieldConfigList(getFieldConfigs()).setNumReplicas(getNumReplicas())
-        .setSegmentVersion(getSegmentVersion()).setLoadMode(getLoadMode()).setTaskConfig(getTaskConfig())
-        .setBrokerTenant(getBrokerTenant()).setServerTenant(getServerTenant()).setIngestionConfig(getIngestionConfig())
-        .setQueryConfig(getQueryConfig()).setNullHandlingEnabled(getNullHandlingEnabled())
-        .setSegmentPartitionConfig(getSegmentPartitionConfig())
-        .build();
-  }
-
-  @Test(dataProvider = "useBothQueryEngines")
-  public void testDigestTimeoutMultipleQueries(boolean useMultiStageQueryEngine)
-      throws Exception {
-    setUseMultiStageQueryEngine(useMultiStageQueryEngine);
-    notSupportedInV2();
-    AtomicReference<JsonNode> queryResponse1 = new AtomicReference<>();
-    AtomicReference<JsonNode> queryResponse2 = new AtomicReference<>();
-    AtomicReference<JsonNode> queryResponse3 = new AtomicReference<>();
-
-    CountDownLatch countDownLatch = new CountDownLatch(3);
-
-    EXECUTOR_SERVICE.submit(
-        () -> {
-          try {
-            queryResponse1.set(postQuery(OOM_QUERY));
-            countDownLatch.countDown();
-          } catch (Exception ignored) {
-          }
-        }
-    );
-    EXECUTOR_SERVICE.submit(
-        () -> {
-          try {
-            queryResponse2.set(postQuery(DIGEST_QUERY_1));
-            countDownLatch.countDown();
-          } catch (Exception ignored) {
-          }
-        }
-    );
-    EXECUTOR_SERVICE.submit(
-        () -> {
-          try {
-            queryResponse3.set(postQuery(COUNT_STAR_QUERY));
-            countDownLatch.countDown();
-          } catch (Exception ignored) {
-          }
-        }
-    );
-    Assert.assertTrue(countDownLatch.await(1, TimeUnit.MINUTES));
-    String exceptions = queryResponse1.get().get("exceptions").toString();
-    assertTrue(exceptions.contains("\"errorCode\":" + QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED.getId()),
-        exceptions);
-    Assert.assertTrue(exceptions.contains("CPU time based killed on SERVER"), exceptions);
-    Assert.assertFalse(StringUtils.isEmpty(queryResponse2.get().get("exceptions").toString()));
-    Assert.assertFalse(StringUtils.isEmpty(queryResponse3.get().get("exceptions").toString()));
-  }
-
-  // TODO: Add integation test after workload configs PR is merged.
-
-  private List<File> createAvroFile()
-      throws IOException {
-
-    // create avro schema
-    org.apache.avro.Schema avroSchema = org.apache.avro.Schema.createRecord("myRecord", null, null, false);
-    avroSchema.setFields(ImmutableList.of(
-        new org.apache.avro.Schema.Field(STRING_DIM_SV1,
-            org.apache.avro.Schema.create(org.apache.avro.Schema.Type.STRING), null, null),
-        new org.apache.avro.Schema.Field(STRING_DIM_SV2,
-            org.apache.avro.Schema.create(org.apache.avro.Schema.Type.STRING), null, null),
-        new org.apache.avro.Schema.Field(INT_DIM_SV1,
-            org.apache.avro.Schema.create(org.apache.avro.Schema.Type.INT), null, null),
-        new org.apache.avro.Schema.Field(LONG_DIM_SV1,
-            org.apache.avro.Schema.create(org.apache.avro.Schema.Type.LONG), null, null),
-        new org.apache.avro.Schema.Field(DOUBLE_DIM_SV1,
-            org.apache.avro.Schema.create(org.apache.avro.Schema.Type.DOUBLE), null, null),
-        new org.apache.avro.Schema.Field(BOOLEAN_DIM_SV1,
-            org.apache.avro.Schema.create(org.apache.avro.Schema.Type.BOOLEAN), null, null)));
-
-    List<File> ret = new ArrayList<>();
-    for (int file = 0; file < 3; file++) {
-      // create avro file
-      File avroFile = new File(_tempDir, "data_" + file + ".avro");
-      try (DataFileWriter<GenericData.Record> fileWriter = new DataFileWriter<>(new GenericDatumWriter<>(avroSchema))) {
-        fileWriter.create(avroSchema, avroFile);
-
-        int numDocs = 1_000_000;
-        int randBound = numDocs / 2;
-        Random random = new Random(0);
-        IntStream randomInt = random.ints(0, 100_000);
-        for (int docId = 0; docId < numDocs; docId++) {
-          GenericData.Record record = new GenericData.Record(avroSchema);
-          record.put(STRING_DIM_SV1, "test query killing");
-          record.put(STRING_DIM_SV2, "test query killing");
-          record.put(INT_DIM_SV1, random.nextInt(randBound));
-          record.put(LONG_DIM_SV1, random.nextLong());
-          record.put(DOUBLE_DIM_SV1, random.nextDouble());
-          record.put(BOOLEAN_DIM_SV1, true);
-          fileWriter.append(record);
-        }
-        ret.add(avroFile);
-      }
-    }
-    return ret;
-  }
-}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineGRPCServerIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineGRPCServerIntegrationTest.java
index 2e7ade25b91..8334d10a007 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineGRPCServerIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineGRPCServerIntegrationTest.java
@@ -37,6 +37,7 @@ import org.apache.pinot.common.response.broker.BrokerResponseNative;
 import org.apache.pinot.common.utils.DataSchema;
 import org.apache.pinot.common.utils.grpc.ServerGrpcQueryClient;
 import org.apache.pinot.common.utils.grpc.ServerGrpcRequestBuilder;
+import org.apache.pinot.core.accounting.ResourceUsageAccountantFactory;
 import org.apache.pinot.core.query.reduce.DataTableReducer;
 import org.apache.pinot.core.query.reduce.DataTableReducerContext;
 import org.apache.pinot.core.query.reduce.ResultReducerFactory;
@@ -45,8 +46,11 @@ import org.apache.pinot.core.query.request.context.utils.QueryContextConverterUt
 import org.apache.pinot.core.transport.ServerRoutingInstance;
 import org.apache.pinot.spi.config.table.TableConfig;
 import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.env.PinotConfiguration;
 import org.apache.pinot.spi.query.QueryThreadContext;
 import org.apache.pinot.spi.utils.CommonConstants;
+import org.apache.pinot.spi.utils.CommonConstants.Accounting;
+import org.apache.pinot.spi.utils.CommonConstants.Broker;
 import org.apache.pinot.sql.parsers.CalciteSqlCompiler;
 import org.apache.pinot.util.TestUtils;
 import org.testng.annotations.AfterClass;
@@ -98,6 +102,32 @@ public class OfflineGRPCServerIntegrationTest extends BaseClusterIntegrationTest
     waitForAllDocsLoaded(600_000L);
   }
 
+  @Override
+  protected void overrideBrokerConf(PinotConfiguration brokerConf) {
+    super.overrideBrokerConf(brokerConf);
+
+    // Enable thread CPU/memory tracking but not killing queries
+    brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_CPU_TIME_MEASUREMENT, true);
+    brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
+    String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
+    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME, ResourceUsageAccountantFactory.class.getName());
+    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_CPU_SAMPLING, true);
+    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
+  }
+
+  @Override
+  protected void overrideServerConf(PinotConfiguration serverConf) {
+    super.overrideServerConf(serverConf);
+
+    // Enable thread CPU/memory tracking but not killing queries
+    serverConf.setProperty(CommonConstants.Server.CONFIG_OF_ENABLE_THREAD_CPU_TIME_MEASUREMENT, true);
+    serverConf.setProperty(CommonConstants.Server.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
+    String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
+    serverConf.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME, ResourceUsageAccountantFactory.class.getName());
+    serverConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_CPU_SAMPLING, true);
+    serverConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
+  }
+
   public ServerGrpcQueryClient getGrpcQueryClient() {
     return new ServerGrpcQueryClient("localhost", getServerGrpcPort());
   }
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineGRPCServerMultiStageIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineGRPCServerMultiStageIntegrationTest.java
index 97863881568..acbbb9b39cd 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineGRPCServerMultiStageIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineGRPCServerMultiStageIntegrationTest.java
@@ -18,12 +18,13 @@
  */
 package org.apache.pinot.integration.tests;
 
-import org.testng.annotations.BeforeTest;
+import org.testng.annotations.BeforeMethod;
 
 
 public class OfflineGRPCServerMultiStageIntegrationTest extends OfflineGRPCServerIntegrationTest {
-  @BeforeTest
-  void enableMultiStage() {
+
+  @BeforeMethod
+  public void enableMultiStage() {
     setUseMultiStageQueryEngine(true);
   }
 }
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineGRPCServerOOMAccountingIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineGRPCServerOOMAccountingIntegrationTest.java
deleted file mode 100644
index 660b745702d..00000000000
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineGRPCServerOOMAccountingIntegrationTest.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.integration.tests;
-
-import org.apache.pinot.core.accounting.PerQueryCPUMemAccountantFactory;
-import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.utils.CommonConstants;
-import org.apache.pinot.spi.utils.CommonConstants.Accounting;
-import org.apache.pinot.spi.utils.CommonConstants.Broker;
-import org.apache.pinot.spi.utils.CommonConstants.Server;
-
-
-public class OfflineGRPCServerOOMAccountingIntegrationTest extends OfflineGRPCServerIntegrationTest {
-
-  @Override
-  protected void overrideBrokerConf(PinotConfiguration brokerConf) {
-    brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_CPU_TIME_MEASUREMENT, true);
-    brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
-
-    String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
-    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME, PerQueryCPUMemAccountantFactory.class.getName());
-    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_CPU_SAMPLING, true);
-    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_CPU_TIME_BASED_KILLING_ENABLED, true);
-    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
-    brokerConf.setProperty(prefix + Accounting.CONFIG_OF_OOM_PROTECTION_KILLING_QUERY, true);
-  }
-
-  @Override
-  protected void overrideServerConf(PinotConfiguration serverConf) {
-    serverConf.setProperty(Server.CONFIG_OF_ENABLE_THREAD_CPU_TIME_MEASUREMENT, true);
-    serverConf.setProperty(Server.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
-
-    String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME, PerQueryCPUMemAccountantFactory.class.getName());
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_CPU_SAMPLING, true);
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_CPU_TIME_BASED_KILLING_ENABLED, true);
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
-    serverConf.setProperty(prefix + Accounting.CONFIG_OF_OOM_PROTECTION_KILLING_QUERY, true);
-  }
-}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineSecureGRPCServerIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineSecureGRPCServerIntegrationTest.java
index c087202587a..202e7a03bbd 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineSecureGRPCServerIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineSecureGRPCServerIntegrationTest.java
@@ -35,6 +35,8 @@ public class OfflineSecureGRPCServerIntegrationTest extends OfflineGRPCServerInt
 
   @Override
   protected void overrideServerConf(PinotConfiguration serverConf) {
+    super.overrideServerConf(serverConf);
+
     serverConf.setProperty(Server.CONFIG_OF_GRPCTLS_SERVER_ENABLED, true);
     serverConf.setProperty("pinot.server.grpctls.client.auth.enabled", true);
     serverConf.setProperty("pinot.server.grpctls.keystore.type", JKS);
diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/mailbox/InMemorySendingMailbox.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/mailbox/InMemorySendingMailbox.java
index 4b0166f5016..6d8233568fa 100644
--- a/pinot-query-runtime/src/main/java/org/apache/pinot/query/mailbox/InMemorySendingMailbox.java
+++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/mailbox/InMemorySendingMailbox.java
@@ -18,19 +18,14 @@
  */
 package org.apache.pinot.query.mailbox;
 
-import java.time.Duration;
-import java.time.temporal.ChronoUnit;
 import java.util.Collections;
 import java.util.List;
-import java.util.concurrent.TimeoutException;
 import org.apache.pinot.common.datatable.StatMap;
 import org.apache.pinot.query.runtime.blocks.ErrorMseBlock;
 import org.apache.pinot.query.runtime.blocks.MseBlock;
 import org.apache.pinot.query.runtime.operator.MailboxSendOperator;
 import org.apache.pinot.segment.spi.memory.DataBuffer;
 import org.apache.pinot.spi.exception.QueryCancelledException;
-import org.apache.pinot.spi.exception.QueryErrorCode;
-import org.apache.pinot.spi.exception.QueryException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -87,18 +82,7 @@ public class InMemorySendingMailbox implements SendingMailbox {
     }
     _statMap.merge(MailboxSendOperator.StatKey.IN_MEMORY_MESSAGES, 1);
     long timeoutMs = _deadlineMs - System.currentTimeMillis();
-    ReceivingMailbox.ReceivingMailboxStatus status;
-    try {
-      status = _receivingMailbox.offer(block, serializedStats, timeoutMs);
-    } catch (InterruptedException e) {
-      // We are not restoring the interrupt status because we are already throwing an exception
-      // Code that catches this exception must finish the work fast enough to comply the interrupt contract
-      // See https://github.com/apache/pinot/pull/16903#discussion_r2409003423
-      throw new QueryException(QueryErrorCode.INTERNAL, "Interrupted while sending data to mailbox: " + _id, e);
-    } catch (TimeoutException e) {
-      throw new QueryException(QueryErrorCode.EXECUTION_TIMEOUT, "Timed out adding block into mailbox: " + _id
-          + " with timeout: " + Duration.of(timeoutMs, ChronoUnit.MILLIS), e);
-    }
+    ReceivingMailbox.ReceivingMailboxStatus status = _receivingMailbox.offer(block, serializedStats, timeoutMs);
     switch (status) {
       case SUCCESS:
         break;
diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/mailbox/ReceivingMailbox.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/mailbox/ReceivingMailbox.java
index 9bef391d21c..4cc69962539 100644
--- a/pinot-query-runtime/src/main/java/org/apache/pinot/query/mailbox/ReceivingMailbox.java
+++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/mailbox/ReceivingMailbox.java
@@ -20,7 +20,6 @@ package org.apache.pinot.query.mailbox;
 
 import com.google.common.base.Preconditions;
 import com.google.errorprone.annotations.ThreadSafe;
-import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.List;
@@ -41,6 +40,8 @@ import org.apache.pinot.query.runtime.blocks.SerializedDataBlock;
 import org.apache.pinot.query.runtime.blocks.SuccessMseBlock;
 import org.apache.pinot.segment.spi.memory.DataBuffer;
 import org.apache.pinot.spi.exception.QueryErrorCode;
+import org.apache.pinot.spi.exception.TerminationException;
+import org.apache.pinot.spi.query.QueryThreadContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -49,16 +50,19 @@ import org.slf4j.LoggerFactory;
 /// initialized even before the corresponding OpChain is registered on the receiver, whereas the SendingMailbox is
 /// initialized when the send operator is running.
 ///
-/// There is a single ReceivingMailbox for each pair of (sender, receiver) opchains. This means that each receive
-/// operator will have multiple ReceivingMailbox instances, one for each sender. They are coordinated by a
-/// [BlockingMultiStreamConsumer].
+/// There is a single ReceivingMailbox for each pair of (sender, receiver). This means that each receive operator will
+/// have multiple ReceivingMailbox instances, one for each sender. They are coordinated by a
+/// [org.apache.pinot.query.runtime.operator.utils.BlockingMultiStreamConsumer].
 ///
 /// A ReceivingMailbox can have at most one reader and one writer at any given time. This means that different threads
 /// writing to the same mailbox must be externally synchronized.
 ///
 /// The offer methods will be called when new blocks are received from different sources. For example local workers will
-/// directly call [#offer(MseBlock, List, long)] while each remote worker opens a GPRC channel where messages
-/// are sent in raw format and [#offerRaw(List, long)] is called from them.
+/// directly call [#offer(MseBlock, List, long)] while each remote worker opens a gPRC channel where messages are sent
+/// in raw format and [#offerRaw(List, long)] is called from them.
+///
+/// All exceptions thrown from the offer methods should be handled within this class, and converted into a proper error
+/// block to be consumed by the reader.
 @ThreadSafe
 public class ReceivingMailbox {
   public static final int DEFAULT_MAX_PENDING_BLOCKS = 5;
@@ -91,53 +95,62 @@ public class ReceivingMailbox {
     return _id;
   }
 
-  ///Offers a raw block into the mailbox within the timeout specified, returns whether the block is successfully added.
-  ///
-  ///Contrary to [#offer(MseBlock, List, long)], the block may be an error block.
-  public ReceivingMailboxStatus offerRaw(List<ByteBuffer> byteBuffers, long timeoutMs)
-      throws IOException, InterruptedException, TimeoutException {
+  /// Offers a raw block into the mailbox within the timeout specified, returns the status of the mailbox.
+  public ReceivingMailboxStatus offerRaw(List<ByteBuffer> byteBuffers, long timeoutMs) {
     updateWaitCpuTime();
 
-    long startTimeMs = System.currentTimeMillis();
-    int totalBytes = 0;
-    for (ByteBuffer bb : byteBuffers) {
-      totalBytes += bb.remaining();
-    }
-    DataBlock dataBlock = DataBlockUtils.deserialize(byteBuffers);
-    _stats.merge(StatKey.DESERIALIZED_MESSAGES, 1);
-    _stats.merge(StatKey.DESERIALIZED_BYTES, totalBytes);
-    _stats.merge(StatKey.DESERIALIZATION_TIME_MS, System.currentTimeMillis() - startTimeMs);
-
     MseBlock block;
-    if (dataBlock instanceof MetadataBlock) {
-      Map<Integer, String> exceptions = dataBlock.getExceptions();
-      if (exceptions.isEmpty()) {
-        block = SuccessMseBlock.INSTANCE;
+    List<DataBuffer> stats;
+    try {
+      long startTimeMs = System.currentTimeMillis();
+      int totalBytes = 0;
+      for (ByteBuffer bb : byteBuffers) {
+        totalBytes += bb.remaining();
+      }
+      DataBlock dataBlock = DataBlockUtils.deserialize(byteBuffers);
+      stats = dataBlock.getStatsByStage();
+      _stats.merge(StatKey.DESERIALIZED_MESSAGES, 1);
+      _stats.merge(StatKey.DESERIALIZED_BYTES, totalBytes);
+      _stats.merge(StatKey.DESERIALIZATION_TIME_MS, System.currentTimeMillis() - startTimeMs);
+
+      if (dataBlock instanceof MetadataBlock) {
+        Map<Integer, String> exceptions = dataBlock.getExceptions();
+        if (exceptions.isEmpty()) {
+          block = SuccessMseBlock.INSTANCE;
+        } else {
+          MetadataBlock metadataBlock = (MetadataBlock) dataBlock;
+          Map<QueryErrorCode, String> exceptionsByQueryError = QueryErrorCode.fromKeyMap(exceptions);
+          block =
+              new ErrorMseBlock(metadataBlock.getStageId(), metadataBlock.getWorkerId(), metadataBlock.getServerId(),
+                  exceptionsByQueryError);
+        }
       } else {
-        MetadataBlock metadataBlock = (MetadataBlock) dataBlock;
-        Map<QueryErrorCode, String> exceptionsByQueryError = QueryErrorCode.fromKeyMap(exceptions);
-        block = new ErrorMseBlock(metadataBlock.getStageId(), metadataBlock.getWorkerId(), metadataBlock.getServerId(),
-                exceptionsByQueryError);
+        block = new SerializedDataBlock(dataBlock);
       }
-    } else {
-      block = new SerializedDataBlock(dataBlock);
+    } catch (Exception e) {
+      // Use the terminate exception when query is explicitly terminated.
+      TerminationException terminateException = QueryThreadContext.getTerminateException();
+      if (terminateException != null) {
+        block = ErrorMseBlock.fromException(terminateException);
+      } else {
+        String errorMessage = "Caught exception while deserializing DataBlock on mailbox: " + _id;
+        LOGGER.error(errorMessage, e);
+        block = ErrorMseBlock.fromError(QueryErrorCode.INTERNAL, errorMessage + ": " + e);
+      }
+      stats = List.of();
     }
-    return offerPrivate(block, dataBlock.getStatsByStage(), timeoutMs);
+    return offerPrivate(block, stats, timeoutMs);
   }
 
-  /// Offers a non-error block into the mailbox within the timeout specified, returns whether the block is successfully
-  /// added.
-  public ReceivingMailboxStatus offer(MseBlock block, List<DataBuffer> serializedStats, long timeoutMs)
-      throws InterruptedException, TimeoutException {
+  /// Offers a block into the mailbox within the timeout specified, returns the status of the mailbox.
+  public ReceivingMailboxStatus offer(MseBlock block, List<DataBuffer> serializedStats, long timeoutMs) {
     updateWaitCpuTime();
     _stats.merge(StatKey.IN_MEMORY_MESSAGES, 1);
     return offerPrivate(block, serializedStats, timeoutMs);
   }
 
-  /// Offers a non-error block into the mailbox within the timeout specified, returns whether the block is successfully
-  /// added.
-  private ReceivingMailboxStatus offerPrivate(MseBlock block, List<DataBuffer> stats, long timeoutMs)
-      throws InterruptedException, TimeoutException {
+  /// Offers a block into the mailbox within the timeout specified, returns the status of the mailbox.
+  private ReceivingMailboxStatus offerPrivate(MseBlock block, List<DataBuffer> stats, long timeoutMs) {
     long start = System.currentTimeMillis();
     try {
       ReceivingMailboxStatus result;
@@ -161,13 +174,27 @@ public class ReceivingMailbox {
           // Nothing to do
       }
       return result;
-    } catch (TimeoutException e) {
+    } catch (Exception e) {
       _stats.merge(StatKey.OFFER_CPU_TIME_MS, System.currentTimeMillis() - start);
-      throw e;
-    } catch (InterruptedException e) {
-      String errorMessage = "Interrupted on mailbox " + _id + " while offering blocks";
-      setErrorBlock(ErrorMseBlock.fromError(QueryErrorCode.INTERNAL, errorMessage), stats);
-      throw e;
+
+      // Use the terminate exception when query is explicitly terminated.
+      TerminationException terminateException = QueryThreadContext.getTerminateException();
+      if (terminateException != null) {
+        return _blocks.offerEos(ErrorMseBlock.fromException(terminateException), stats);
+      }
+
+      // TODO: Revisit if we should log the exception.
+      if (e instanceof TimeoutException) {
+        return _blocks.offerEos(ErrorMseBlock.fromError(QueryErrorCode.EXECUTION_TIMEOUT,
+            "Timed out while waiting for receive operator to consume data from mailbox: " + _id), stats);
+      }
+      if (e instanceof InterruptedException) {
+        return _blocks.offerEos(ErrorMseBlock.fromError(QueryErrorCode.INTERNAL,
+            "Interrupted on mailbox: " + _id + " while offering blocks"), stats);
+      }
+
+      LOGGER.error("Caught unexpected exception on mailbox: {} while offering blocks", _id, e);
+      return _blocks.offerEos(ErrorMseBlock.fromException(e), stats);
     }
   }
 
@@ -515,14 +542,8 @@ public class ReceivingMailbox {
               throw new IllegalStateException("Unexpected state: " + _state);
           }
         }
-        if (nanos <= 0L) { // timed out
-          String errorMessage = "Timed out while waiting for receive operator to consume data from mailbox: " + _id;
-          ErrorMseBlock timeoutBlock = ErrorMseBlock.fromError(QueryErrorCode.EXECUTION_TIMEOUT, errorMessage);
-          changeState(State.FULL_CLOSED, "timed out while waiting to offer data block");
-          drainDataBlocks();
-          _eos = new MseBlockWithStats(timeoutBlock, List.of());
-          notifyReader();
-          throw new TimeoutException(errorMessage);
+        if (nanos <= 0L) {
+          throw new TimeoutException();
         }
         items[_putIndex] = block;
         if (++_putIndex == items.length) {
diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/mailbox/channel/MailboxContentObserver.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/mailbox/channel/MailboxContentObserver.java
index d8a2f04d0c0..a37f93598f3 100644
--- a/pinot-query-runtime/src/main/java/org/apache/pinot/query/mailbox/channel/MailboxContentObserver.java
+++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/mailbox/channel/MailboxContentObserver.java
@@ -25,7 +25,6 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.pinot.common.proto.Mailbox.MailboxContent;
 import org.apache.pinot.common.proto.Mailbox.MailboxStatus;
@@ -77,21 +76,7 @@ public class MailboxContentObserver implements StreamObserver<MailboxContent> {
     }
     try {
       long timeoutMs = Context.current().getDeadline().timeRemaining(TimeUnit.MILLISECONDS);
-      ReceivingMailbox.ReceivingMailboxStatus status = null;
-      try {
-        status = _mailbox.offerRaw(_mailboxBuffers, timeoutMs);
-      } catch (TimeoutException e) {
-        LOGGER.debug("Timed out adding block into mailbox: {} with timeout: {}ms", mailboxId, timeoutMs);
-        closeStream();
-        return;
-      } catch (InterruptedException e) {
-        // We are not restoring the interrupt status because we are already throwing an exception
-        // Code that catches this exception must finish the work fast enough to comply the interrupt contract
-        // See https://github.com/apache/pinot/pull/16903#discussion_r2409003423
-        LOGGER.debug("Interrupted while processing blocks for mailbox: {}", mailboxId, e);
-        closeStream();
-        return;
-      }
+      ReceivingMailbox.ReceivingMailboxStatus status = _mailbox.offerRaw(_mailboxBuffers, timeoutMs);
       switch (status) {
         case SUCCESS:
           _responseObserver.onNext(MailboxStatus.newBuilder().setMailboxId(mailboxId)
diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/QueryRunner.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/QueryRunner.java
index 72da1a20aba..cbb5086f8ac 100644
--- a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/QueryRunner.java
+++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/QueryRunner.java
@@ -224,7 +224,7 @@ public class QueryRunner {
       _executorService = new ThrottleOnCriticalHeapUsageExecutor(_executorService);
     }
 
-    _opChainScheduler = new OpChainSchedulerService(_executorService, serverConf);
+    _opChainScheduler = new OpChainSchedulerService(instanceDataManager.getInstanceId(), _executorService, serverConf);
     _mailboxService = new MailboxService(hostname, port, InstanceType.SERVER, serverConf, tlsConfig);
     try {
       _leafQueryExecutor = new ServerQueryExecutorV1Impl();
diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/executor/OpChainSchedulerService.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/executor/OpChainSchedulerService.java
index 2a931acf5df..794e320eefa 100644
--- a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/executor/OpChainSchedulerService.java
+++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/executor/OpChainSchedulerService.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pinot.query.runtime.executor;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.util.concurrent.FutureCallback;
@@ -57,14 +58,15 @@ public class OpChainSchedulerService {
   private static final int NUM_QUERY_LOCKS = 1 << 10; // 1024 locks
   private static final int QUERY_LOCK_MASK = NUM_QUERY_LOCKS - 1;
 
+  private final String _instanceId;
   /// This [ExecutorService] must be wrapped with [QueryThreadContext#contextAwareExecutorService].
   private final ExecutorService _executorService;
   private final Cache<OpChainId, Pair<MultiStageOperator, QueryExecutionContext>> _opChainCache;
   private final ReadWriteLock[] _queryLocks;
   private final Cache<Long, Boolean> _cancelledQueryCache;
 
-  public OpChainSchedulerService(ExecutorService executorService, PinotConfiguration config) {
-    this(executorService, config.getProperty(MultiStageQueryRunner.KEY_OF_OP_STATS_CACHE_SIZE,
+  public OpChainSchedulerService(String instanceId, ExecutorService executorService, PinotConfiguration config) {
+    this(instanceId, executorService, config.getProperty(MultiStageQueryRunner.KEY_OF_OP_STATS_CACHE_SIZE,
             MultiStageQueryRunner.DEFAULT_OF_OP_STATS_CACHE_SIZE),
         config.getProperty(MultiStageQueryRunner.KEY_OF_OP_STATS_CACHE_EXPIRE_MS,
             MultiStageQueryRunner.DEFAULT_OF_OP_STATS_CACHE_EXPIRE_MS),
@@ -74,15 +76,17 @@ public class OpChainSchedulerService {
             MultiStageQueryRunner.DEFAULT_OF_CANCELLED_QUERY_CACHE_EXPIRE_MS));
   }
 
+  @VisibleForTesting
   public OpChainSchedulerService(ExecutorService executorService) {
-    this(executorService, MultiStageQueryRunner.DEFAULT_OF_OP_STATS_CACHE_SIZE,
+    this("testServer", executorService, MultiStageQueryRunner.DEFAULT_OF_OP_STATS_CACHE_SIZE,
         MultiStageQueryRunner.DEFAULT_OF_OP_STATS_CACHE_EXPIRE_MS,
         MultiStageQueryRunner.DEFAULT_OF_CANCELLED_QUERY_CACHE_SIZE,
         MultiStageQueryRunner.DEFAULT_OF_CANCELLED_QUERY_CACHE_EXPIRE_MS);
   }
 
-  public OpChainSchedulerService(ExecutorService executorService, int opStatsCacheSize, long opStatsCacheExpireMs,
-      int cancelledQueryCacheSize, long cancelledQueryCacheExpireMs) {
+  private OpChainSchedulerService(String instanceId, ExecutorService executorService, int opStatsCacheSize,
+      long opStatsCacheExpireMs, int cancelledQueryCacheSize, long cancelledQueryCacheExpireMs) {
+    _instanceId = instanceId;
     _executorService = executorService;
     _opChainCache = CacheBuilder.newBuilder()
         .weigher((OpChainId k, Pair<MultiStageOperator, QueryExecutionContext> v) -> countOperators(v.getLeft()))
@@ -113,7 +117,7 @@ public class OpChainSchedulerService {
       // Do not schedule the operator chain if the query has been cancelled.
       if (_cancelledQueryCache.getIfPresent(requestId) != null) {
         LOGGER.debug("({}): Query has been cancelled", operatorChain);
-        executionContext.terminate(QueryErrorCode.QUERY_CANCELLATION, "Cancelled by user");
+        executionContext.terminate(QueryErrorCode.QUERY_CANCELLATION, "Cancelled on: " + _instanceId);
         throw new QueryCancelledException(
             "Query has been cancelled before op-chain: " + operatorChain.getId() + " being scheduled");
       } else {
@@ -194,7 +198,7 @@ public class OpChainSchedulerService {
     }
 
     if (cancelledExecutionContext != null) {
-      cancelledExecutionContext.terminate(QueryErrorCode.QUERY_CANCELLATION, "Cancelled by user");
+      cancelledExecutionContext.terminate(QueryErrorCode.QUERY_CANCELLATION, "Cancelled on: " + _instanceId);
       _opChainCache.invalidateAll(cancelledOperators.keySet());
       Map<Integer, MultiStageQueryStats.StageStats.Closed> statsMap = new HashMap<>();
       for (Map.Entry<OpChainId, MultiStageOperator> entry : cancelledOperators.entrySet()) {
diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/MailboxSendOperator.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/MailboxSendOperator.java
index fe24b253f55..e442eab1ca9 100644
--- a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/MailboxSendOperator.java
+++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/MailboxSendOperator.java
@@ -45,7 +45,7 @@ import org.apache.pinot.query.runtime.plan.OpChainExecutionContext;
 import org.apache.pinot.segment.spi.memory.DataBuffer;
 import org.apache.pinot.spi.exception.QueryCancelledException;
 import org.apache.pinot.spi.exception.QueryException;
-import org.apache.pinot.spi.exception.TerminationException;
+import org.apache.pinot.spi.query.QueryThreadContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -168,8 +168,8 @@ public class MailboxSendOperator extends MultiStageOperator {
     List<MailboxInfo> mailboxInfos =
         context.getWorkerMetadata().getMailboxInfosMap().get(receiverStageId).getMailboxInfos();
     List<RoutingInfo> routingInfos =
-          MailboxIdUtils.toRoutingInfos(requestId, context.getStageId(), context.getWorkerId(), receiverStageId,
-              mailboxInfos);
+        MailboxIdUtils.toRoutingInfos(requestId, context.getStageId(), context.getWorkerId(), receiverStageId,
+            mailboxInfos);
     List<SendingMailbox> sendingMailboxes = routingInfos.stream()
         .map(v -> mailboxService.getSendingMailbox(v.getHostname(), v.getPort(), v.getMailboxId(), deadlineMs, statMap))
         .collect(Collectors.toList());
@@ -214,21 +214,29 @@ public class MailboxSendOperator extends MultiStageOperator {
         sendEos((MseBlock.Eos) block);
       } else {
         sendMseBlock(((MseBlock.Data) block));
+        checkTerminationAndSampleUsage();
       }
-      checkTerminationAndSampleUsage();
       return block;
-    } catch (QueryCancelledException e) {
-      LOGGER.debug("Query was cancelled for opChain: {}", _context.getId());
-      return SuccessMseBlock.INSTANCE;
-    } catch (TerminationException e) {
-      LOGGER.info("Query was terminated for opChain: {}", _context.getId(), e);
-      return ErrorMseBlock.fromException(e);
-    } catch (QueryException e) {
-      return ErrorMseBlock.fromException(e);
     } catch (RuntimeException e) {
-      ErrorMseBlock errorBlock = ErrorMseBlock.fromException(e);
-      try {
+      if (e instanceof QueryCancelledException) {
+        LOGGER.debug("Query was cancelled for opChain: {}", _context.getId());
+        // TODO: Revisit if we should return success block here.
+        return SuccessMseBlock.INSTANCE;
+      }
+      ErrorMseBlock errorBlock;
+      // First check terminate exception and use it as the results block if exists. We want to return the termination
+      // reason when query is explicitly terminated.
+      QueryException queryException = QueryThreadContext.getTerminateException();
+      if (queryException == null && e instanceof QueryException) {
+        queryException = (QueryException) e;
+      }
+      if (queryException != null) {
+        errorBlock = ErrorMseBlock.fromException(queryException);
+      } else {
         LOGGER.error("Exception while transferring data on opChain: {}", _context.getId(), e);
+        errorBlock = ErrorMseBlock.fromException(e);
+      }
+      try {
         sendEos(errorBlock);
       } catch (RuntimeException e2) {
         LOGGER.error("Exception while sending error block.", e2);
diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/utils/BlockingMultiStreamConsumer.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/utils/BlockingMultiStreamConsumer.java
index ca6e3dcc8ff..7a6f3627191 100644
--- a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/utils/BlockingMultiStreamConsumer.java
+++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/utils/BlockingMultiStreamConsumer.java
@@ -33,7 +33,7 @@ import org.apache.pinot.query.runtime.plan.MultiStageQueryStats;
 import org.apache.pinot.query.runtime.plan.OpChainExecutionContext;
 import org.apache.pinot.segment.spi.memory.DataBuffer;
 import org.apache.pinot.spi.exception.QueryErrorCode;
-import org.apache.pinot.spi.exception.QueryException;
+import org.apache.pinot.spi.exception.TerminationException;
 import org.apache.pinot.spi.query.QueryThreadContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -390,7 +390,7 @@ public abstract class BlockingMultiStreamConsumer<E> implements AutoCloseable {
     @Override
     protected ReceivingMailbox.MseBlockWithStats onTimeout() {
       // Use the terminate exception when query is explicitly terminated.
-      QueryException terminateException = QueryThreadContext.getTerminateException();
+      TerminationException terminateException = QueryThreadContext.getTerminateException();
       if (terminateException != null) {
         return onException(terminateException.getErrorCode(), terminateException.getMessage());
       }
@@ -408,7 +408,7 @@ public abstract class BlockingMultiStreamConsumer<E> implements AutoCloseable {
     @Override
     protected ReceivingMailbox.MseBlockWithStats onException(Exception e) {
       // Use the terminate exception when query is explicitly terminated.
-      QueryException terminateException = QueryThreadContext.getTerminateException();
+      TerminationException terminateException = QueryThreadContext.getTerminateException();
       if (terminateException != null) {
         return onException(terminateException.getErrorCode(), terminateException.getMessage());
       }
diff --git a/pinot-query-runtime/src/test/java/org/apache/pinot/query/mailbox/MailboxServiceTest.java b/pinot-query-runtime/src/test/java/org/apache/pinot/query/mailbox/MailboxServiceTest.java
index 78f21ac0569..40049f513f2 100644
--- a/pinot-query-runtime/src/test/java/org/apache/pinot/query/mailbox/MailboxServiceTest.java
+++ b/pinot-query-runtime/src/test/java/org/apache/pinot/query/mailbox/MailboxServiceTest.java
@@ -44,7 +44,10 @@ import org.testng.annotations.BeforeClass;
 import org.testng.annotations.BeforeTest;
 import org.testng.annotations.Test;
 
-import static org.testng.Assert.*;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
 
 
 public class MailboxServiceTest {
@@ -245,12 +248,10 @@ public class MailboxServiceTest {
     // Send one data block, sleep until timed out, then send one more block
     sendingMailbox.send(OperatorTestUtil.block(DATA_SCHEMA, new Object[]{"0"}));
     Thread.sleep(deadlineMs - System.currentTimeMillis() + 10);
-    try {
-      sendingMailbox.send(OperatorTestUtil.block(DATA_SCHEMA, new Object[]{"1"}));
-      fail("Expect exception when sending data after timing out");
-    } catch (Exception e) {
-      // Expected
-    }
+    sendingMailbox.send(OperatorTestUtil.block(DATA_SCHEMA, new Object[]{"1"}));
+
+    // Mailbox should be terminated due to timeout
+    assertTrue(sendingMailbox.isTerminated());
 
     // Data blocks will be cleaned up
     assertEquals(numCallbacks.get(), 2);
@@ -282,14 +283,11 @@ public class MailboxServiceTest {
       sendingMailbox.send(OperatorTestUtil.block(DATA_SCHEMA, new Object[]{Integer.toString(i)}));
     }
 
-    // Next send will throw exception because buffer is full
-    try {
-      // The block sent must be a data block, as error and eos blocks are always accepted
-      sendingMailbox.send(OperatorTestUtil.block(DATA_SCHEMA, new Object[]{Integer.toString(1)}));
-      fail("Except exception when sending data after buffer is full");
-    } catch (Exception e) {
-      // Expected
-    }
+    // Next send will block until timeout because buffer is full
+    sendingMailbox.send(OperatorTestUtil.block(DATA_SCHEMA, new Object[]{Integer.toString(1)}));
+
+    // Mailbox should be terminated due to timeout
+    assertTrue(sendingMailbox.isTerminated());
 
     // Data blocks will be cleaned up
     assertEquals(numCallbacks.get(), ReceivingMailbox.DEFAULT_MAX_PENDING_BLOCKS + 1);
@@ -567,20 +565,10 @@ public class MailboxServiceTest {
       receiveMailLatch.countDown();
     });
 
-    // Send one data block, RPC will timeout after deadline
+    // Send one data block, receiver will time out after deadline
     sendingMailbox.send(OperatorTestUtil.block(DATA_SCHEMA, new Object[]{"0"}));
-    Thread.sleep(deadlineMs - System.currentTimeMillis() + 10);
     receiveMailLatch.await();
     assertEquals(numCallbacks.get(), 2);
-    // TODO: Currently we cannot differentiate early termination vs stream error
-    assertTrue(sendingMailbox.isTerminated());
-//    try {
-//      sendingMailbox.send(OperatorTestUtil.block(DATA_SCHEMA, new Object[]{1}));
-//      fail("Expect exception when sending data after timing out");
-//    } catch (Exception e) {
-//      // Expected
-//    }
-//    assertEquals(numCallbacks.get(), 2);
 
     // Data blocks will be cleaned up
     assertEquals(receivingMailbox.getNumPendingBlocks(), 0);
@@ -588,6 +576,9 @@ public class MailboxServiceTest {
     assertNotNull(block);
     assertTrue(block.isError());
 
+    // Sender side should be terminated after receiver side times out and closes the connection
+    TestUtils.waitForCondition(aVoid -> sendingMailbox.isTerminated(), 1000L, "Failed to terminate sender");
+
     // Cancel is idempotent for both sending and receiving mailbox, so safe to call multiple times
     sendingMailbox.cancel(new Exception("TEST ERROR"));
     receivingMailbox.cancel();
@@ -610,18 +601,14 @@ public class MailboxServiceTest {
       receiveMailLatch.countDown();
     });
 
-    // send blocks big enough to avoid block combining in the sending side
-    // although fragile, this works because we know that outboundMax = inboundMax/2
-    String payload = new String(new char[MAX_DATA_BLOCK_SIZE / 3]).replace('\0', 'a');
-
     // Sends are non-blocking as long as channel capacity is not breached
     for (int i = 0; i < ReceivingMailbox.DEFAULT_MAX_PENDING_BLOCKS; i++) {
-      sendingMailbox.send(OperatorTestUtil.block(DATA_SCHEMA, new Object[]{payload}));
+      sendingMailbox.send(OperatorTestUtil.block(DATA_SCHEMA, new Object[]{"0"}));
     }
 
     // Next send will be blocked on the receiver side and cause exception after timeout
     // We need to send a data block, given we don't block on EOS
-    sendingMailbox.send(OperatorTestUtil.block(DATA_SCHEMA, new Object[]{payload}));
+    sendingMailbox.send(OperatorTestUtil.block(DATA_SCHEMA, new Object[]{"0"}));
     receiveMailLatch.await();
     assertEquals(numCallbacks.get(), ReceivingMailbox.DEFAULT_MAX_PENDING_BLOCKS + 1);
 
@@ -676,8 +663,7 @@ public class MailboxServiceTest {
     ReceivingMailbox.MseBlockWithStats block = receivingMailbox.poll();
     assertNotNull(block);
     assertTrue(block.getBlock().isData());
-    List<Object[]> rows = ((MseBlock.Data) block.getBlock()).asRowHeap().getRows();
-    return rows;
+    return ((MseBlock.Data) block.getBlock()).asRowHeap().getRows();
   }
 
   @Nullable
diff --git a/pinot-query-runtime/src/test/java/org/apache/pinot/query/mailbox/ReceivingMailboxTest.java b/pinot-query-runtime/src/test/java/org/apache/pinot/query/mailbox/ReceivingMailboxTest.java
index b9c35169afd..063d4a1cc20 100644
--- a/pinot-query-runtime/src/test/java/org/apache/pinot/query/mailbox/ReceivingMailboxTest.java
+++ b/pinot-query-runtime/src/test/java/org/apache/pinot/query/mailbox/ReceivingMailboxTest.java
@@ -35,7 +35,10 @@ import org.mockito.Mockito;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
-import static org.testng.Assert.*;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
 
 
 public class ReceivingMailboxTest {
@@ -51,27 +54,31 @@ public class ReceivingMailboxTest {
   }
 
   @Test
-  public void tooManyDataBlocksTheWriter()
-      throws InterruptedException, TimeoutException {
+  public void tooManyDataBlocksTheWriter() {
     int size = 2;
     ReceivingMailbox receivingMailbox = new ReceivingMailbox("id", size);
+    receivingMailbox.registeredReader(_reader);
 
     // Offer up to capacity
     for (int i = 0; i < size; i++) {
       ReceivingMailbox.ReceivingMailboxStatus status = receivingMailbox.offer(DATA_BLOCK, List.of(), 10);
       assertEquals(status, ReceivingMailbox.ReceivingMailboxStatus.SUCCESS, "Should be able to offer up to capacity");
     }
-    // Offer one more should be rejected
-    try {
-      ReceivingMailbox.ReceivingMailboxStatus status = receivingMailbox.offer(DATA_BLOCK, List.of(), 10);
-      fail("Should have thrown timeout exception, but " + status + " was returned");
-    } catch (TimeoutException e) {
-      // expected
-    }
+    // Offer one more should cause timeout
+    ReceivingMailbox.ReceivingMailboxStatus status = receivingMailbox.offer(DATA_BLOCK, List.of(), 10);
+    assertEquals(status, ReceivingMailbox.ReceivingMailboxStatus.LAST_BLOCK,
+        "Should timeout when offering over capacity");
+    ReceivingMailbox.MseBlockWithStats read = receivingMailbox.poll();
+    assertNotNull(read, "Should be able to read the timeout error block");
+    MseBlock block = read.getBlock();
+    assertTrue(block.isError(), "The block should be an error block");
+    ErrorMseBlock errorBlock = (ErrorMseBlock) block;
+    assertTrue(errorBlock.getErrorMessages().containsKey(QueryErrorCode.EXECUTION_TIMEOUT),
+        "The error block should contain timeout error");
   }
 
   @Test
-  public void offerAfterEos() throws InterruptedException, TimeoutException {
+  public void offerAfterEos() {
     ReceivingMailbox receivingMailbox = new ReceivingMailbox("id", 10);
 
     ReceivingMailbox.ReceivingMailboxStatus status = receivingMailbox.offer(DATA_BLOCK, List.of(), 10);
@@ -97,7 +104,7 @@ public class ReceivingMailboxTest {
   }
 
   @Test
-  public void shouldReadDataInOrder() throws InterruptedException, TimeoutException {
+  public void shouldReadDataInOrder() {
     ReceivingMailbox receivingMailbox = new ReceivingMailbox("id", 10);
     receivingMailbox.registeredReader(_reader);
 
@@ -121,7 +128,7 @@ public class ReceivingMailboxTest {
   }
 
   @Test
-  public void lateEosRead() throws InterruptedException, TimeoutException {
+  public void lateEosRead() {
     ReceivingMailbox receivingMailbox = new ReceivingMailbox("id", 10);
     receivingMailbox.registeredReader(_reader);
 
@@ -161,7 +168,7 @@ public class ReceivingMailboxTest {
   }
 
   @Test
-  public void bufferedDataIsKeptOnSuccess() throws InterruptedException, TimeoutException {
+  public void bufferedDataIsKeptOnSuccess() {
     ReceivingMailbox receivingMailbox = new ReceivingMailbox("id", 10);
     receivingMailbox.registeredReader(_reader);
 
@@ -189,7 +196,7 @@ public class ReceivingMailboxTest {
   }
 
   @Test
-  public void bufferedDataIsDiscardedOnError() throws InterruptedException, TimeoutException {
+  public void bufferedDataIsDiscardedOnError() {
     ReceivingMailbox receivingMailbox = new ReceivingMailbox("id", 10);
     receivingMailbox.registeredReader(_reader);
     ErrorMseBlock errorBlock = ErrorMseBlock.fromException(new RuntimeException("Test error"));
@@ -213,7 +220,7 @@ public class ReceivingMailboxTest {
   }
 
   @Test
-  public void dataAfterSuccess() throws InterruptedException, TimeoutException {
+  public void dataAfterSuccess() {
     ReceivingMailbox receivingMailbox = new ReceivingMailbox("id", 10);
     receivingMailbox.registeredReader(_reader);
 
@@ -237,7 +244,7 @@ public class ReceivingMailboxTest {
   }
 
   @Test
-  public void dataAfterError() throws InterruptedException, TimeoutException {
+  public void dataAfterError() {
     ReceivingMailbox receivingMailbox = new ReceivingMailbox("id", 10);
     receivingMailbox.registeredReader(_reader);
 
@@ -285,7 +292,7 @@ public class ReceivingMailboxTest {
 
   @Test(timeOut = 10_000)
   public void readingUnblocksWriters()
-      throws ExecutionException, InterruptedException, TimeoutException {
+      throws ExecutionException, InterruptedException {
     int maxPendingBlocks = 2;
     ReceivingMailbox mailbox = new ReceivingMailbox("id", maxPendingBlocks);
     mailbox.registeredReader(_reader);
@@ -319,15 +326,6 @@ public class ReceivingMailboxTest {
 
   CompletableFuture<ReceivingMailbox.ReceivingMailboxStatus> offer(MseBlock block, ReceivingMailbox receivingMailbox,
       ExecutorService executor) {
-    return CompletableFuture.supplyAsync(() -> {
-      try {
-        return receivingMailbox.offer(block, List.of(), 10_000);
-      } catch (InterruptedException e) {
-        Thread.currentThread().interrupt();
-        throw new RuntimeException(e);
-      } catch (TimeoutException e) {
-        throw new RuntimeException(e);
-      }
-    }, executor);
+    return CompletableFuture.supplyAsync(() -> receivingMailbox.offer(block, List.of(), 10_000), executor);
   }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org