You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2025/10/16 00:42:55 UTC
(pinot) branch master updated: Ensure proper exception is returned when query is cancelled/killed (#17009)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 01912c4aa88 Ensure proper exception is returned when query is cancelled/killed (#17009)
01912c4aa88 is described below
commit 01912c4aa88bfe045c4f101dbf6a15e654fa6657
Author: Xiaotian (Jackie) Jiang <17...@users.noreply.github.com>
AuthorDate: Wed Oct 15 17:42:48 2025 -0700
Ensure proper exception is returned when query is cancelled/killed (#17009)
---
.../StreamingInstanceResponseOperator.java | 14 +-
.../core/query/reduce/BrokerReduceService.java | 20 +-
.../pinot/core/query/scheduler/QueryScheduler.java | 56 ++--
.../core/transport/InstanceRequestHandler.java | 2 +-
.../tests/BaseQueryKillingIntegrationTest.java | 209 +++++++++++++
.../CpuBasedBrokerQueryKillingIntegrationTest.java | 123 ++++++++
.../CpuBasedServerQueryKillingIntegrationTest.java | 120 ++++++++
...moryBasedServerQueryKillingIntegrationTest.java | 119 ++++++++
.../MultiNodesOfflineClusterIntegrationTest.java | 27 ++
.../tests/OOMProtectionEnabledIntegrationTest.java | 107 -------
...flineClusterCpuBasedServerQueryKillingTest.java | 239 ---------------
...flineClusterMemBasedBrokerQueryKillingTest.java | 241 ---------------
...flineClusterMemBasedServerQueryKillingTest.java | 331 ---------------------
...rMemBasedServerResourceUsageAccountantTest.java | 302 -------------------
.../OfflineClusterResourceUsageAccountantTest.java | 272 -----------------
.../tests/OfflineGRPCServerIntegrationTest.java | 30 ++
...OfflineGRPCServerMultiStageIntegrationTest.java | 7 +-
...lineGRPCServerOOMAccountingIntegrationTest.java | 56 ----
.../OfflineSecureGRPCServerIntegrationTest.java | 2 +
.../query/mailbox/InMemorySendingMailbox.java | 18 +-
.../pinot/query/mailbox/ReceivingMailbox.java | 129 ++++----
.../mailbox/channel/MailboxContentObserver.java | 17 +-
.../apache/pinot/query/runtime/QueryRunner.java | 2 +-
.../runtime/executor/OpChainSchedulerService.java | 18 +-
.../runtime/operator/MailboxSendOperator.java | 36 ++-
.../utils/BlockingMultiStreamConsumer.java | 6 +-
.../pinot/query/mailbox/MailboxServiceTest.java | 54 ++--
.../pinot/query/mailbox/ReceivingMailboxTest.java | 54 ++--
28 files changed, 846 insertions(+), 1765 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/streaming/StreamingInstanceResponseOperator.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/streaming/StreamingInstanceResponseOperator.java
index 03b0d93f9bf..857c853bcd0 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/streaming/StreamingInstanceResponseOperator.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/streaming/StreamingInstanceResponseOperator.java
@@ -32,7 +32,6 @@ import org.apache.pinot.segment.spi.SegmentContext;
import org.apache.pinot.spi.exception.QueryErrorCode;
import org.apache.pinot.spi.exception.QueryErrorMessage;
import org.apache.pinot.spi.exception.QueryException;
-import org.apache.pinot.spi.exception.TerminationException;
import org.apache.pinot.spi.query.QueryThreadContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -91,11 +90,14 @@ public class StreamingInstanceResponseOperator extends InstanceResponseOperator
return buildInstanceResponseBlock(resultsBlock).toMetadataOnlyResponseBlock();
}
} catch (Throwable t) {
- TerminationException terminateException = QueryThreadContext.getTerminateException();
- if (terminateException != null) {
- return new InstanceResponseBlock(new ExceptionResultsBlock(terminateException));
- } else if (t instanceof QueryException) {
- return new InstanceResponseBlock(new ExceptionResultsBlock((QueryException) t));
+ // First check terminate exception and use it as the results block if exists. We want to return the termination
+ // reason when query is explicitly terminated.
+ QueryException queryException = QueryThreadContext.getTerminateException();
+ if (queryException == null && t instanceof QueryException) {
+ queryException = (QueryException) t;
+ }
+ if (queryException != null) {
+ return new InstanceResponseBlock(new ExceptionResultsBlock(queryException));
} else {
LOGGER.error("Caught exception while streaming results (query: {})", _queryContext, t);
return new InstanceResponseBlock(new ExceptionResultsBlock(QueryErrorMessage.safeMsg(QueryErrorCode.INTERNAL,
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/BrokerReduceService.java b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/BrokerReduceService.java
index dbdb805ae6c..d938deee1ec 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/BrokerReduceService.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/BrokerReduceService.java
@@ -40,7 +40,6 @@ import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.exception.BadQueryRequestException;
import org.apache.pinot.spi.exception.QueryErrorCode;
import org.apache.pinot.spi.exception.QueryException;
-import org.apache.pinot.spi.exception.TerminationException;
import org.apache.pinot.spi.query.QueryThreadContext;
import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
@@ -173,17 +172,16 @@ public class BrokerReduceService extends BaseReduceService {
} catch (RuntimeException e) {
// First check terminate exception and use it as the results block if exists. We want to return the termination
// reason when query is explicitly terminated.
- TerminationException terminateException = QueryThreadContext.getTerminateException();
- if (terminateException != null) {
- brokerResponseNative.addException(QueryProcessingException.fromQueryException(terminateException));
+ QueryException queryException = QueryThreadContext.getTerminateException();
+ if (queryException == null && e instanceof QueryException) {
+ queryException = (QueryException) e;
+ }
+ if (queryException != null) {
+ brokerResponseNative.addException(QueryProcessingException.fromQueryException(queryException));
} else {
- if (e instanceof QueryException) {
- brokerResponseNative.addException(QueryProcessingException.fromQueryException((QueryException) e));
- } else {
- LOGGER.error("Caught exception while reducing data tables (query: {})", serverQueryContext, e);
- brokerResponseNative.addException(new QueryProcessingException(QueryErrorCode.MERGE_RESPONSE,
- "Caught exception while reducing data tables: " + e.getMessage()));
- }
+ LOGGER.error("Caught exception while reducing data tables (query: {})", serverQueryContext, e);
+ brokerResponseNative.addException(new QueryProcessingException(QueryErrorCode.MERGE_RESPONSE,
+ "Caught exception while reducing data tables: " + e.getMessage()));
}
}
QueryContext queryContext;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/scheduler/QueryScheduler.java b/pinot-core/src/main/java/org/apache/pinot/core/query/scheduler/QueryScheduler.java
index 9e75cf2e372..f123de77cf8 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/scheduler/QueryScheduler.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/scheduler/QueryScheduler.java
@@ -25,7 +25,6 @@ import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.LongAccumulator;
import javax.annotation.Nullable;
-import org.apache.commons.lang3.StringUtils;
import org.apache.pinot.common.datatable.DataTable.MetadataKey;
import org.apache.pinot.common.metrics.ServerMeter;
import org.apache.pinot.common.metrics.ServerMetrics;
@@ -40,9 +39,7 @@ import org.apache.pinot.core.query.request.context.TimerContext;
import org.apache.pinot.core.query.scheduler.resources.ResourceManager;
import org.apache.pinot.spi.accounting.ThreadAccountant;
import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.exception.EarlyTerminationException;
import org.apache.pinot.spi.exception.QueryErrorCode;
-import org.apache.pinot.spi.exception.QueryErrorMessage;
import org.apache.pinot.spi.exception.QueryException;
import org.apache.pinot.spi.query.QueryExecutionContext;
import org.apache.pinot.spi.query.QueryThreadContext;
@@ -197,34 +194,49 @@ public abstract class QueryScheduler {
TimerContext timerContext = queryRequest.getTimerContext();
TimerContext.Timer responseSerializationTimer =
timerContext.startNewPhaseTimer(ServerQueryPhase.RESPONSE_SERIALIZATION);
+ long requestId = queryRequest.getRequestId();
+ String brokerId = queryRequest.getBrokerId();
- byte[] responseByte = null;
+ byte[] responseBytes = null;
try {
- responseByte = instanceResponse.toDataTable().toBytes();
- } catch (EarlyTerminationException e) {
- QueryException terminateException = QueryThreadContext.getTerminateException();
- String userMsg = "Cancelled while building data table" + (terminateException != null ? ": " + terminateException
- : StringUtils.EMPTY);
- LOGGER.error(userMsg);
- QueryErrorMessage errMsg = QueryErrorMessage.safeMsg(QueryErrorCode.QUERY_CANCELLATION, userMsg);
- Map<String, String> queryOptions = queryRequest.getQueryContext().getQueryOptions();
- String workloadName = QueryOptionsUtils.getWorkloadName(queryOptions);
- instanceResponse = new InstanceResponseBlock(new ExceptionResultsBlock(errMsg));
- instanceResponse.addMetadata(MetadataKey.REQUEST_ID.getName(), Long.toString(queryRequest.getRequestId()));
- instanceResponse.addMetadata(MetadataKey.QUERY_ID.getName(), queryRequest.getCid());
- instanceResponse.addMetadata(MetadataKey.WORKLOAD_NAME.getName(), workloadName);
- return serializeResponse(queryRequest, instanceResponse);
+ responseBytes = instanceResponse.toDataTable().toBytes();
} catch (Exception e) {
- _serverMetrics.addMeteredGlobalValue(ServerMeter.RESPONSE_SERIALIZATION_EXCEPTIONS, 1);
- LOGGER.error("Caught exception while serializing response for requestId: {}, brokerId: {}",
- queryRequest.getRequestId(), queryRequest.getBrokerId(), e);
+ // First check terminate exception and use it as the response if exists. We want to return the termination reason
+ // when query is explicitly terminated.
+ QueryException queryException = QueryThreadContext.getTerminateException();
+ // Do not log exception when query is explicitly terminated
+ if (queryException == null) {
+ if (e instanceof QueryException) {
+ queryException = (QueryException) e;
+ // TODO: Revisit if we should log exception here
+ LOGGER.warn("Caught QueryException while serializing response from response for requestId: {}, brokerId: {}",
+ requestId, brokerId, queryException);
+ } else {
+ _serverMetrics.addMeteredGlobalValue(ServerMeter.RESPONSE_SERIALIZATION_EXCEPTIONS, 1);
+ LOGGER.error("Caught exception while serializing response from response for requestId: {}, brokerId: {}",
+ requestId, brokerId, e);
+ queryException = QueryErrorCode.INTERNAL.asException("Error serializing response", e);
+ }
+ }
+ try {
+ InstanceResponseBlock errorResponse = new InstanceResponseBlock(new ExceptionResultsBlock(queryException));
+ errorResponse.addMetadata(MetadataKey.REQUEST_ID.getName(), Long.toString(requestId));
+ errorResponse.addMetadata(MetadataKey.QUERY_ID.getName(), queryRequest.getCid());
+ String workloadName = QueryOptionsUtils.getWorkloadName(queryRequest.getQueryContext().getQueryOptions());
+ errorResponse.addMetadata(MetadataKey.WORKLOAD_NAME.getName(), workloadName);
+ responseBytes = errorResponse.toDataTable().toBytes();
+ } catch (Exception e1) {
+ _serverMetrics.addMeteredGlobalValue(ServerMeter.RESPONSE_SERIALIZATION_EXCEPTIONS, 1);
+ LOGGER.error("Caught exception while constructing error response for requestId: {}, brokerId: {}",
+ requestId, brokerId, e1);
+ }
}
responseSerializationTimer.stopAndRecord();
timerContext.startNewPhaseTimer(ServerQueryPhase.TOTAL_QUERY_TIME, timerContext.getQueryArrivalTimeMs())
.stopAndRecord();
- return responseByte;
+ return responseBytes;
}
/**
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/transport/InstanceRequestHandler.java b/pinot-core/src/main/java/org/apache/pinot/core/transport/InstanceRequestHandler.java
index 1753fc914d3..f0e3d728809 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/transport/InstanceRequestHandler.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/transport/InstanceRequestHandler.java
@@ -246,7 +246,7 @@ public class InstanceRequestHandler extends SimpleChannelInboundHandler<ByteBuf>
if (executionContext == null) {
return false;
}
- boolean cancelled = executionContext.terminate(QueryErrorCode.QUERY_CANCELLATION, "Cancelled by user");
+ boolean cancelled = executionContext.terminate(QueryErrorCode.QUERY_CANCELLATION, "Cancelled on: " + _instanceName);
if (LOGGER.isDebugEnabled()) {
if (cancelled) {
LOGGER.debug("Cancelled query: {}", queryId);
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseQueryKillingIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseQueryKillingIntegrationTest.java
new file mode 100644
index 00000000000..871d909f82c
--- /dev/null
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseQueryKillingIntegrationTest.java
@@ -0,0 +1,209 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import org.apache.avro.Schema.Field;
+import org.apache.avro.Schema.Type;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.commons.io.FileUtils;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.env.PinotConfiguration;
+import org.apache.pinot.spi.utils.CommonConstants.Broker;
+import org.apache.pinot.spi.utils.CommonConstants.Server;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+import org.apache.pinot.util.TestUtils;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+
+/// Base class for query killing integration tests.
+public abstract class BaseQueryKillingIntegrationTest extends BaseClusterIntegrationTest {
+ protected static final String STRING_DIM_SV1 = "stringDimSV1";
+ protected static final String STRING_DIM_SV2 = "stringDimSV2";
+ protected static final String INT_DIM_SV1 = "intDimSV1";
+ protected static final String LONG_DIM_SV1 = "longDimSV1";
+ protected static final String DOUBLE_DIM_SV1 = "doubleDimSV1";
+ protected static final String BOOLEAN_DIM_SV1 = "booleanDimSV1";
+ protected static final int NUM_SEGMENTS = 3;
+ protected static final int NUM_DOCS_PER_SEGMENT = 3_000_000;
+
+ protected static final String LARGE_SELECT_STAR_QUERY = "SELECT * FROM mytable LIMIT 9000000";
+
+ protected static final String LARGE_DISTINCT_QUERY =
+ "SELECT DISTINCT stringDimSV2 FROM mytable ORDER BY stringDimSV2 LIMIT 3000000";
+
+ protected static final String LARGE_GROUP_BY_QUERY =
+ "SELECT DISTINCT_COUNT_HLL(intDimSV1, 14), stringDimSV2 FROM mytable GROUP BY 2 ORDER BY 1 LIMIT 3000000";
+
+ /// SafeTrim sort-aggregation case, pair-wise combine
+ protected static final String LARGE_GROUP_BY_PAIRWISE_COMBINE_QUERY =
+ "SET sortAggregateSingleThreadedNumSegmentsThreshold=1; SET sortAggregateLimitThreshold=3000001; "
+ + "SELECT DISTINCT_COUNT_HLL(intDimSV1, 14), stringDimSV2 FROM mytable GROUP BY 2 ORDER BY 2 LIMIT 3000000";
+
+ /// SafeTrim sort-aggregation case, sequential combine
+ protected static final String LARGE_GROUP_BY_SEQUENTIAL_COMBINE_QUERY =
+ "SET sortAggregateSingleThreadedNumSegmentsThreshold=10000; SET sortAggregateLimitThreshold=3000001; "
+ + "SELECT DISTINCT_COUNT_HLL(intDimSV1, 14), stringDimSV2 FROM mytable GROUP BY 2 ORDER BY 2 LIMIT 3000000";
+
+ protected static final String AGGREGATE_QUERY = "SELECT DISTINCT_COUNT_HLL(intDimSV1, 14) FROM mytable";
+ protected static final String SELECT_STAR_QUERY = "SELECT * FROM mytable LIMIT 5";
+
+ @BeforeClass
+ public void setUp()
+ throws Exception {
+ TestUtils.ensureDirectoriesExistAndEmpty(_tempDir, _segmentDir, _tarDir);
+
+ // Start the Pinot cluster
+ startZk();
+ startController();
+ startBroker();
+ startServer();
+
+ // Create and upload the schema and table config
+ Schema schema = new Schema.SchemaBuilder().setSchemaName(DEFAULT_TABLE_NAME)
+ .addSingleValueDimension(STRING_DIM_SV1, FieldSpec.DataType.STRING)
+ .addSingleValueDimension(STRING_DIM_SV2, FieldSpec.DataType.STRING)
+ .addSingleValueDimension(INT_DIM_SV1, FieldSpec.DataType.INT)
+ .addSingleValueDimension(LONG_DIM_SV1, FieldSpec.DataType.LONG)
+ .addSingleValueDimension(DOUBLE_DIM_SV1, FieldSpec.DataType.DOUBLE)
+ .addSingleValueDimension(BOOLEAN_DIM_SV1, FieldSpec.DataType.BOOLEAN)
+ .build();
+ addSchema(schema);
+ TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(DEFAULT_TABLE_NAME).build();
+ addTableConfig(tableConfig);
+
+ List<File> avroFiles = createAvroFiles();
+ ClusterIntegrationTestUtils.buildSegmentsFromAvro(avroFiles, tableConfig, schema, 0, _segmentDir, _tarDir);
+ uploadSegments(DEFAULT_TABLE_NAME, _tarDir);
+
+ // Wait for all documents loaded
+ waitForAllDocsLoaded(600_000L);
+ }
+
+ @AfterClass
+ public void tearDown()
+ throws IOException {
+ stopServer();
+ stopBroker();
+ stopController();
+ stopZk();
+ FileUtils.deleteDirectory(_tempDir);
+ }
+
+ @Override
+ protected void overrideBrokerConf(PinotConfiguration brokerConf) {
+ brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_CPU_TIME_MEASUREMENT, true);
+ brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
+ }
+
+ @Override
+ protected void overrideServerConf(PinotConfiguration serverConf) {
+ serverConf.setProperty(Server.CONFIG_OF_ENABLE_THREAD_CPU_TIME_MEASUREMENT, true);
+ serverConf.setProperty(Server.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
+ serverConf.setProperty(Server.CONFIG_OF_QUERY_EXECUTOR_NUM_GROUPS_LIMIT, Integer.MAX_VALUE);
+ }
+
+ @Override
+ protected long getCountStarResult() {
+ return NUM_SEGMENTS * NUM_DOCS_PER_SEGMENT;
+ }
+
+ @Test
+ public void testResourceUsageStats()
+ throws Exception {
+ JsonNode queryResponse = postQuery(SELECT_STAR_QUERY);
+ long offlineThreadMemAllocatedBytes = queryResponse.get("offlineThreadMemAllocatedBytes").asLong();
+ long offlineResponseSerMemAllocatedBytes = queryResponse.get("offlineResponseSerMemAllocatedBytes").asLong();
+ long offlineTotalMemAllocatedBytes = queryResponse.get("offlineTotalMemAllocatedBytes").asLong();
+
+ assertTrue(offlineThreadMemAllocatedBytes > 0);
+ assertTrue(offlineResponseSerMemAllocatedBytes > 0);
+ assertEquals(offlineThreadMemAllocatedBytes + offlineResponseSerMemAllocatedBytes, offlineTotalMemAllocatedBytes);
+
+ long offlineThreadCpuTimeNs = queryResponse.get("offlineThreadCpuTimeNs").asLong();
+ long offlineSystemActivitiesCpuTimeNs = queryResponse.get("offlineSystemActivitiesCpuTimeNs").asLong();
+ long offlineResponseSerializationCpuTimeNs = queryResponse.get("offlineResponseSerializationCpuTimeNs").asLong();
+ long offlineTotalCpuTimeNs = queryResponse.get("offlineTotalCpuTimeNs").asLong();
+ assertTrue(offlineThreadCpuTimeNs > 0);
+ assertTrue(offlineSystemActivitiesCpuTimeNs > 0);
+ assertTrue(offlineResponseSerializationCpuTimeNs > 0);
+ assertEquals(offlineThreadCpuTimeNs + offlineSystemActivitiesCpuTimeNs + offlineResponseSerializationCpuTimeNs,
+ offlineTotalCpuTimeNs);
+ }
+
+ @DataProvider
+ public String[] expensiveQueries() {
+ return new String[]{
+ LARGE_SELECT_STAR_QUERY, LARGE_DISTINCT_QUERY, LARGE_GROUP_BY_QUERY, LARGE_GROUP_BY_PAIRWISE_COMBINE_QUERY,
+ LARGE_GROUP_BY_SEQUENTIAL_COMBINE_QUERY
+ };
+ }
+
+ protected List<File> createAvroFiles()
+ throws IOException {
+ // Create Avro schema
+ org.apache.avro.Schema avroSchema = org.apache.avro.Schema.createRecord("myRecord", null, null, false);
+ avroSchema.setFields(List.of(new Field(STRING_DIM_SV1, org.apache.avro.Schema.create(Type.STRING), null, null),
+ new Field(STRING_DIM_SV2, org.apache.avro.Schema.create(Type.STRING), null, null),
+ new Field(INT_DIM_SV1, org.apache.avro.Schema.create(Type.INT), null, null),
+ new Field(LONG_DIM_SV1, org.apache.avro.Schema.create(Type.LONG), null, null),
+ new Field(DOUBLE_DIM_SV1, org.apache.avro.Schema.create(Type.DOUBLE), null, null),
+ new Field(BOOLEAN_DIM_SV1, org.apache.avro.Schema.create(Type.BOOLEAN), null, null)));
+
+ // Create Avro files
+ List<File> ret = new ArrayList<>();
+ for (int segmentId = 0; segmentId < NUM_SEGMENTS; segmentId++) {
+ File avroFile = new File(_tempDir, "data_" + segmentId + ".avro");
+ try (DataFileWriter<GenericData.Record> fileWriter = new DataFileWriter<>(new GenericDatumWriter<>(avroSchema))) {
+ fileWriter.create(avroSchema, avroFile);
+
+ int randBound = NUM_DOCS_PER_SEGMENT / 2;
+ Random random = new Random(0);
+ for (int docId = 0; docId < NUM_DOCS_PER_SEGMENT; docId++) {
+ GenericData.Record record = new GenericData.Record(avroSchema);
+ record.put(STRING_DIM_SV1, "test query killing");
+ record.put(STRING_DIM_SV2, "test query killing" + docId);
+ record.put(INT_DIM_SV1, random.nextInt(randBound));
+ record.put(LONG_DIM_SV1, random.nextLong());
+ record.put(DOUBLE_DIM_SV1, random.nextDouble());
+ record.put(BOOLEAN_DIM_SV1, true);
+ fileWriter.append(record);
+ }
+ ret.add(avroFile);
+ }
+ }
+ return ret;
+ }
+}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/CpuBasedBrokerQueryKillingIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/CpuBasedBrokerQueryKillingIntegrationTest.java
new file mode 100644
index 00000000000..8a633505bb4
--- /dev/null
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/CpuBasedBrokerQueryKillingIntegrationTest.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.pinot.core.accounting.ResourceUsageAccountantFactory;
+import org.apache.pinot.spi.env.PinotConfiguration;
+import org.apache.pinot.spi.exception.QueryErrorCode;
+import org.apache.pinot.spi.utils.CommonConstants;
+import org.apache.pinot.spi.utils.CommonConstants.Accounting;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+
+public class CpuBasedBrokerQueryKillingIntegrationTest extends BaseQueryKillingIntegrationTest {
+
+ @Override
+ protected void overrideBrokerConf(PinotConfiguration brokerConf) {
+ super.overrideBrokerConf(brokerConf);
+
+ String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
+ brokerConf.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME, ResourceUsageAccountantFactory.class.getName());
+ brokerConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_CPU_SAMPLING, true);
+ brokerConf.setProperty(prefix + Accounting.CONFIG_OF_CPU_TIME_BASED_KILLING_ENABLED, true);
+ brokerConf.setProperty(prefix + Accounting.CONFIG_OF_CPU_TIME_BASED_KILLING_THRESHOLD_MS, 500);
+ brokerConf.setProperty(prefix + Accounting.CONFIG_OF_CRITICAL_LEVEL_HEAP_USAGE_RATIO, 1.1f);
+ brokerConf.setProperty(prefix + Accounting.CONFIG_OF_PANIC_LEVEL_HEAP_USAGE_RATIO, 1.1f);
+ }
+
+ @Test
+ public void testCpuTimeKill()
+ throws Exception {
+ // Trigger broker side CPU kill with a large streaming query
+ // Do not run other queries because they can cause OOM on server first
+ setUseMultiStageQueryEngine(true);
+ verifyCpuTimeKill(LARGE_SELECT_STAR_QUERY, postQuery(LARGE_SELECT_STAR_QUERY));
+ }
+
+ @Test
+ public void testCpuTimeKillMultipleQueries()
+ throws Exception {
+ // Trigger broker side CPU kill with a large streaming query
+ // Do not run other queries because they can cause OOM on server first
+ setUseMultiStageQueryEngine(true);
+ AtomicReference<JsonNode> queryResponse1 = new AtomicReference<>();
+ AtomicReference<JsonNode> queryResponse2 = new AtomicReference<>();
+ AtomicReference<JsonNode> queryResponse3 = new AtomicReference<>();
+ CountDownLatch countDownLatch = new CountDownLatch(3);
+ ExecutorService executor = Executors.newFixedThreadPool(3);
+ executor.submit(() -> {
+ queryResponse1.set(postQuery(LARGE_SELECT_STAR_QUERY));
+ countDownLatch.countDown();
+ return null;
+ });
+ executor.submit(() -> {
+ queryResponse2.set(postQuery(AGGREGATE_QUERY));
+ countDownLatch.countDown();
+ return null;
+ });
+ executor.submit(() -> {
+ queryResponse3.set(postQuery(SELECT_STAR_QUERY));
+ countDownLatch.countDown();
+ return null;
+ });
+ executor.shutdown();
+ countDownLatch.await();
+
+ JsonNode response = queryResponse1.get();
+ verifyCpuTimeKill(LARGE_SELECT_STAR_QUERY, response);
+
+ response = queryResponse2.get();
+ JsonNode exceptionsNode = response.get("exceptions");
+ assertNotNull(exceptionsNode, "Missing exceptions for query: " + AGGREGATE_QUERY);
+ assertTrue(exceptionsNode.isEmpty(),
+ "Expected no exceptions for query: " + AGGREGATE_QUERY + ", but got: " + exceptionsNode);
+
+ response = queryResponse3.get();
+ exceptionsNode = response.get("exceptions");
+ assertNotNull(exceptionsNode, "Missing exceptions for query: " + SELECT_STAR_QUERY);
+ assertTrue(exceptionsNode.isEmpty(),
+ "Expected no exceptions for query: " + SELECT_STAR_QUERY + ", but got: " + exceptionsNode);
+ }
+
+ private void verifyCpuTimeKill(String query, JsonNode response) {
+ JsonNode exceptionsNode = response.get("exceptions");
+ assertNotNull(exceptionsNode, "Missing exceptions for query: " + query);
+ assertEquals(exceptionsNode.size(), 1, "Expected 1 exception for query: " + query + ", but got: " + exceptionsNode);
+ JsonNode exceptionNode = exceptionsNode.get(0);
+ JsonNode errorCodeNode = exceptionNode.get("errorCode");
+ assertNotNull(errorCodeNode, "Missing errorCode from exception: " + exceptionNode);
+ int errorCode = errorCodeNode.asInt();
+ assertEquals(errorCode, QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED.getId(),
+ "Unexpected error code: " + errorCode + " from exception: " + exceptionNode);
+ JsonNode messageNode = exceptionNode.get("message");
+ assertNotNull(messageNode, "Missing message from exception: " + exceptionNode);
+ String message = messageNode.asText();
+ assertTrue(message.contains("CPU time based killed on BROKER"),
+ "Unexpected exception message: " + message + " from exception: " + exceptionNode);
+ }
+}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/CpuBasedServerQueryKillingIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/CpuBasedServerQueryKillingIntegrationTest.java
new file mode 100644
index 00000000000..17b51bd77a1
--- /dev/null
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/CpuBasedServerQueryKillingIntegrationTest.java
@@ -0,0 +1,120 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.pinot.core.accounting.ResourceUsageAccountantFactory;
+import org.apache.pinot.spi.env.PinotConfiguration;
+import org.apache.pinot.spi.exception.QueryErrorCode;
+import org.apache.pinot.spi.utils.CommonConstants;
+import org.apache.pinot.spi.utils.CommonConstants.Accounting;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+
+public class CpuBasedServerQueryKillingIntegrationTest extends BaseQueryKillingIntegrationTest {
+
+ @Override
+ protected void overrideServerConf(PinotConfiguration serverConf) {
+ super.overrideServerConf(serverConf);
+
+ String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
+ serverConf.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME, ResourceUsageAccountantFactory.class.getName());
+ serverConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_CPU_SAMPLING, true);
+ serverConf.setProperty(prefix + Accounting.CONFIG_OF_CPU_TIME_BASED_KILLING_ENABLED, true);
+ serverConf.setProperty(prefix + Accounting.CONFIG_OF_CPU_TIME_BASED_KILLING_THRESHOLD_MS, 500);
+ serverConf.setProperty(prefix + Accounting.CONFIG_OF_CRITICAL_LEVEL_HEAP_USAGE_RATIO, 1.1f);
+ serverConf.setProperty(prefix + Accounting.CONFIG_OF_PANIC_LEVEL_HEAP_USAGE_RATIO, 1.1f);
+ }
+
+ @Test(dataProvider = "expensiveQueries")
+ public void testCpuTimeKill(String query)
+ throws Exception {
+ verifyCpuTimeKill(query, postQuery(query));
+ setUseMultiStageQueryEngine(true);
+ verifyCpuTimeKill(query, postQuery(query));
+ }
+
+ @Test(dataProvider = "useBothQueryEngines")
+ public void testCpuTimeKillMultipleQueries(boolean useMultiStageQueryEngine)
+ throws Exception {
+ setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+ AtomicReference<JsonNode> queryResponse1 = new AtomicReference<>();
+ AtomicReference<JsonNode> queryResponse2 = new AtomicReference<>();
+ AtomicReference<JsonNode> queryResponse3 = new AtomicReference<>();
+ CountDownLatch countDownLatch = new CountDownLatch(3);
+ ExecutorService executor = Executors.newFixedThreadPool(3);
+ executor.submit(() -> {
+ queryResponse1.set(postQuery(LARGE_DISTINCT_QUERY));
+ countDownLatch.countDown();
+ return null;
+ });
+ executor.submit(() -> {
+ queryResponse2.set(postQuery(AGGREGATE_QUERY));
+ countDownLatch.countDown();
+ return null;
+ });
+ executor.submit(() -> {
+ queryResponse3.set(postQuery(SELECT_STAR_QUERY));
+ countDownLatch.countDown();
+ return null;
+ });
+ executor.shutdown();
+ countDownLatch.await();
+
+ JsonNode response = queryResponse1.get();
+ verifyCpuTimeKill(LARGE_DISTINCT_QUERY, response);
+
+ response = queryResponse2.get();
+ JsonNode exceptionsNode = response.get("exceptions");
+ assertNotNull(exceptionsNode, "Missing exceptions for query: " + AGGREGATE_QUERY);
+ assertTrue(exceptionsNode.isEmpty(),
+ "Expected no exceptions for query: " + AGGREGATE_QUERY + ", but got: " + exceptionsNode);
+
+ response = queryResponse3.get();
+ exceptionsNode = response.get("exceptions");
+ assertNotNull(exceptionsNode, "Missing exceptions for query: " + SELECT_STAR_QUERY);
+ assertTrue(exceptionsNode.isEmpty(),
+ "Expected no exceptions for query: " + SELECT_STAR_QUERY + ", but got: " + exceptionsNode);
+ }
+
+ private void verifyCpuTimeKill(String query, JsonNode response) {
+ JsonNode exceptionsNode = response.get("exceptions");
+ assertNotNull(exceptionsNode, "Missing exceptions for query: " + query);
+ assertEquals(exceptionsNode.size(), 1, "Expected 1 exception for query: " + query + ", but got: " + exceptionsNode);
+ JsonNode exceptionNode = exceptionsNode.get(0);
+ JsonNode errorCodeNode = exceptionNode.get("errorCode");
+ assertNotNull(errorCodeNode, "Missing errorCode from exception: " + exceptionNode);
+ int errorCode = errorCodeNode.asInt();
+ assertEquals(errorCode, QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED.getId(),
+ "Unexpected error code: " + errorCode + " from exception: " + exceptionNode);
+ JsonNode messageNode = exceptionNode.get("message");
+ assertNotNull(messageNode, "Missing message from exception: " + exceptionNode);
+ String message = messageNode.asText();
+ assertTrue(message.contains("CPU time based killed on SERVER"),
+ "Unexpected exception message: " + message + " from exception: " + exceptionNode);
+ }
+}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MemoryBasedServerQueryKillingIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MemoryBasedServerQueryKillingIntegrationTest.java
new file mode 100644
index 00000000000..eb99471023c
--- /dev/null
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MemoryBasedServerQueryKillingIntegrationTest.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.pinot.core.accounting.ResourceUsageAccountantFactory;
+import org.apache.pinot.spi.env.PinotConfiguration;
+import org.apache.pinot.spi.exception.QueryErrorCode;
+import org.apache.pinot.spi.utils.CommonConstants;
+import org.apache.pinot.spi.utils.CommonConstants.Accounting;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+
+public class MemoryBasedServerQueryKillingIntegrationTest extends BaseQueryKillingIntegrationTest {
+
+ @Override
+ protected void overrideServerConf(PinotConfiguration serverConf) {
+ super.overrideServerConf(serverConf);
+
+ String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
+ serverConf.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME, ResourceUsageAccountantFactory.class.getName());
+ serverConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
+ serverConf.setProperty(prefix + Accounting.CONFIG_OF_OOM_PROTECTION_KILLING_QUERY, true);
+ serverConf.setProperty(prefix + Accounting.CONFIG_OF_ALARMING_LEVEL_HEAP_USAGE_RATIO, 0f);
+ serverConf.setProperty(prefix + Accounting.CONFIG_OF_CRITICAL_LEVEL_HEAP_USAGE_RATIO, 0.15f);
+ }
+
+ @Test(dataProvider = "expensiveQueries")
+ public void testOomKill(String query)
+ throws Exception {
+ verifyOomKill(query, postQuery(query));
+ setUseMultiStageQueryEngine(true);
+ verifyOomKill(query, postQuery(query));
+ }
+
+ @Test(dataProvider = "useBothQueryEngines")
+ public void testOomKillMultipleQueries(boolean useMultiStageQueryEngine)
+ throws Exception {
+ setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+ AtomicReference<JsonNode> queryResponse1 = new AtomicReference<>();
+ AtomicReference<JsonNode> queryResponse2 = new AtomicReference<>();
+ AtomicReference<JsonNode> queryResponse3 = new AtomicReference<>();
+ CountDownLatch countDownLatch = new CountDownLatch(3);
+ ExecutorService executor = Executors.newFixedThreadPool(3);
+ executor.submit(() -> {
+ queryResponse1.set(postQuery(LARGE_DISTINCT_QUERY));
+ countDownLatch.countDown();
+ return null;
+ });
+ executor.submit(() -> {
+ queryResponse2.set(postQuery(AGGREGATE_QUERY));
+ countDownLatch.countDown();
+ return null;
+ });
+ executor.submit(() -> {
+ queryResponse3.set(postQuery(SELECT_STAR_QUERY));
+ countDownLatch.countDown();
+ return null;
+ });
+ executor.shutdown();
+ countDownLatch.await();
+
+ JsonNode response = queryResponse1.get();
+ verifyOomKill(LARGE_DISTINCT_QUERY, response);
+
+ response = queryResponse2.get();
+ JsonNode exceptionsNode = response.get("exceptions");
+ assertNotNull(exceptionsNode, "Missing exceptions for query: " + AGGREGATE_QUERY);
+ assertTrue(exceptionsNode.isEmpty(),
+ "Expected no exceptions for query: " + AGGREGATE_QUERY + ", but got: " + exceptionsNode);
+
+ response = queryResponse3.get();
+ exceptionsNode = response.get("exceptions");
+ assertNotNull(exceptionsNode, "Missing exceptions for query: " + SELECT_STAR_QUERY);
+ assertTrue(exceptionsNode.isEmpty(),
+ "Expected no exceptions for query: " + SELECT_STAR_QUERY + ", but got: " + exceptionsNode);
+ }
+
+ private void verifyOomKill(String query, JsonNode response) {
+ JsonNode exceptionsNode = response.get("exceptions");
+ assertNotNull(exceptionsNode, "Missing exceptions for query: " + query);
+ assertEquals(exceptionsNode.size(), 1, "Expected 1 exception for query: " + query + ", but got: " + exceptionsNode);
+ JsonNode exceptionNode = exceptionsNode.get(0);
+ JsonNode errorCodeNode = exceptionNode.get("errorCode");
+ assertNotNull(errorCodeNode, "Missing errorCode from exception: " + exceptionNode);
+ int errorCode = errorCodeNode.asInt();
+ assertEquals(errorCode, QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED.getId(),
+ "Unexpected error code: " + errorCode + " from exception: " + exceptionNode);
+ JsonNode messageNode = exceptionNode.get("message");
+ assertNotNull(messageNode, "Missing message from exception: " + exceptionNode);
+ String message = messageNode.asText();
+ assertTrue(message.contains("OOM killed on SERVER"),
+ "Unexpected exception message: " + message + " from exception: " + exceptionNode);
+ }
+}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiNodesOfflineClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiNodesOfflineClusterIntegrationTest.java
index 851e3de140c..fc269a0c648 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiNodesOfflineClusterIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiNodesOfflineClusterIntegrationTest.java
@@ -24,13 +24,16 @@ import java.util.Map;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.IdealState;
import org.apache.pinot.broker.broker.helix.BaseBrokerStarter;
+import org.apache.pinot.core.accounting.ResourceUsageAccountantFactory;
import org.apache.pinot.server.starter.helix.BaseServerStarter;
import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.exception.QueryErrorCode;
+import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.spi.utils.CommonConstants.Broker;
import org.apache.pinot.spi.utils.CommonConstants.Broker.FailureDetector;
import org.apache.pinot.spi.utils.CommonConstants.Helix;
import org.apache.pinot.spi.utils.CommonConstants.Helix.StateModel.BrokerResourceStateModel;
+import org.apache.pinot.spi.utils.CommonConstants.Server;
import org.apache.pinot.util.TestUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -68,8 +71,32 @@ public class MultiNodesOfflineClusterIntegrationTest extends OfflineClusterInteg
@Override
protected void overrideBrokerConf(PinotConfiguration brokerConf) {
super.overrideBrokerConf(brokerConf);
+
brokerConf.setProperty(FailureDetector.CONFIG_OF_TYPE, FailureDetector.Type.CONNECTION.name());
brokerConf.setProperty(Broker.CONFIG_OF_USE_LEAF_SERVER_FOR_INTERMEDIATE_STAGE, true);
+
+ // Enable thread CPU/memory tracking but not killing queries
+ brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_CPU_TIME_MEASUREMENT, true);
+ brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
+ String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
+ brokerConf.setProperty(prefix + CommonConstants.Accounting.CONFIG_OF_FACTORY_NAME,
+ ResourceUsageAccountantFactory.class.getName());
+ brokerConf.setProperty(prefix + CommonConstants.Accounting.CONFIG_OF_ENABLE_THREAD_CPU_SAMPLING, true);
+ brokerConf.setProperty(prefix + CommonConstants.Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
+ }
+
+ @Override
+ protected void overrideServerConf(PinotConfiguration serverConf) {
+ super.overrideServerConf(serverConf);
+
+ // Enable thread CPU/memory tracking but not killing queries
+ serverConf.setProperty(Server.CONFIG_OF_ENABLE_THREAD_CPU_TIME_MEASUREMENT, true);
+ serverConf.setProperty(Server.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
+ String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
+ serverConf.setProperty(prefix + CommonConstants.Accounting.CONFIG_OF_FACTORY_NAME,
+ ResourceUsageAccountantFactory.class.getName());
+ serverConf.setProperty(prefix + CommonConstants.Accounting.CONFIG_OF_ENABLE_THREAD_CPU_SAMPLING, true);
+ serverConf.setProperty(prefix + CommonConstants.Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
}
@Test
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OOMProtectionEnabledIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OOMProtectionEnabledIntegrationTest.java
deleted file mode 100644
index cf88aa41429..00000000000
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OOMProtectionEnabledIntegrationTest.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.integration.tests;
-
-import java.io.File;
-import java.util.List;
-import org.apache.pinot.core.accounting.PerQueryCPUMemAccountantFactory;
-import org.apache.pinot.spi.config.table.TableConfig;
-import org.apache.pinot.spi.data.Schema;
-import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.utils.CommonConstants;
-import org.apache.pinot.spi.utils.CommonConstants.Accounting;
-import org.apache.pinot.spi.utils.CommonConstants.Broker;
-import org.apache.pinot.spi.utils.CommonConstants.Server;
-import org.apache.pinot.util.TestUtils;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-
-public class OOMProtectionEnabledIntegrationTest extends BaseClusterIntegrationTestSet {
- private static final int NUM_OFFLINE_SEGMENTS = 8;
- private static final int NUM_REALTIME_SEGMENTS = 6;
-
- @Override
- protected void overrideBrokerConf(PinotConfiguration configuration) {
- configuration.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
-
- String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
- configuration.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
- configuration.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME,
- PerQueryCPUMemAccountantFactory.class.getName());
- }
-
- @Override
- protected void overrideServerConf(PinotConfiguration configuration) {
- configuration.setProperty(Server.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
-
- String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
- configuration.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
- configuration.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME,
- PerQueryCPUMemAccountantFactory.class.getName());
- }
-
- @BeforeClass
- public void setUp()
- throws Exception {
- TestUtils.ensureDirectoriesExistAndEmpty(_tempDir, _segmentDir, _tarDir);
-
- // Start Zk, Kafka and Pinot
- startZk();
- startController();
- startBroker();
- startServer();
- startKafka();
-
- List<File> avroFiles = getAllAvroFiles();
- List<File> offlineAvroFiles = getOfflineAvroFiles(avroFiles, NUM_OFFLINE_SEGMENTS);
- List<File> realtimeAvroFiles = getRealtimeAvroFiles(avroFiles, NUM_REALTIME_SEGMENTS);
-
- // Create and upload the schema and table config
- Schema schema = createSchema();
- addSchema(schema);
- TableConfig offlineTableConfig = createOfflineTableConfig();
- addTableConfig(offlineTableConfig);
- addTableConfig(createRealtimeTableConfig(realtimeAvroFiles.get(0)));
-
- // Create and upload segments
- ClusterIntegrationTestUtils.buildSegmentsFromAvro(offlineAvroFiles, offlineTableConfig, schema, 0, _segmentDir,
- _tarDir);
- uploadSegments(getTableName(), _tarDir);
-
- // Push data into Kafka
- pushAvroIntoKafka(realtimeAvroFiles);
-
- // Set up the H2 connection
- setUpH2Connection(avroFiles);
-
- // Initialize the query generator
- setUpQueryGenerator(avroFiles);
-
- // Wait for all documents loaded
- waitForAllDocsLoaded(100_000L);
- }
-
- @Test(dataProvider = "useBothQueryEngines")
- public void testHardcodedQueries(boolean useMultiStageEngine)
- throws Exception {
- setUseMultiStageQueryEngine(useMultiStageEngine);
- super.testHardcodedQueries();
- }
-}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterCpuBasedServerQueryKillingTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterCpuBasedServerQueryKillingTest.java
deleted file mode 100644
index dffffcda82b..00000000000
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterCpuBasedServerQueryKillingTest.java
+++ /dev/null
@@ -1,239 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.integration.tests;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.collect.ImmutableList;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.IntStream;
-import org.apache.avro.file.DataFileWriter;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericDatumWriter;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.pinot.core.accounting.PerQueryCPUMemAccountantFactory;
-import org.apache.pinot.spi.config.table.TableConfig;
-import org.apache.pinot.spi.config.table.TableType;
-import org.apache.pinot.spi.data.FieldSpec;
-import org.apache.pinot.spi.data.Schema;
-import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.utils.CommonConstants;
-import org.apache.pinot.spi.utils.CommonConstants.Accounting;
-import org.apache.pinot.spi.utils.CommonConstants.Broker;
-import org.apache.pinot.spi.utils.CommonConstants.Server;
-import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
-import org.apache.pinot.util.TestUtils;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertTrue;
-
-
-/**
- * Integration test for heap size based server query killing, this works only for xmx4G
- * <p>
- * Query killing isn't currently supported in the v2 multi-stage query engine so these tests only run on the v1 engine.
- */
-public class OfflineClusterCpuBasedServerQueryKillingTest extends BaseClusterIntegrationTestSet {
- public static final String STRING_DIM_SV1 = "stringDimSV1";
- public static final String STRING_DIM_SV2 = "stringDimSV2";
- public static final String INT_DIM_SV1 = "intDimSV1";
- public static final String LONG_DIM_SV1 = "longDimSV1";
- public static final String DOUBLE_DIM_SV1 = "doubleDimSV1";
- public static final String BOOLEAN_DIM_SV1 = "booleanDimSV1";
- private static final String OOM_QUERY =
- "SELECT PERCENTILETDigest(doubleDimSV1, 50) AS digest, intDimSV1 FROM mytable GROUP BY intDimSV1"
- + " ORDER BY digest LIMIT 15000";
-
- private static final String DIGEST_QUERY_1 =
- "SELECT PERCENTILETDigest(doubleDimSV1, 50) AS digest FROM mytable";
- private static final String COUNT_STAR_QUERY =
- "SELECT * FROM mytable LIMIT 5";
-
- private static final ExecutorService EXECUTOR_SERVICE = Executors.newFixedThreadPool(20, r -> {
- Thread thread = new Thread(r);
- thread.setDaemon(false);
- return thread;
- });
-
- @BeforeClass
- public void setUp()
- throws Exception {
- TestUtils.ensureDirectoriesExistAndEmpty(_tempDir, _segmentDir, _tarDir);
-
- // Start the Pinot cluster
- startZk();
- startController();
- startBroker();
- startServer();
-
- // Create and upload the schema and table config
- Schema schema = new Schema.SchemaBuilder().setSchemaName(DEFAULT_SCHEMA_NAME)
- .addSingleValueDimension(STRING_DIM_SV1, FieldSpec.DataType.STRING)
- .addSingleValueDimension(STRING_DIM_SV2, FieldSpec.DataType.STRING)
- .addSingleValueDimension(INT_DIM_SV1, FieldSpec.DataType.INT)
- .addSingleValueDimension(LONG_DIM_SV1, FieldSpec.DataType.LONG)
- .addSingleValueDimension(DOUBLE_DIM_SV1, FieldSpec.DataType.DOUBLE)
- .addSingleValueDimension(BOOLEAN_DIM_SV1, FieldSpec.DataType.BOOLEAN)
- .build();
- addSchema(schema);
- TableConfig tableConfig = createOfflineTableConfig();
- addTableConfig(tableConfig);
-
- List<File> avroFiles = createAvroFile();
- ClusterIntegrationTestUtils.buildSegmentsFromAvro(avroFiles, tableConfig, schema, 0, _segmentDir, _tarDir);
- uploadSegments(DEFAULT_TABLE_NAME, _tarDir);
-
- //Wait for all documents loaded
- waitForAllDocsLoaded(10_000L);
- }
-
- protected void overrideBrokerConf(PinotConfiguration brokerConf) {
- brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_CPU_TIME_MEASUREMENT, true);
- }
-
- protected void overrideServerConf(PinotConfiguration serverConf) {
- serverConf.setProperty(Server.CONFIG_OF_ENABLE_THREAD_CPU_TIME_MEASUREMENT, true);
-
- String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME, PerQueryCPUMemAccountantFactory.class.getName());
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_CPU_SAMPLING, true);
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_CPU_TIME_BASED_KILLING_ENABLED, true);
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_CPU_TIME_BASED_KILLING_THRESHOLD_MS, 1_000);
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_CRITICAL_LEVEL_HEAP_USAGE_RATIO, 1.1f);
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_PANIC_LEVEL_HEAP_USAGE_RATIO, 1.1f);
- }
-
- protected long getCountStarResult() {
- return 3_000_000;
- }
-
- protected String getTimeColumnName() {
- return null;
- }
-
- protected TableConfig createOfflineTableConfig() {
- return new TableConfigBuilder(TableType.OFFLINE).setTableName(getTableName())
- .setTimeColumnName(getTimeColumnName()).setFieldConfigList(getFieldConfigs()).setNumReplicas(getNumReplicas())
- .setSegmentVersion(getSegmentVersion()).setLoadMode(getLoadMode()).setTaskConfig(getTaskConfig())
- .setBrokerTenant(getBrokerTenant()).setServerTenant(getServerTenant()).setIngestionConfig(getIngestionConfig())
- .setQueryConfig(getQueryConfig()).setNullHandlingEnabled(getNullHandlingEnabled())
- .setSegmentPartitionConfig(getSegmentPartitionConfig())
- .build();
- }
-
- @Test(dataProvider = "useBothQueryEngines")
- public void testDigestTimeoutMultipleQueries(boolean useMultiStageQueryEngine)
- throws Exception {
- setUseMultiStageQueryEngine(useMultiStageQueryEngine);
- notSupportedInV2();
- AtomicReference<JsonNode> queryResponse1 = new AtomicReference<>();
- AtomicReference<JsonNode> queryResponse2 = new AtomicReference<>();
- AtomicReference<JsonNode> queryResponse3 = new AtomicReference<>();
-
- CountDownLatch countDownLatch = new CountDownLatch(3);
-
- EXECUTOR_SERVICE.submit(
- () -> {
- try {
- queryResponse1.set(postQuery(OOM_QUERY));
- countDownLatch.countDown();
- } catch (Exception ignored) {
- }
- }
- );
- EXECUTOR_SERVICE.submit(
- () -> {
- try {
- queryResponse2.set(postQuery(DIGEST_QUERY_1));
- countDownLatch.countDown();
- } catch (Exception ignored) {
- }
- }
- );
- EXECUTOR_SERVICE.submit(
- () -> {
- try {
- queryResponse3.set(postQuery(COUNT_STAR_QUERY));
- countDownLatch.countDown();
- } catch (Exception ignored) {
- }
- }
- );
- countDownLatch.await();
- String exceptions = queryResponse1.get().get("exceptions").toString();
- assertTrue(exceptions.contains("CPU time based killed on SERVER"));
- assertFalse(StringUtils.isEmpty(queryResponse2.get().get("exceptions").toString()));
- assertFalse(StringUtils.isEmpty(queryResponse3.get().get("exceptions").toString()));
- }
-
- private List<File> createAvroFile()
- throws IOException {
-
- // create avro schema
- org.apache.avro.Schema avroSchema = org.apache.avro.Schema.createRecord("myRecord", null, null, false);
- avroSchema.setFields(ImmutableList.of(
- new org.apache.avro.Schema.Field(STRING_DIM_SV1,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.STRING), null, null),
- new org.apache.avro.Schema.Field(STRING_DIM_SV2,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.STRING), null, null),
- new org.apache.avro.Schema.Field(INT_DIM_SV1,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.INT), null, null),
- new org.apache.avro.Schema.Field(LONG_DIM_SV1,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.LONG), null, null),
- new org.apache.avro.Schema.Field(DOUBLE_DIM_SV1,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.DOUBLE), null, null),
- new org.apache.avro.Schema.Field(BOOLEAN_DIM_SV1,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.BOOLEAN), null, null)));
-
- List<File> ret = new ArrayList<>();
- for (int file = 0; file < 3; file++) {
- // create avro file
- File avroFile = new File(_tempDir, "data_" + file + ".avro");
- try (DataFileWriter<GenericData.Record> fileWriter = new DataFileWriter<>(new GenericDatumWriter<>(avroSchema))) {
- fileWriter.create(avroSchema, avroFile);
-
- int numDocs = 1_000_000;
- int randBound = numDocs / 2;
- Random random = new Random(0);
- IntStream randomInt = random.ints(0, 100_000);
- for (int docId = 0; docId < numDocs; docId++) {
- GenericData.Record record = new GenericData.Record(avroSchema);
- record.put(STRING_DIM_SV1, "test query killing");
- record.put(STRING_DIM_SV2, "test query killing");
- record.put(INT_DIM_SV1, random.nextInt(randBound));
- record.put(LONG_DIM_SV1, random.nextLong());
- record.put(DOUBLE_DIM_SV1, random.nextDouble());
- record.put(BOOLEAN_DIM_SV1, true);
- fileWriter.append(record);
- }
- ret.add(avroFile);
- }
- }
- return ret;
- }
-}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedBrokerQueryKillingTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedBrokerQueryKillingTest.java
deleted file mode 100644
index 4b05492afd1..00000000000
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedBrokerQueryKillingTest.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.integration.tests;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.collect.ImmutableList;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.IntStream;
-import org.apache.avro.file.DataFileWriter;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericDatumWriter;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.pinot.core.accounting.PerQueryCPUMemAccountantFactory;
-import org.apache.pinot.spi.config.table.TableConfig;
-import org.apache.pinot.spi.config.table.TableType;
-import org.apache.pinot.spi.data.FieldSpec;
-import org.apache.pinot.spi.data.Schema;
-import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.exception.QueryErrorCode;
-import org.apache.pinot.spi.utils.CommonConstants;
-import org.apache.pinot.spi.utils.CommonConstants.Accounting;
-import org.apache.pinot.spi.utils.CommonConstants.Broker;
-import org.apache.pinot.spi.utils.CommonConstants.Server;
-import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
-import org.apache.pinot.util.TestUtils;
-import org.testng.annotations.BeforeClass;
-
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertTrue;
-
-
-/**
- * Integration test for heap size based broker query killing, this works only for xmx4G
- * TODO: re-enable or remove this after we resolve https://github.com/apache/pinot/issues/11099
- */
-public class OfflineClusterMemBasedBrokerQueryKillingTest extends BaseClusterIntegrationTestSet {
-
- public static final String STRING_DIM_SV1 = "stringDimSV1";
- public static final String STRING_DIM_SV2 = "stringDimSV2";
- public static final String INT_DIM_SV1 = "intDimSV1";
- public static final String LONG_DIM_SV1 = "longDimSV1";
- public static final String DOUBLE_DIM_SV1 = "doubleDimSV1";
- public static final String BOOLEAN_DIM_SV1 = "booleanDimSV1";
- private static final int NUM_SERVERS = 3;
- private static final String OOM_QUERY =
- "SELECT PERCENTILETDigest(doubleDimSV1, 50) AS digest, intDimSV1 FROM mytable WHERE intDimSV1 > 450000 GROUP BY"
- + " intDimSV1 ORDER BY digest LIMIT 15000";
-
- private static final String DIGEST_QUERY_1 =
- "SELECT PERCENTILETDigest(doubleDimSV1, 50) AS digest FROM mytable";
- private static final String COUNT_STAR_QUERY =
- "SELECT * FROM mytable LIMIT 5";
-
- private static final ExecutorService EXECUTOR_SERVICE = Executors.newFixedThreadPool(20, r -> {
- Thread thread = new Thread(r);
- thread.setDaemon(false);
- return thread;
- });
-
- @BeforeClass
- public void setUp()
- throws Exception {
- TestUtils.ensureDirectoriesExistAndEmpty(_tempDir, _segmentDir, _tarDir);
-
- // Start the Pinot cluster
- startZk();
- startController();
- startBroker();
- startServers(NUM_SERVERS);
-
- // Create and upload the schema and table config
- Schema schema = new Schema.SchemaBuilder().setSchemaName(DEFAULT_SCHEMA_NAME)
- .addSingleValueDimension(STRING_DIM_SV1, FieldSpec.DataType.STRING)
- .addSingleValueDimension(STRING_DIM_SV2, FieldSpec.DataType.STRING)
- .addSingleValueDimension(INT_DIM_SV1, FieldSpec.DataType.INT)
- .addSingleValueDimension(LONG_DIM_SV1, FieldSpec.DataType.LONG)
- .addSingleValueDimension(DOUBLE_DIM_SV1, FieldSpec.DataType.DOUBLE)
- .addSingleValueDimension(BOOLEAN_DIM_SV1, FieldSpec.DataType.BOOLEAN)
- .build();
- addSchema(schema);
- TableConfig tableConfig = createOfflineTableConfig();
- addTableConfig(tableConfig);
-
- List<File> avroFiles = createAvroFile();
- ClusterIntegrationTestUtils.buildSegmentsFromAvro(avroFiles, tableConfig, schema, 0, _segmentDir, _tarDir);
- uploadSegments(DEFAULT_TABLE_NAME, _tarDir);
-
- //Wait for all documents loaded
- waitForAllDocsLoaded(10_000L);
- }
-
- protected void overrideBrokerConf(PinotConfiguration brokerConf) {
- brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
-
- String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
- brokerConf.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME, PerQueryCPUMemAccountantFactory.class.getName());
- brokerConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
- brokerConf.setProperty(prefix + Accounting.CONFIG_OF_OOM_PROTECTION_KILLING_QUERY, true);
- brokerConf.setProperty(prefix + Accounting.CONFIG_OF_ALARMING_LEVEL_HEAP_USAGE_RATIO, 0f);
- brokerConf.setProperty(prefix + Accounting.CONFIG_OF_CRITICAL_LEVEL_HEAP_USAGE_RATIO, 0.4f);
- brokerConf.setProperty(prefix + Accounting.CONFIG_OF_PANIC_LEVEL_HEAP_USAGE_RATIO, 1.1f);
- brokerConf.setProperty(prefix + Accounting.CONFIG_OF_MIN_MEMORY_FOOTPRINT_TO_KILL_RATIO, 0.0025f);
- }
-
- @Override
- protected void overrideServerConf(PinotConfiguration serverConf) {
- serverConf.setProperty(Server.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
- }
-
- protected long getCountStarResult() {
- return 3_000_000;
- }
-
- protected String getTimeColumnName() {
- return null;
- }
-
- protected TableConfig createOfflineTableConfig() {
- return new TableConfigBuilder(TableType.OFFLINE).setTableName(getTableName())
- .setTimeColumnName(getTimeColumnName()).setFieldConfigList(getFieldConfigs()).setNumReplicas(getNumReplicas())
- .setSegmentVersion(getSegmentVersion()).setLoadMode(getLoadMode()).setTaskConfig(getTaskConfig())
- .setBrokerTenant(getBrokerTenant()).setServerTenant(getServerTenant()).setIngestionConfig(getIngestionConfig())
- .setQueryConfig(getQueryConfig()).setNullHandlingEnabled(getNullHandlingEnabled())
- .setSegmentPartitionConfig(getSegmentPartitionConfig())
- .build();
- }
-
- public void testDigestOOMMultipleQueries()
- throws Exception {
- AtomicReference<JsonNode> queryResponse1 = new AtomicReference<>();
- AtomicReference<JsonNode> queryResponse2 = new AtomicReference<>();
- AtomicReference<JsonNode> queryResponse3 = new AtomicReference<>();
-
- CountDownLatch countDownLatch = new CountDownLatch(3);
-
- EXECUTOR_SERVICE.submit(
- () -> {
- try {
- queryResponse1.set(postQuery(OOM_QUERY));
- countDownLatch.countDown();
- } catch (Exception ignored) {
- }
- }
- );
- EXECUTOR_SERVICE.submit(
- () -> {
- try {
- queryResponse2.set(postQuery(DIGEST_QUERY_1));
- countDownLatch.countDown();
- } catch (Exception ignored) {
- }
- }
- );
- EXECUTOR_SERVICE.submit(
- () -> {
- try {
- queryResponse3.set(postQuery(COUNT_STAR_QUERY));
- countDownLatch.countDown();
- } catch (Exception ignored) {
- }
- }
- );
- countDownLatch.await();
- String exceptions = queryResponse1.get().get("exceptions").toString();
- assertTrue(exceptions.contains("Interrupted in broker reduce phase"));
- assertTrue(exceptions.contains("\"errorCode\":" + QueryErrorCode.QUERY_CANCELLATION.getId()));
- assertTrue(exceptions.contains("got killed on BROKER"));
- assertFalse(StringUtils.isEmpty(queryResponse2.get().get("exceptions").toString()));
- assertFalse(StringUtils.isEmpty(queryResponse3.get().get("exceptions").toString()));
- }
-
- private List<File> createAvroFile()
- throws IOException {
-
- // create avro schema
- org.apache.avro.Schema avroSchema = org.apache.avro.Schema.createRecord("myRecord", null, null, false);
- avroSchema.setFields(ImmutableList.of(
- new org.apache.avro.Schema.Field(STRING_DIM_SV1,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.STRING), null, null),
- new org.apache.avro.Schema.Field(STRING_DIM_SV2,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.STRING), null, null),
- new org.apache.avro.Schema.Field(INT_DIM_SV1,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.INT), null, null),
- new org.apache.avro.Schema.Field(LONG_DIM_SV1,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.LONG), null, null),
- new org.apache.avro.Schema.Field(DOUBLE_DIM_SV1,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.DOUBLE), null, null),
- new org.apache.avro.Schema.Field(BOOLEAN_DIM_SV1,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.BOOLEAN), null, null)));
-
- List<File> ret = new ArrayList<>();
- for (int file = 0; file < 3; file++) {
- // create avro file
- File avroFile = new File(_tempDir, "data_" + file + ".avro");
- try (DataFileWriter<GenericData.Record> fileWriter = new DataFileWriter<>(new GenericDatumWriter<>(avroSchema))) {
- fileWriter.create(avroSchema, avroFile);
-
- int numDocs = 1_000_000;
- int randBound = numDocs / 2;
- Random random = new Random(0);
- IntStream randomInt = random.ints(0, 100_000);
- for (int docId = 0; docId < numDocs; docId++) {
- GenericData.Record record = new GenericData.Record(avroSchema);
- record.put(STRING_DIM_SV1, "test query killing");
- record.put(STRING_DIM_SV2, "test query killing");
- record.put(INT_DIM_SV1, random.nextInt(randBound));
- record.put(LONG_DIM_SV1, random.nextLong());
- record.put(DOUBLE_DIM_SV1, random.nextDouble());
- record.put(BOOLEAN_DIM_SV1, true);
- fileWriter.append(record);
- }
- ret.add(avroFile);
- }
- }
- return ret;
- }
-}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedServerQueryKillingTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedServerQueryKillingTest.java
deleted file mode 100644
index 311d93361bc..00000000000
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedServerQueryKillingTest.java
+++ /dev/null
@@ -1,331 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.integration.tests;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.atomic.AtomicReference;
-import org.apache.avro.Schema.Field;
-import org.apache.avro.Schema.Type;
-import org.apache.avro.file.DataFileWriter;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericDatumWriter;
-import org.apache.pinot.core.accounting.PerQueryCPUMemAccountantFactory.PerQueryCPUMemResourceUsageAccountant;
-import org.apache.pinot.spi.accounting.QueryResourceTracker;
-import org.apache.pinot.spi.accounting.ThreadAccountantFactory;
-import org.apache.pinot.spi.config.instance.InstanceType;
-import org.apache.pinot.spi.config.table.TableConfig;
-import org.apache.pinot.spi.config.table.TableType;
-import org.apache.pinot.spi.data.FieldSpec;
-import org.apache.pinot.spi.data.Schema;
-import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.exception.QueryErrorCode;
-import org.apache.pinot.spi.utils.CommonConstants;
-import org.apache.pinot.spi.utils.CommonConstants.Accounting;
-import org.apache.pinot.spi.utils.CommonConstants.Broker;
-import org.apache.pinot.spi.utils.CommonConstants.Server;
-import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
-import org.apache.pinot.util.TestUtils;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.DataProvider;
-import org.testng.annotations.Test;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertTrue;
-
-
-/**
- * Integration test for heap size based server query killing, this works only for xmx4G.
- */
-public class OfflineClusterMemBasedServerQueryKillingTest extends BaseClusterIntegrationTestSet {
- private static final String STRING_DIM_SV1 = "stringDimSV1";
- private static final String STRING_DIM_SV2 = "stringDimSV2";
- private static final String INT_DIM_SV1 = "intDimSV1";
- private static final String LONG_DIM_SV1 = "longDimSV1";
- private static final String DOUBLE_DIM_SV1 = "doubleDimSV1";
- private static final String BOOLEAN_DIM_SV1 = "booleanDimSV1";
- private static final int NUM_SEGMENTS = 3;
- private static final int NUM_DOCS_PER_SEGMENT = 3_000_000;
-
- private static final String OOM_QUERY =
- "SELECT DISTINCT stringDimSV2 FROM mytable ORDER BY stringDimSV2 LIMIT 3000000";
-
- private static final String OOM_QUERY_2 =
- "SELECT DISTINCT_COUNT_HLL(intDimSV1, 14), stringDimSV2 FROM mytable GROUP BY 2 ORDER BY 1 LIMIT 3000000";
-
- /// SafeTrim sort-aggregation case, pair-wise combine
- private static final String OOM_QUERY_3 =
- "SET sortAggregateSingleThreadedNumSegmentsThreshold=1; SET sortAggregateLimitThreshold=3000001; "
- + "SELECT DISTINCT_COUNT_HLL(intDimSV1, 14), stringDimSV2 FROM mytable GROUP BY 2 ORDER BY 2 LIMIT 3000000";
-
- /// SafeTrim sort-aggregation case, sequential combine
- private static final String OOM_QUERY_4 =
- "SET sortAggregateSingleThreadedNumSegmentsThreshold=10000; SET sortAggregateLimitThreshold=3000001; "
- + "SELECT DISTINCT_COUNT_HLL(intDimSV1, 14), stringDimSV2 FROM mytable GROUP BY 2 ORDER BY 2 LIMIT 3000000";
-
- private static final String AGGREGATE_QUERY = "SELECT DISTINCT_COUNT_HLL(intDimSV1, 14) FROM mytable";
- private static final String SELECT_STAR_QUERY = "SELECT * FROM mytable LIMIT 5";
- private static final String OOM_SELECT_STAR_QUERY = "SELECT * FROM mytable LIMIT 3000000";
-
- /**
- * Keeps track of metadata of queries that have been terminated due to OOM.
- * This is used to verify that the query was killed and the method used to kill the query.
- */
- public static class TestAccountantFactory implements ThreadAccountantFactory {
-
- @Override
- public TestAccountant init(PinotConfiguration config, String instanceId, InstanceType instanceType) {
- return new TestAccountant(config, instanceId, instanceType);
- }
-
- public static class TestAccountant extends PerQueryCPUMemResourceUsageAccountant {
- private QueryResourceTracker _queryResourceTracker = null;
-
- public TestAccountant(PinotConfiguration config, String instanceId, InstanceType instanceType) {
- super(config, instanceId, instanceType);
- }
-
- @Override
- public void logTerminatedQuery(QueryResourceTracker queryTracker, long totalHeapUsage) {
- super.logTerminatedQuery(queryTracker, totalHeapUsage);
- _queryResourceTracker = queryTracker;
- }
-
- public void reset() {
- _queryResourceTracker = null;
- }
-
- public QueryResourceTracker getQueryResourceTracker() {
- return _queryResourceTracker;
- }
- }
- }
-
- private TestAccountantFactory.TestAccountant _testAccountant;
-
- @BeforeClass
- public void setUp()
- throws Exception {
- TestUtils.ensureDirectoriesExistAndEmpty(_tempDir, _segmentDir, _tarDir);
-
- // Start the Pinot cluster
- startZk();
- startController();
- startBroker();
- startServer();
- _testAccountant =
- (TestAccountantFactory.TestAccountant) _serverStarters.get(0).getServerInstance().getThreadAccountant();
-
- // Create and upload the schema and table config
- Schema schema = new Schema.SchemaBuilder().setSchemaName(DEFAULT_TABLE_NAME)
- .addSingleValueDimension(STRING_DIM_SV1, FieldSpec.DataType.STRING)
- .addSingleValueDimension(STRING_DIM_SV2, FieldSpec.DataType.STRING)
- .addSingleValueDimension(INT_DIM_SV1, FieldSpec.DataType.INT)
- .addSingleValueDimension(LONG_DIM_SV1, FieldSpec.DataType.LONG)
- .addSingleValueDimension(DOUBLE_DIM_SV1, FieldSpec.DataType.DOUBLE)
- .addSingleValueDimension(BOOLEAN_DIM_SV1, FieldSpec.DataType.BOOLEAN)
- .build();
- addSchema(schema);
- TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(DEFAULT_TABLE_NAME).build();
- addTableConfig(tableConfig);
-
- List<File> avroFiles = createAvroFiles();
- ClusterIntegrationTestUtils.buildSegmentsFromAvro(avroFiles, tableConfig, schema, 0, _segmentDir, _tarDir);
- uploadSegments(DEFAULT_TABLE_NAME, _tarDir);
-
- // Wait for all documents loaded
- waitForAllDocsLoaded(600_000L);
- }
-
- @AfterMethod
- public void resetTestAccountant() {
- _testAccountant.reset();
- }
-
- protected void overrideBrokerConf(PinotConfiguration brokerConf) {
- brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
- }
-
- protected void overrideServerConf(PinotConfiguration serverConf) {
- serverConf.setProperty(Server.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
- serverConf.setProperty(Server.CONFIG_OF_QUERY_EXECUTOR_NUM_GROUPS_LIMIT, Integer.MAX_VALUE);
-
- String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME, TestAccountantFactory.class.getName());
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_OOM_PROTECTION_KILLING_QUERY, true);
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_ALARMING_LEVEL_HEAP_USAGE_RATIO, 0f);
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_CRITICAL_LEVEL_HEAP_USAGE_RATIO, 0.15f);
- }
-
- protected long getCountStarResult() {
- return NUM_SEGMENTS * NUM_DOCS_PER_SEGMENT;
- }
-
- @Test
- public void testMemoryAllocationStats()
- throws Exception {
- JsonNode queryResponse = postQuery(SELECT_STAR_QUERY);
- long offlineThreadMemAllocatedBytes = queryResponse.get("offlineThreadMemAllocatedBytes").asLong();
- long offlineResponseSerMemAllocatedBytes = queryResponse.get("offlineResponseSerMemAllocatedBytes").asLong();
- long offlineTotalMemAllocatedBytes = queryResponse.get("offlineTotalMemAllocatedBytes").asLong();
-
- assertTrue(offlineThreadMemAllocatedBytes > 0);
- assertTrue(offlineResponseSerMemAllocatedBytes > 0);
- assertEquals(offlineThreadMemAllocatedBytes + offlineResponseSerMemAllocatedBytes, offlineTotalMemAllocatedBytes);
- }
-
- @DataProvider
- public String[] oomQueries() {
- return new String[]{OOM_QUERY, OOM_QUERY_2, OOM_QUERY_3, OOM_QUERY_4, OOM_SELECT_STAR_QUERY};
- }
-
- @Test(dataProvider = "oomQueries")
- public void testOomSse(String query)
- throws Exception {
- testOom(query);
- setUseMultiStageQueryEngine(true);
- testOom(query);
- }
-
- private void testOom(String query)
- throws Exception {
- JsonNode queryResponse = postQuery(query);
- JsonNode exceptionsNode = queryResponse.get("exceptions");
- assertNotNull(exceptionsNode, "Missing exceptions for query: " + query);
- assertEquals(exceptionsNode.size(), 1, "Expected 1 exception for query: " + query + ", but got: " + exceptionsNode);
- JsonNode exceptionNode = exceptionsNode.get(0);
- JsonNode errorCodeNode = exceptionNode.get("errorCode");
- assertNotNull(errorCodeNode, "Missing errorCode from exception: " + exceptionNode);
- int errorCode = errorCodeNode.asInt();
- assertEquals(errorCode, QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED.getId(),
- "Unexpected error code: " + errorCode + " from exception: " + exceptionNode);
- JsonNode messageNode = exceptionNode.get("message");
- assertNotNull(messageNode, "Missing message from exception: " + exceptionNode);
- String message = messageNode.asText();
- assertTrue(message.contains("OOM killed on SERVER"),
- "Unexpected exception message: " + message + " from exception: " + exceptionNode);
- assertEquals(queryResponse.get("requestId").asLong(),
- _testAccountant.getQueryResourceTracker().getExecutionContext().getRequestId());
- }
-
- @Test(dataProvider = "useBothQueryEngines")
- public void testOomMultipleQueries(boolean useMultiStageQueryEngine)
- throws Exception {
- setUseMultiStageQueryEngine(useMultiStageQueryEngine);
- AtomicReference<JsonNode> queryResponse1 = new AtomicReference<>();
- AtomicReference<JsonNode> queryResponse2 = new AtomicReference<>();
- AtomicReference<JsonNode> queryResponse3 = new AtomicReference<>();
- CountDownLatch countDownLatch = new CountDownLatch(3);
- ExecutorService executor = Executors.newFixedThreadPool(3);
- executor.submit(() -> {
- queryResponse1.set(postQuery(OOM_QUERY));
- countDownLatch.countDown();
- return null;
- });
- executor.submit(() -> {
- queryResponse2.set(postQuery(AGGREGATE_QUERY));
- countDownLatch.countDown();
- return null;
- });
- executor.submit(() -> {
- queryResponse3.set(postQuery(SELECT_STAR_QUERY));
- countDownLatch.countDown();
- return null;
- });
- executor.shutdown();
- countDownLatch.await();
-
- JsonNode queryResponse = queryResponse1.get();
- JsonNode exceptionsNode = queryResponse.get("exceptions");
- assertNotNull(exceptionsNode, "Missing exceptions for query: " + OOM_QUERY);
- assertEquals(exceptionsNode.size(), 1,
- "Expected 1 exception for query: " + OOM_QUERY + ", but got: " + exceptionsNode);
- JsonNode exceptionNode = exceptionsNode.get(0);
- JsonNode errorCodeNode = exceptionNode.get("errorCode");
- assertNotNull(errorCodeNode, "Missing errorCode from exception: " + exceptionNode);
- int errorCode = errorCodeNode.asInt();
- assertEquals(errorCode, QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED.getId(),
- "Unexpected error code: " + errorCode + " from exception: " + exceptionNode);
- JsonNode messageNode = exceptionNode.get("message");
- assertNotNull(messageNode, "Missing message from exception: " + exceptionNode);
- String message = messageNode.asText();
- assertTrue(message.contains("OOM killed on SERVER"),
- "Unexpected exception message: " + message + " from exception: " + exceptionNode);
- assertEquals(queryResponse.get("requestId").asLong(),
- _testAccountant.getQueryResourceTracker().getExecutionContext().getRequestId());
-
- queryResponse = queryResponse2.get();
- exceptionsNode = queryResponse.get("exceptions");
- assertNotNull(exceptionsNode, "Missing exceptions for query: " + AGGREGATE_QUERY);
- assertTrue(exceptionsNode.isEmpty(),
- "Expected no exceptions for query: " + AGGREGATE_QUERY + ", but got: " + exceptionsNode);
-
- queryResponse = queryResponse3.get();
- exceptionsNode = queryResponse.get("exceptions");
- assertNotNull(exceptionsNode, "Missing exceptions for query: " + SELECT_STAR_QUERY);
- assertTrue(exceptionsNode.isEmpty(),
- "Expected no exceptions for query: " + SELECT_STAR_QUERY + ", but got: " + exceptionsNode);
- }
-
- private List<File> createAvroFiles()
- throws IOException {
- // Create Avro schema
- org.apache.avro.Schema avroSchema = org.apache.avro.Schema.createRecord("myRecord", null, null, false);
- avroSchema.setFields(List.of(new Field(STRING_DIM_SV1, org.apache.avro.Schema.create(Type.STRING), null, null),
- new Field(STRING_DIM_SV2, org.apache.avro.Schema.create(Type.STRING), null, null),
- new Field(INT_DIM_SV1, org.apache.avro.Schema.create(Type.INT), null, null),
- new Field(LONG_DIM_SV1, org.apache.avro.Schema.create(Type.LONG), null, null),
- new Field(DOUBLE_DIM_SV1, org.apache.avro.Schema.create(Type.DOUBLE), null, null),
- new Field(BOOLEAN_DIM_SV1, org.apache.avro.Schema.create(Type.BOOLEAN), null, null)));
-
- // Create Avro files
- List<File> ret = new ArrayList<>();
- for (int segmentId = 0; segmentId < NUM_SEGMENTS; segmentId++) {
- File avroFile = new File(_tempDir, "data_" + segmentId + ".avro");
- try (DataFileWriter<GenericData.Record> fileWriter = new DataFileWriter<>(new GenericDatumWriter<>(avroSchema))) {
- fileWriter.create(avroSchema, avroFile);
-
- int randBound = NUM_DOCS_PER_SEGMENT / 2;
- Random random = new Random(0);
- for (int docId = 0; docId < NUM_DOCS_PER_SEGMENT; docId++) {
- GenericData.Record record = new GenericData.Record(avroSchema);
- record.put(STRING_DIM_SV1, "test query killing");
- record.put(STRING_DIM_SV2, "test query killing" + docId);
- record.put(INT_DIM_SV1, random.nextInt(randBound));
- record.put(LONG_DIM_SV1, random.nextLong());
- record.put(DOUBLE_DIM_SV1, random.nextDouble());
- record.put(BOOLEAN_DIM_SV1, true);
- fileWriter.append(record);
- }
- ret.add(avroFile);
- }
- }
- return ret;
- }
-}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedServerResourceUsageAccountantTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedServerResourceUsageAccountantTest.java
deleted file mode 100644
index 631bf4bc029..00000000000
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedServerResourceUsageAccountantTest.java
+++ /dev/null
@@ -1,302 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.integration.tests;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.collect.ImmutableList;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.IntStream;
-import org.apache.avro.file.DataFileWriter;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericDatumWriter;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.log4j.Level;
-import org.apache.log4j.LogManager;
-import org.apache.pinot.core.accounting.QueryResourceAggregator;
-import org.apache.pinot.core.accounting.ResourceUsageAccountantFactory;
-import org.apache.pinot.spi.accounting.ThreadResourceUsageProvider;
-import org.apache.pinot.spi.config.table.TableConfig;
-import org.apache.pinot.spi.config.table.TableType;
-import org.apache.pinot.spi.data.FieldSpec;
-import org.apache.pinot.spi.data.Schema;
-import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.exception.QueryErrorCode;
-import org.apache.pinot.spi.trace.Tracing;
-import org.apache.pinot.spi.utils.CommonConstants;
-import org.apache.pinot.spi.utils.CommonConstants.Accounting;
-import org.apache.pinot.spi.utils.CommonConstants.Broker;
-import org.apache.pinot.spi.utils.CommonConstants.Server;
-import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
-import org.apache.pinot.util.TestUtils;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertTrue;
-
-
-/**
- * Integration test for heap size based server query killing, this works only for xmx4G.
- * <p>
- * Query killing isn't currently supported in the v2 multi-stage query engine so these tests only run on the v1 engine.
- */
-public class OfflineClusterMemBasedServerResourceUsageAccountantTest extends BaseClusterIntegrationTestSet {
- public static final String STRING_DIM_SV1 = "stringDimSV1";
- public static final String STRING_DIM_SV2 = "stringDimSV2";
- public static final String INT_DIM_SV1 = "intDimSV1";
- public static final String LONG_DIM_SV1 = "longDimSV1";
- public static final String DOUBLE_DIM_SV1 = "doubleDimSV1";
- public static final String BOOLEAN_DIM_SV1 = "booleanDimSV1";
- private static final int NUM_DOCS = 3_000_000;
-
- private static final String OOM_QUERY =
- "SELECT PERCENTILETDigest(doubleDimSV1, 50) AS digest, intDimSV1 FROM mytable GROUP BY intDimSV1"
- + " ORDER BY digest LIMIT 30000";
-
- private static final String OOM_QUERY_2 =
- "SELECT stringDimSV2 FROM mytable GROUP BY stringDimSV2"
- + " ORDER BY stringDimSV2 LIMIT 3000000";
-
- private static final String DIGEST_QUERY_1 =
- "SELECT PERCENTILETDigest(doubleDimSV1, 50) AS digest FROM mytable";
- private static final String COUNT_STAR_QUERY =
- "SELECT * FROM mytable LIMIT 5";
- private static final String OOM_QUERY_SELECTION_ONLY = "SELECT * FROM mytable LIMIT 9000000";
-
- private static final ExecutorService EXECUTOR_SERVICE = Executors.newFixedThreadPool(20, r -> {
- Thread thread = new Thread(r);
- thread.setDaemon(false);
- return thread;
- });
-
- @BeforeClass
- public void setUp()
- throws Exception {
- TestUtils.ensureDirectoriesExistAndEmpty(_tempDir, _segmentDir, _tarDir);
-
- // Start the Pinot cluster
- startZk();
- startController();
- startBroker();
- startServer();
-
- // Create and upload the schema and table config
- Schema schema = new Schema.SchemaBuilder().setSchemaName(DEFAULT_SCHEMA_NAME)
- .addSingleValueDimension(STRING_DIM_SV1, FieldSpec.DataType.STRING)
- .addSingleValueDimension(STRING_DIM_SV2, FieldSpec.DataType.STRING)
- .addSingleValueDimension(INT_DIM_SV1, FieldSpec.DataType.INT)
- .addSingleValueDimension(LONG_DIM_SV1, FieldSpec.DataType.LONG)
- .addSingleValueDimension(DOUBLE_DIM_SV1, FieldSpec.DataType.DOUBLE)
- .addSingleValueDimension(BOOLEAN_DIM_SV1, FieldSpec.DataType.BOOLEAN)
- .build();
- addSchema(schema);
- TableConfig tableConfig = createOfflineTableConfig();
- addTableConfig(tableConfig);
-
- List<File> avroFiles = createAvroFile();
- ClusterIntegrationTestUtils.buildSegmentsFromAvro(avroFiles, tableConfig, schema, 0, _segmentDir, _tarDir);
- uploadSegments(DEFAULT_TABLE_NAME, _tarDir);
-
- //Wait for all documents loaded
- waitForAllDocsLoaded(10_000L);
-
- // Setup logging and resource accounting
- LogManager.getLogger(OfflineClusterMemBasedServerQueryKillingTest.class).setLevel(Level.INFO);
- LogManager.getLogger(ResourceUsageAccountantFactory.ResourceUsageAccountant.class)
- .setLevel(Level.INFO);
- LogManager.getLogger(ThreadResourceUsageProvider.class).setLevel(Level.INFO);
- LogManager.getLogger(Tracing.class).setLevel(Level.INFO);
- LogManager.getLogger(QueryResourceAggregator.class).setLevel(Level.INFO);
- }
-
- protected void overrideBrokerConf(PinotConfiguration brokerConf) {
- brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
- }
-
- protected void overrideServerConf(PinotConfiguration serverConf) {
- serverConf.setProperty(Server.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
-
- String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME, ResourceUsageAccountantFactory.class.getName());
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_OOM_PROTECTION_KILLING_QUERY, true);
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_ALARMING_LEVEL_HEAP_USAGE_RATIO, 0f);
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_CRITICAL_LEVEL_HEAP_USAGE_RATIO, 0.15f);
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_WORKLOAD_ENABLE_COST_COLLECTION, true);
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_WORKLOAD_ENABLE_COST_ENFORCEMENT, true);
- }
-
- protected long getCountStarResult() {
- return NUM_DOCS * 3;
- }
-
- protected String getTimeColumnName() {
- return null;
- }
-
- protected TableConfig createOfflineTableConfig() {
- return new TableConfigBuilder(TableType.OFFLINE).setTableName(getTableName())
- .setTimeColumnName(getTimeColumnName()).setFieldConfigList(getFieldConfigs()).setNumReplicas(getNumReplicas())
- .setSegmentVersion(getSegmentVersion()).setLoadMode(getLoadMode()).setTaskConfig(getTaskConfig())
- .setBrokerTenant(getBrokerTenant()).setServerTenant(getServerTenant()).setIngestionConfig(getIngestionConfig())
- .setQueryConfig(getQueryConfig()).setNullHandlingEnabled(getNullHandlingEnabled())
- .setSegmentPartitionConfig(getSegmentPartitionConfig())
- .build();
- }
-
- @Test(dataProvider = "useBothQueryEngines")
- public void testDigestOOM(boolean useMultiStageQueryEngine)
- throws Exception {
- setUseMultiStageQueryEngine(useMultiStageQueryEngine);
- notSupportedInV2();
- JsonNode queryResponse = postQuery(OOM_QUERY);
- String exceptions = queryResponse.get("exceptions").toString();
- assertTrue(exceptions.contains("\"errorCode\":" + QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED.getId()),
- exceptions);
- assertTrue(exceptions.contains("OOM killed on SERVER"), exceptions);
- }
-
- @Test(dataProvider = "useBothQueryEngines")
- public void testSelectionOnlyOOM(boolean useMultiStageQueryEngine)
- throws Exception {
- setUseMultiStageQueryEngine(useMultiStageQueryEngine);
- notSupportedInV2();
- JsonNode queryResponse = postQuery(OOM_QUERY_SELECTION_ONLY);
-
- String exceptions = queryResponse.get("exceptions").toString();
- assertTrue(exceptions.contains("\"errorCode\":" + QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED.getId()),
- exceptions);
- assertTrue(exceptions.contains("OOM killed on SERVER"), exceptions);
- }
-
- @Test(dataProvider = "useBothQueryEngines")
- public void testDigestOOM2(boolean useMultiStageQueryEngine)
- throws Exception {
- setUseMultiStageQueryEngine(useMultiStageQueryEngine);
- notSupportedInV2();
- JsonNode queryResponse = postQuery(OOM_QUERY_2);
- String exceptions = queryResponse.get("exceptions").toString();
- assertTrue(exceptions.contains("\"errorCode\":" + QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED.getId()),
- exceptions);
- assertTrue(exceptions.contains("OOM killed on SERVER"), exceptions);
- }
-
- @Test(dataProvider = "useBothQueryEngines")
- public void testDigestOOMMultipleQueries(boolean useMultiStageQueryEngine)
- throws Exception {
- setUseMultiStageQueryEngine(useMultiStageQueryEngine);
- notSupportedInV2();
- AtomicReference<JsonNode> queryResponse1 = new AtomicReference<>();
- AtomicReference<JsonNode> queryResponse2 = new AtomicReference<>();
- AtomicReference<JsonNode> queryResponse3 = new AtomicReference<>();
-
- CountDownLatch countDownLatch = new CountDownLatch(3);
-
- EXECUTOR_SERVICE.submit(
- () -> {
- try {
- queryResponse1.set(postQuery(OOM_QUERY));
- countDownLatch.countDown();
- } catch (Exception ignored) {
- }
- }
- );
- EXECUTOR_SERVICE.submit(
- () -> {
- try {
- queryResponse2.set(postQuery(DIGEST_QUERY_1));
- countDownLatch.countDown();
- } catch (Exception ignored) {
- }
- }
- );
- EXECUTOR_SERVICE.submit(
- () -> {
- try {
- queryResponse3.set(postQuery(COUNT_STAR_QUERY));
- countDownLatch.countDown();
- } catch (Exception ignored) {
- }
- }
- );
- countDownLatch.await();
- String exceptions = queryResponse1.get().get("exceptions").toString();
- assertTrue(exceptions.contains("\"errorCode\":" + QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED.getId()),
- exceptions);
- assertTrue(exceptions.contains("OOM killed on SERVER"), exceptions);
- assertFalse(StringUtils.isEmpty(queryResponse2.get().get("exceptions").toString()));
- assertFalse(StringUtils.isEmpty(queryResponse3.get().get("exceptions").toString()));
- }
-
- // TODO: Add integation test after workload configs PR is merged.
-
- private List<File> createAvroFile()
- throws IOException {
-
- // create avro schema
- org.apache.avro.Schema avroSchema = org.apache.avro.Schema.createRecord("myRecord", null, null, false);
- avroSchema.setFields(ImmutableList.of(
- new org.apache.avro.Schema.Field(STRING_DIM_SV1,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.STRING), null, null),
- new org.apache.avro.Schema.Field(STRING_DIM_SV2,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.STRING), null, null),
- new org.apache.avro.Schema.Field(INT_DIM_SV1,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.INT), null, null),
- new org.apache.avro.Schema.Field(LONG_DIM_SV1,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.LONG), null, null),
- new org.apache.avro.Schema.Field(DOUBLE_DIM_SV1,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.DOUBLE), null, null),
- new org.apache.avro.Schema.Field(BOOLEAN_DIM_SV1,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.BOOLEAN), null, null)));
-
- List<File> ret = new ArrayList<>();
- for (int file = 0; file < 3; file++) {
- // create avro file
- File avroFile = new File(_tempDir, "data_" + file + ".avro");
- try (DataFileWriter<GenericData.Record> fileWriter = new DataFileWriter<>(new GenericDatumWriter<>(avroSchema))) {
- fileWriter.create(avroSchema, avroFile);
-
- int numDocs = NUM_DOCS;
- int randBound = numDocs / 2;
- Random random = new Random(0);
- IntStream randomInt = random.ints(0, 100_000);
- for (int docId = 0; docId < numDocs; docId++) {
- GenericData.Record record = new GenericData.Record(avroSchema);
- record.put(STRING_DIM_SV1, "test query killing");
- record.put(STRING_DIM_SV2, "test query killing" + docId);
- record.put(INT_DIM_SV1, random.nextInt(randBound));
- record.put(LONG_DIM_SV1, random.nextLong());
- record.put(DOUBLE_DIM_SV1, random.nextDouble());
- record.put(BOOLEAN_DIM_SV1, true);
- fileWriter.append(record);
- }
- ret.add(avroFile);
- }
- }
- return ret;
- }
-}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterResourceUsageAccountantTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterResourceUsageAccountantTest.java
deleted file mode 100644
index 66af8cd3154..00000000000
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterResourceUsageAccountantTest.java
+++ /dev/null
@@ -1,272 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.integration.tests;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.collect.ImmutableList;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.IntStream;
-import org.apache.avro.file.DataFileWriter;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericDatumWriter;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.log4j.Level;
-import org.apache.log4j.LogManager;
-import org.apache.pinot.core.accounting.QueryResourceAggregator;
-import org.apache.pinot.core.accounting.ResourceUsageAccountantFactory;
-import org.apache.pinot.spi.accounting.ThreadResourceUsageProvider;
-import org.apache.pinot.spi.config.table.TableConfig;
-import org.apache.pinot.spi.config.table.TableType;
-import org.apache.pinot.spi.data.FieldSpec;
-import org.apache.pinot.spi.data.Schema;
-import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.exception.QueryErrorCode;
-import org.apache.pinot.spi.trace.Tracing;
-import org.apache.pinot.spi.utils.CommonConstants;
-import org.apache.pinot.spi.utils.CommonConstants.Accounting;
-import org.apache.pinot.spi.utils.CommonConstants.Broker;
-import org.apache.pinot.spi.utils.CommonConstants.Server;
-import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
-import org.apache.pinot.util.TestUtils;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import static org.testng.Assert.assertTrue;
-
-
-public class OfflineClusterResourceUsageAccountantTest extends BaseClusterIntegrationTestSet {
- public static final String STRING_DIM_SV1 = "stringDimSV1";
- public static final String STRING_DIM_SV2 = "stringDimSV2";
- public static final String INT_DIM_SV1 = "intDimSV1";
- public static final String LONG_DIM_SV1 = "longDimSV1";
- public static final String DOUBLE_DIM_SV1 = "doubleDimSV1";
- public static final String BOOLEAN_DIM_SV1 = "booleanDimSV1";
- private static final String OOM_QUERY =
- "SELECT PERCENTILETDigest(doubleDimSV1, 50) AS digest, intDimSV1 FROM mytable GROUP BY intDimSV1"
- + " ORDER BY digest LIMIT 15000";
-
- private static final String DIGEST_QUERY_1 =
- "SELECT PERCENTILETDigest(doubleDimSV1, 50) AS digest FROM mytable";
- private static final String COUNT_STAR_QUERY =
- "SELECT * FROM mytable LIMIT 5";
-
- private static final ExecutorService EXECUTOR_SERVICE = Executors.newFixedThreadPool(20, r -> {
- Thread thread = new Thread(r);
- thread.setDaemon(false);
- return thread;
- });
-
- @BeforeClass
- public void setUp()
- throws Exception {
- TestUtils.ensureDirectoriesExistAndEmpty(_tempDir, _segmentDir, _tarDir);
-
- // Start the Pinot cluster
- startZk();
- startController();
- startBroker();
- startServer();
-
- // Create and upload the schema and table config
- Schema schema = new Schema.SchemaBuilder().setSchemaName(DEFAULT_SCHEMA_NAME)
- .addSingleValueDimension(STRING_DIM_SV1, FieldSpec.DataType.STRING)
- .addSingleValueDimension(STRING_DIM_SV2, FieldSpec.DataType.STRING)
- .addSingleValueDimension(INT_DIM_SV1, FieldSpec.DataType.INT)
- .addSingleValueDimension(LONG_DIM_SV1, FieldSpec.DataType.LONG)
- .addSingleValueDimension(DOUBLE_DIM_SV1, FieldSpec.DataType.DOUBLE)
- .addSingleValueDimension(BOOLEAN_DIM_SV1, FieldSpec.DataType.BOOLEAN)
- .build();
- addSchema(schema);
- TableConfig tableConfig = createOfflineTableConfig();
- addTableConfig(tableConfig);
-
- List<File> avroFiles = createAvroFile();
- ClusterIntegrationTestUtils.buildSegmentsFromAvro(avroFiles, tableConfig, schema, 0, _segmentDir, _tarDir);
- uploadSegments(DEFAULT_TABLE_NAME, _tarDir);
-
- //Wait for all documents loaded
- waitForAllDocsLoaded(10_000L);
-
- // Setup logging and resource accounting
- LogManager.getLogger(OfflineClusterCpuBasedServerQueryKillingTest.class).setLevel(Level.INFO);
- LogManager.getLogger(ResourceUsageAccountantFactory.ResourceUsageAccountant.class)
- .setLevel(Level.INFO);
- LogManager.getLogger(ThreadResourceUsageProvider.class).setLevel(Level.INFO);
- LogManager.getLogger(Tracing.class).setLevel(Level.INFO);
- LogManager.getLogger(QueryResourceAggregator.class).setLevel(Level.INFO);
- }
-
- protected void overrideBrokerConf(PinotConfiguration brokerConf) {
- brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_CPU_TIME_MEASUREMENT, true);
- brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
-
- String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
- brokerConf.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME, ResourceUsageAccountantFactory.class.getName());
- brokerConf.setProperty(prefix + Accounting.CONFIG_OF_ALARMING_LEVEL_HEAP_USAGE_RATIO, 0.2f);
- brokerConf.setProperty(prefix + Accounting.CONFIG_OF_CRITICAL_LEVEL_HEAP_USAGE_RATIO, 1.1f);
- brokerConf.setProperty(prefix + Accounting.CONFIG_OF_PANIC_LEVEL_HEAP_USAGE_RATIO, 1.1f);
- brokerConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
- brokerConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_CPU_SAMPLING, true);
- brokerConf.setProperty(prefix + Accounting.CONFIG_OF_OOM_PROTECTION_KILLING_QUERY, true);
- brokerConf.setProperty(prefix + Accounting.CONFIG_OF_CPU_TIME_BASED_KILLING_ENABLED, true);
- brokerConf.setProperty(prefix + Accounting.CONFIG_OF_WORKLOAD_ENABLE_COST_COLLECTION, true);
- brokerConf.setProperty(prefix + Accounting.CONFIG_OF_WORKLOAD_ENABLE_COST_ENFORCEMENT, true);
- }
-
- protected void overrideServerConf(PinotConfiguration serverConf) {
- serverConf.setProperty(Server.CONFIG_OF_ENABLE_THREAD_CPU_TIME_MEASUREMENT, true);
- serverConf.setProperty(Server.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
-
- String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_ALARMING_LEVEL_HEAP_USAGE_RATIO, 0.2f);
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_CRITICAL_LEVEL_HEAP_USAGE_RATIO, 1.1f);
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME, ResourceUsageAccountantFactory.class.getName());
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_PANIC_LEVEL_HEAP_USAGE_RATIO, 1.1f);
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_CPU_SAMPLING, true);
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_OOM_PROTECTION_KILLING_QUERY, true);
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_CPU_TIME_BASED_KILLING_ENABLED, true);
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_CPU_TIME_BASED_KILLING_THRESHOLD_MS, 1_000);
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_WORKLOAD_ENABLE_COST_COLLECTION, true);
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_WORKLOAD_ENABLE_COST_ENFORCEMENT, true);
- }
-
- protected long getCountStarResult() {
- return 3_000_000;
- }
-
- protected String getTimeColumnName() {
- return null;
- }
-
- protected TableConfig createOfflineTableConfig() {
- return new TableConfigBuilder(TableType.OFFLINE).setTableName(getTableName())
- .setTimeColumnName(getTimeColumnName()).setFieldConfigList(getFieldConfigs()).setNumReplicas(getNumReplicas())
- .setSegmentVersion(getSegmentVersion()).setLoadMode(getLoadMode()).setTaskConfig(getTaskConfig())
- .setBrokerTenant(getBrokerTenant()).setServerTenant(getServerTenant()).setIngestionConfig(getIngestionConfig())
- .setQueryConfig(getQueryConfig()).setNullHandlingEnabled(getNullHandlingEnabled())
- .setSegmentPartitionConfig(getSegmentPartitionConfig())
- .build();
- }
-
- @Test(dataProvider = "useBothQueryEngines")
- public void testDigestTimeoutMultipleQueries(boolean useMultiStageQueryEngine)
- throws Exception {
- setUseMultiStageQueryEngine(useMultiStageQueryEngine);
- notSupportedInV2();
- AtomicReference<JsonNode> queryResponse1 = new AtomicReference<>();
- AtomicReference<JsonNode> queryResponse2 = new AtomicReference<>();
- AtomicReference<JsonNode> queryResponse3 = new AtomicReference<>();
-
- CountDownLatch countDownLatch = new CountDownLatch(3);
-
- EXECUTOR_SERVICE.submit(
- () -> {
- try {
- queryResponse1.set(postQuery(OOM_QUERY));
- countDownLatch.countDown();
- } catch (Exception ignored) {
- }
- }
- );
- EXECUTOR_SERVICE.submit(
- () -> {
- try {
- queryResponse2.set(postQuery(DIGEST_QUERY_1));
- countDownLatch.countDown();
- } catch (Exception ignored) {
- }
- }
- );
- EXECUTOR_SERVICE.submit(
- () -> {
- try {
- queryResponse3.set(postQuery(COUNT_STAR_QUERY));
- countDownLatch.countDown();
- } catch (Exception ignored) {
- }
- }
- );
- Assert.assertTrue(countDownLatch.await(1, TimeUnit.MINUTES));
- String exceptions = queryResponse1.get().get("exceptions").toString();
- assertTrue(exceptions.contains("\"errorCode\":" + QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED.getId()),
- exceptions);
- Assert.assertTrue(exceptions.contains("CPU time based killed on SERVER"), exceptions);
- Assert.assertFalse(StringUtils.isEmpty(queryResponse2.get().get("exceptions").toString()));
- Assert.assertFalse(StringUtils.isEmpty(queryResponse3.get().get("exceptions").toString()));
- }
-
- // TODO: Add integation test after workload configs PR is merged.
-
- private List<File> createAvroFile()
- throws IOException {
-
- // create avro schema
- org.apache.avro.Schema avroSchema = org.apache.avro.Schema.createRecord("myRecord", null, null, false);
- avroSchema.setFields(ImmutableList.of(
- new org.apache.avro.Schema.Field(STRING_DIM_SV1,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.STRING), null, null),
- new org.apache.avro.Schema.Field(STRING_DIM_SV2,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.STRING), null, null),
- new org.apache.avro.Schema.Field(INT_DIM_SV1,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.INT), null, null),
- new org.apache.avro.Schema.Field(LONG_DIM_SV1,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.LONG), null, null),
- new org.apache.avro.Schema.Field(DOUBLE_DIM_SV1,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.DOUBLE), null, null),
- new org.apache.avro.Schema.Field(BOOLEAN_DIM_SV1,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.BOOLEAN), null, null)));
-
- List<File> ret = new ArrayList<>();
- for (int file = 0; file < 3; file++) {
- // create avro file
- File avroFile = new File(_tempDir, "data_" + file + ".avro");
- try (DataFileWriter<GenericData.Record> fileWriter = new DataFileWriter<>(new GenericDatumWriter<>(avroSchema))) {
- fileWriter.create(avroSchema, avroFile);
-
- int numDocs = 1_000_000;
- int randBound = numDocs / 2;
- Random random = new Random(0);
- IntStream randomInt = random.ints(0, 100_000);
- for (int docId = 0; docId < numDocs; docId++) {
- GenericData.Record record = new GenericData.Record(avroSchema);
- record.put(STRING_DIM_SV1, "test query killing");
- record.put(STRING_DIM_SV2, "test query killing");
- record.put(INT_DIM_SV1, random.nextInt(randBound));
- record.put(LONG_DIM_SV1, random.nextLong());
- record.put(DOUBLE_DIM_SV1, random.nextDouble());
- record.put(BOOLEAN_DIM_SV1, true);
- fileWriter.append(record);
- }
- ret.add(avroFile);
- }
- }
- return ret;
- }
-}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineGRPCServerIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineGRPCServerIntegrationTest.java
index 2e7ade25b91..8334d10a007 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineGRPCServerIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineGRPCServerIntegrationTest.java
@@ -37,6 +37,7 @@ import org.apache.pinot.common.response.broker.BrokerResponseNative;
import org.apache.pinot.common.utils.DataSchema;
import org.apache.pinot.common.utils.grpc.ServerGrpcQueryClient;
import org.apache.pinot.common.utils.grpc.ServerGrpcRequestBuilder;
+import org.apache.pinot.core.accounting.ResourceUsageAccountantFactory;
import org.apache.pinot.core.query.reduce.DataTableReducer;
import org.apache.pinot.core.query.reduce.DataTableReducerContext;
import org.apache.pinot.core.query.reduce.ResultReducerFactory;
@@ -45,8 +46,11 @@ import org.apache.pinot.core.query.request.context.utils.QueryContextConverterUt
import org.apache.pinot.core.transport.ServerRoutingInstance;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.query.QueryThreadContext;
import org.apache.pinot.spi.utils.CommonConstants;
+import org.apache.pinot.spi.utils.CommonConstants.Accounting;
+import org.apache.pinot.spi.utils.CommonConstants.Broker;
import org.apache.pinot.sql.parsers.CalciteSqlCompiler;
import org.apache.pinot.util.TestUtils;
import org.testng.annotations.AfterClass;
@@ -98,6 +102,32 @@ public class OfflineGRPCServerIntegrationTest extends BaseClusterIntegrationTest
waitForAllDocsLoaded(600_000L);
}
+ @Override
+ protected void overrideBrokerConf(PinotConfiguration brokerConf) {
+ super.overrideBrokerConf(brokerConf);
+
+ // Enable thread CPU/memory tracking but not killing queries
+ brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_CPU_TIME_MEASUREMENT, true);
+ brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
+ String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
+ brokerConf.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME, ResourceUsageAccountantFactory.class.getName());
+ brokerConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_CPU_SAMPLING, true);
+ brokerConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
+ }
+
+ @Override
+ protected void overrideServerConf(PinotConfiguration serverConf) {
+ super.overrideServerConf(serverConf);
+
+ // Enable thread CPU/memory tracking but not killing queries
+ serverConf.setProperty(CommonConstants.Server.CONFIG_OF_ENABLE_THREAD_CPU_TIME_MEASUREMENT, true);
+ serverConf.setProperty(CommonConstants.Server.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
+ String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
+ serverConf.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME, ResourceUsageAccountantFactory.class.getName());
+ serverConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_CPU_SAMPLING, true);
+ serverConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
+ }
+
public ServerGrpcQueryClient getGrpcQueryClient() {
return new ServerGrpcQueryClient("localhost", getServerGrpcPort());
}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineGRPCServerMultiStageIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineGRPCServerMultiStageIntegrationTest.java
index 97863881568..acbbb9b39cd 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineGRPCServerMultiStageIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineGRPCServerMultiStageIntegrationTest.java
@@ -18,12 +18,13 @@
*/
package org.apache.pinot.integration.tests;
-import org.testng.annotations.BeforeTest;
+import org.testng.annotations.BeforeMethod;
public class OfflineGRPCServerMultiStageIntegrationTest extends OfflineGRPCServerIntegrationTest {
- @BeforeTest
- void enableMultiStage() {
+
+ @BeforeMethod
+ public void enableMultiStage() {
setUseMultiStageQueryEngine(true);
}
}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineGRPCServerOOMAccountingIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineGRPCServerOOMAccountingIntegrationTest.java
deleted file mode 100644
index 660b745702d..00000000000
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineGRPCServerOOMAccountingIntegrationTest.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.integration.tests;
-
-import org.apache.pinot.core.accounting.PerQueryCPUMemAccountantFactory;
-import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.utils.CommonConstants;
-import org.apache.pinot.spi.utils.CommonConstants.Accounting;
-import org.apache.pinot.spi.utils.CommonConstants.Broker;
-import org.apache.pinot.spi.utils.CommonConstants.Server;
-
-
-public class OfflineGRPCServerOOMAccountingIntegrationTest extends OfflineGRPCServerIntegrationTest {
-
- @Override
- protected void overrideBrokerConf(PinotConfiguration brokerConf) {
- brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_CPU_TIME_MEASUREMENT, true);
- brokerConf.setProperty(Broker.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
-
- String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
- brokerConf.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME, PerQueryCPUMemAccountantFactory.class.getName());
- brokerConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_CPU_SAMPLING, true);
- brokerConf.setProperty(prefix + Accounting.CONFIG_OF_CPU_TIME_BASED_KILLING_ENABLED, true);
- brokerConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
- brokerConf.setProperty(prefix + Accounting.CONFIG_OF_OOM_PROTECTION_KILLING_QUERY, true);
- }
-
- @Override
- protected void overrideServerConf(PinotConfiguration serverConf) {
- serverConf.setProperty(Server.CONFIG_OF_ENABLE_THREAD_CPU_TIME_MEASUREMENT, true);
- serverConf.setProperty(Server.CONFIG_OF_ENABLE_THREAD_ALLOCATED_BYTES_MEASUREMENT, true);
-
- String prefix = CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + ".";
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_FACTORY_NAME, PerQueryCPUMemAccountantFactory.class.getName());
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_CPU_SAMPLING, true);
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_CPU_TIME_BASED_KILLING_ENABLED, true);
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_ENABLE_THREAD_MEMORY_SAMPLING, true);
- serverConf.setProperty(prefix + Accounting.CONFIG_OF_OOM_PROTECTION_KILLING_QUERY, true);
- }
-}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineSecureGRPCServerIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineSecureGRPCServerIntegrationTest.java
index c087202587a..202e7a03bbd 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineSecureGRPCServerIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineSecureGRPCServerIntegrationTest.java
@@ -35,6 +35,8 @@ public class OfflineSecureGRPCServerIntegrationTest extends OfflineGRPCServerInt
@Override
protected void overrideServerConf(PinotConfiguration serverConf) {
+ super.overrideServerConf(serverConf);
+
serverConf.setProperty(Server.CONFIG_OF_GRPCTLS_SERVER_ENABLED, true);
serverConf.setProperty("pinot.server.grpctls.client.auth.enabled", true);
serverConf.setProperty("pinot.server.grpctls.keystore.type", JKS);
diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/mailbox/InMemorySendingMailbox.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/mailbox/InMemorySendingMailbox.java
index 4b0166f5016..6d8233568fa 100644
--- a/pinot-query-runtime/src/main/java/org/apache/pinot/query/mailbox/InMemorySendingMailbox.java
+++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/mailbox/InMemorySendingMailbox.java
@@ -18,19 +18,14 @@
*/
package org.apache.pinot.query.mailbox;
-import java.time.Duration;
-import java.time.temporal.ChronoUnit;
import java.util.Collections;
import java.util.List;
-import java.util.concurrent.TimeoutException;
import org.apache.pinot.common.datatable.StatMap;
import org.apache.pinot.query.runtime.blocks.ErrorMseBlock;
import org.apache.pinot.query.runtime.blocks.MseBlock;
import org.apache.pinot.query.runtime.operator.MailboxSendOperator;
import org.apache.pinot.segment.spi.memory.DataBuffer;
import org.apache.pinot.spi.exception.QueryCancelledException;
-import org.apache.pinot.spi.exception.QueryErrorCode;
-import org.apache.pinot.spi.exception.QueryException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -87,18 +82,7 @@ public class InMemorySendingMailbox implements SendingMailbox {
}
_statMap.merge(MailboxSendOperator.StatKey.IN_MEMORY_MESSAGES, 1);
long timeoutMs = _deadlineMs - System.currentTimeMillis();
- ReceivingMailbox.ReceivingMailboxStatus status;
- try {
- status = _receivingMailbox.offer(block, serializedStats, timeoutMs);
- } catch (InterruptedException e) {
- // We are not restoring the interrupt status because we are already throwing an exception
- // Code that catches this exception must finish the work fast enough to comply the interrupt contract
- // See https://github.com/apache/pinot/pull/16903#discussion_r2409003423
- throw new QueryException(QueryErrorCode.INTERNAL, "Interrupted while sending data to mailbox: " + _id, e);
- } catch (TimeoutException e) {
- throw new QueryException(QueryErrorCode.EXECUTION_TIMEOUT, "Timed out adding block into mailbox: " + _id
- + " with timeout: " + Duration.of(timeoutMs, ChronoUnit.MILLIS), e);
- }
+ ReceivingMailbox.ReceivingMailboxStatus status = _receivingMailbox.offer(block, serializedStats, timeoutMs);
switch (status) {
case SUCCESS:
break;
diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/mailbox/ReceivingMailbox.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/mailbox/ReceivingMailbox.java
index 9bef391d21c..4cc69962539 100644
--- a/pinot-query-runtime/src/main/java/org/apache/pinot/query/mailbox/ReceivingMailbox.java
+++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/mailbox/ReceivingMailbox.java
@@ -20,7 +20,6 @@ package org.apache.pinot.query.mailbox;
import com.google.common.base.Preconditions;
import com.google.errorprone.annotations.ThreadSafe;
-import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.List;
@@ -41,6 +40,8 @@ import org.apache.pinot.query.runtime.blocks.SerializedDataBlock;
import org.apache.pinot.query.runtime.blocks.SuccessMseBlock;
import org.apache.pinot.segment.spi.memory.DataBuffer;
import org.apache.pinot.spi.exception.QueryErrorCode;
+import org.apache.pinot.spi.exception.TerminationException;
+import org.apache.pinot.spi.query.QueryThreadContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,16 +50,19 @@ import org.slf4j.LoggerFactory;
/// initialized even before the corresponding OpChain is registered on the receiver, whereas the SendingMailbox is
/// initialized when the send operator is running.
///
-/// There is a single ReceivingMailbox for each pair of (sender, receiver) opchains. This means that each receive
-/// operator will have multiple ReceivingMailbox instances, one for each sender. They are coordinated by a
-/// [BlockingMultiStreamConsumer].
+/// There is a single ReceivingMailbox for each pair of (sender, receiver). This means that each receive operator will
+/// have multiple ReceivingMailbox instances, one for each sender. They are coordinated by a
+/// [org.apache.pinot.query.runtime.operator.utils.BlockingMultiStreamConsumer].
///
/// A ReceivingMailbox can have at most one reader and one writer at any given time. This means that different threads
/// writing to the same mailbox must be externally synchronized.
///
/// The offer methods will be called when new blocks are received from different sources. For example local workers will
-/// directly call [#offer(MseBlock, List, long)] while each remote worker opens a GPRC channel where messages
-/// are sent in raw format and [#offerRaw(List, long)] is called from them.
+/// directly call [#offer(MseBlock, List, long)] while each remote worker opens a gPRC channel where messages are sent
+/// in raw format and [#offerRaw(List, long)] is called from them.
+///
+/// All exceptions thrown from the offer methods should be handled within this class, and converted into a proper error
+/// block to be consumed by the reader.
@ThreadSafe
public class ReceivingMailbox {
public static final int DEFAULT_MAX_PENDING_BLOCKS = 5;
@@ -91,53 +95,62 @@ public class ReceivingMailbox {
return _id;
}
- ///Offers a raw block into the mailbox within the timeout specified, returns whether the block is successfully added.
- ///
- ///Contrary to [#offer(MseBlock, List, long)], the block may be an error block.
- public ReceivingMailboxStatus offerRaw(List<ByteBuffer> byteBuffers, long timeoutMs)
- throws IOException, InterruptedException, TimeoutException {
+ /// Offers a raw block into the mailbox within the timeout specified, returns the status of the mailbox.
+ public ReceivingMailboxStatus offerRaw(List<ByteBuffer> byteBuffers, long timeoutMs) {
updateWaitCpuTime();
- long startTimeMs = System.currentTimeMillis();
- int totalBytes = 0;
- for (ByteBuffer bb : byteBuffers) {
- totalBytes += bb.remaining();
- }
- DataBlock dataBlock = DataBlockUtils.deserialize(byteBuffers);
- _stats.merge(StatKey.DESERIALIZED_MESSAGES, 1);
- _stats.merge(StatKey.DESERIALIZED_BYTES, totalBytes);
- _stats.merge(StatKey.DESERIALIZATION_TIME_MS, System.currentTimeMillis() - startTimeMs);
-
MseBlock block;
- if (dataBlock instanceof MetadataBlock) {
- Map<Integer, String> exceptions = dataBlock.getExceptions();
- if (exceptions.isEmpty()) {
- block = SuccessMseBlock.INSTANCE;
+ List<DataBuffer> stats;
+ try {
+ long startTimeMs = System.currentTimeMillis();
+ int totalBytes = 0;
+ for (ByteBuffer bb : byteBuffers) {
+ totalBytes += bb.remaining();
+ }
+ DataBlock dataBlock = DataBlockUtils.deserialize(byteBuffers);
+ stats = dataBlock.getStatsByStage();
+ _stats.merge(StatKey.DESERIALIZED_MESSAGES, 1);
+ _stats.merge(StatKey.DESERIALIZED_BYTES, totalBytes);
+ _stats.merge(StatKey.DESERIALIZATION_TIME_MS, System.currentTimeMillis() - startTimeMs);
+
+ if (dataBlock instanceof MetadataBlock) {
+ Map<Integer, String> exceptions = dataBlock.getExceptions();
+ if (exceptions.isEmpty()) {
+ block = SuccessMseBlock.INSTANCE;
+ } else {
+ MetadataBlock metadataBlock = (MetadataBlock) dataBlock;
+ Map<QueryErrorCode, String> exceptionsByQueryError = QueryErrorCode.fromKeyMap(exceptions);
+ block =
+ new ErrorMseBlock(metadataBlock.getStageId(), metadataBlock.getWorkerId(), metadataBlock.getServerId(),
+ exceptionsByQueryError);
+ }
} else {
- MetadataBlock metadataBlock = (MetadataBlock) dataBlock;
- Map<QueryErrorCode, String> exceptionsByQueryError = QueryErrorCode.fromKeyMap(exceptions);
- block = new ErrorMseBlock(metadataBlock.getStageId(), metadataBlock.getWorkerId(), metadataBlock.getServerId(),
- exceptionsByQueryError);
+ block = new SerializedDataBlock(dataBlock);
}
- } else {
- block = new SerializedDataBlock(dataBlock);
+ } catch (Exception e) {
+ // Use the terminate exception when query is explicitly terminated.
+ TerminationException terminateException = QueryThreadContext.getTerminateException();
+ if (terminateException != null) {
+ block = ErrorMseBlock.fromException(terminateException);
+ } else {
+ String errorMessage = "Caught exception while deserializing DataBlock on mailbox: " + _id;
+ LOGGER.error(errorMessage, e);
+ block = ErrorMseBlock.fromError(QueryErrorCode.INTERNAL, errorMessage + ": " + e);
+ }
+ stats = List.of();
}
- return offerPrivate(block, dataBlock.getStatsByStage(), timeoutMs);
+ return offerPrivate(block, stats, timeoutMs);
}
- /// Offers a non-error block into the mailbox within the timeout specified, returns whether the block is successfully
- /// added.
- public ReceivingMailboxStatus offer(MseBlock block, List<DataBuffer> serializedStats, long timeoutMs)
- throws InterruptedException, TimeoutException {
+ /// Offers a block into the mailbox within the timeout specified, returns the status of the mailbox.
+ public ReceivingMailboxStatus offer(MseBlock block, List<DataBuffer> serializedStats, long timeoutMs) {
updateWaitCpuTime();
_stats.merge(StatKey.IN_MEMORY_MESSAGES, 1);
return offerPrivate(block, serializedStats, timeoutMs);
}
- /// Offers a non-error block into the mailbox within the timeout specified, returns whether the block is successfully
- /// added.
- private ReceivingMailboxStatus offerPrivate(MseBlock block, List<DataBuffer> stats, long timeoutMs)
- throws InterruptedException, TimeoutException {
+ /// Offers a block into the mailbox within the timeout specified, returns the status of the mailbox.
+ private ReceivingMailboxStatus offerPrivate(MseBlock block, List<DataBuffer> stats, long timeoutMs) {
long start = System.currentTimeMillis();
try {
ReceivingMailboxStatus result;
@@ -161,13 +174,27 @@ public class ReceivingMailbox {
// Nothing to do
}
return result;
- } catch (TimeoutException e) {
+ } catch (Exception e) {
_stats.merge(StatKey.OFFER_CPU_TIME_MS, System.currentTimeMillis() - start);
- throw e;
- } catch (InterruptedException e) {
- String errorMessage = "Interrupted on mailbox " + _id + " while offering blocks";
- setErrorBlock(ErrorMseBlock.fromError(QueryErrorCode.INTERNAL, errorMessage), stats);
- throw e;
+
+ // Use the terminate exception when query is explicitly terminated.
+ TerminationException terminateException = QueryThreadContext.getTerminateException();
+ if (terminateException != null) {
+ return _blocks.offerEos(ErrorMseBlock.fromException(terminateException), stats);
+ }
+
+ // TODO: Revisit if we should log the exception.
+ if (e instanceof TimeoutException) {
+ return _blocks.offerEos(ErrorMseBlock.fromError(QueryErrorCode.EXECUTION_TIMEOUT,
+ "Timed out while waiting for receive operator to consume data from mailbox: " + _id), stats);
+ }
+ if (e instanceof InterruptedException) {
+ return _blocks.offerEos(ErrorMseBlock.fromError(QueryErrorCode.INTERNAL,
+ "Interrupted on mailbox: " + _id + " while offering blocks"), stats);
+ }
+
+ LOGGER.error("Caught unexpected exception on mailbox: {} while offering blocks", _id, e);
+ return _blocks.offerEos(ErrorMseBlock.fromException(e), stats);
}
}
@@ -515,14 +542,8 @@ public class ReceivingMailbox {
throw new IllegalStateException("Unexpected state: " + _state);
}
}
- if (nanos <= 0L) { // timed out
- String errorMessage = "Timed out while waiting for receive operator to consume data from mailbox: " + _id;
- ErrorMseBlock timeoutBlock = ErrorMseBlock.fromError(QueryErrorCode.EXECUTION_TIMEOUT, errorMessage);
- changeState(State.FULL_CLOSED, "timed out while waiting to offer data block");
- drainDataBlocks();
- _eos = new MseBlockWithStats(timeoutBlock, List.of());
- notifyReader();
- throw new TimeoutException(errorMessage);
+ if (nanos <= 0L) {
+ throw new TimeoutException();
}
items[_putIndex] = block;
if (++_putIndex == items.length) {
diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/mailbox/channel/MailboxContentObserver.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/mailbox/channel/MailboxContentObserver.java
index d8a2f04d0c0..a37f93598f3 100644
--- a/pinot-query-runtime/src/main/java/org/apache/pinot/query/mailbox/channel/MailboxContentObserver.java
+++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/mailbox/channel/MailboxContentObserver.java
@@ -25,7 +25,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
import org.apache.commons.lang3.StringUtils;
import org.apache.pinot.common.proto.Mailbox.MailboxContent;
import org.apache.pinot.common.proto.Mailbox.MailboxStatus;
@@ -77,21 +76,7 @@ public class MailboxContentObserver implements StreamObserver<MailboxContent> {
}
try {
long timeoutMs = Context.current().getDeadline().timeRemaining(TimeUnit.MILLISECONDS);
- ReceivingMailbox.ReceivingMailboxStatus status = null;
- try {
- status = _mailbox.offerRaw(_mailboxBuffers, timeoutMs);
- } catch (TimeoutException e) {
- LOGGER.debug("Timed out adding block into mailbox: {} with timeout: {}ms", mailboxId, timeoutMs);
- closeStream();
- return;
- } catch (InterruptedException e) {
- // We are not restoring the interrupt status because we are already throwing an exception
- // Code that catches this exception must finish the work fast enough to comply the interrupt contract
- // See https://github.com/apache/pinot/pull/16903#discussion_r2409003423
- LOGGER.debug("Interrupted while processing blocks for mailbox: {}", mailboxId, e);
- closeStream();
- return;
- }
+ ReceivingMailbox.ReceivingMailboxStatus status = _mailbox.offerRaw(_mailboxBuffers, timeoutMs);
switch (status) {
case SUCCESS:
_responseObserver.onNext(MailboxStatus.newBuilder().setMailboxId(mailboxId)
diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/QueryRunner.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/QueryRunner.java
index 72da1a20aba..cbb5086f8ac 100644
--- a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/QueryRunner.java
+++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/QueryRunner.java
@@ -224,7 +224,7 @@ public class QueryRunner {
_executorService = new ThrottleOnCriticalHeapUsageExecutor(_executorService);
}
- _opChainScheduler = new OpChainSchedulerService(_executorService, serverConf);
+ _opChainScheduler = new OpChainSchedulerService(instanceDataManager.getInstanceId(), _executorService, serverConf);
_mailboxService = new MailboxService(hostname, port, InstanceType.SERVER, serverConf, tlsConfig);
try {
_leafQueryExecutor = new ServerQueryExecutorV1Impl();
diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/executor/OpChainSchedulerService.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/executor/OpChainSchedulerService.java
index 2a931acf5df..794e320eefa 100644
--- a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/executor/OpChainSchedulerService.java
+++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/executor/OpChainSchedulerService.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.query.runtime.executor;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.util.concurrent.FutureCallback;
@@ -57,14 +58,15 @@ public class OpChainSchedulerService {
private static final int NUM_QUERY_LOCKS = 1 << 10; // 1024 locks
private static final int QUERY_LOCK_MASK = NUM_QUERY_LOCKS - 1;
+ private final String _instanceId;
/// This [ExecutorService] must be wrapped with [QueryThreadContext#contextAwareExecutorService].
private final ExecutorService _executorService;
private final Cache<OpChainId, Pair<MultiStageOperator, QueryExecutionContext>> _opChainCache;
private final ReadWriteLock[] _queryLocks;
private final Cache<Long, Boolean> _cancelledQueryCache;
- public OpChainSchedulerService(ExecutorService executorService, PinotConfiguration config) {
- this(executorService, config.getProperty(MultiStageQueryRunner.KEY_OF_OP_STATS_CACHE_SIZE,
+ public OpChainSchedulerService(String instanceId, ExecutorService executorService, PinotConfiguration config) {
+ this(instanceId, executorService, config.getProperty(MultiStageQueryRunner.KEY_OF_OP_STATS_CACHE_SIZE,
MultiStageQueryRunner.DEFAULT_OF_OP_STATS_CACHE_SIZE),
config.getProperty(MultiStageQueryRunner.KEY_OF_OP_STATS_CACHE_EXPIRE_MS,
MultiStageQueryRunner.DEFAULT_OF_OP_STATS_CACHE_EXPIRE_MS),
@@ -74,15 +76,17 @@ public class OpChainSchedulerService {
MultiStageQueryRunner.DEFAULT_OF_CANCELLED_QUERY_CACHE_EXPIRE_MS));
}
+ @VisibleForTesting
public OpChainSchedulerService(ExecutorService executorService) {
- this(executorService, MultiStageQueryRunner.DEFAULT_OF_OP_STATS_CACHE_SIZE,
+ this("testServer", executorService, MultiStageQueryRunner.DEFAULT_OF_OP_STATS_CACHE_SIZE,
MultiStageQueryRunner.DEFAULT_OF_OP_STATS_CACHE_EXPIRE_MS,
MultiStageQueryRunner.DEFAULT_OF_CANCELLED_QUERY_CACHE_SIZE,
MultiStageQueryRunner.DEFAULT_OF_CANCELLED_QUERY_CACHE_EXPIRE_MS);
}
- public OpChainSchedulerService(ExecutorService executorService, int opStatsCacheSize, long opStatsCacheExpireMs,
- int cancelledQueryCacheSize, long cancelledQueryCacheExpireMs) {
+ private OpChainSchedulerService(String instanceId, ExecutorService executorService, int opStatsCacheSize,
+ long opStatsCacheExpireMs, int cancelledQueryCacheSize, long cancelledQueryCacheExpireMs) {
+ _instanceId = instanceId;
_executorService = executorService;
_opChainCache = CacheBuilder.newBuilder()
.weigher((OpChainId k, Pair<MultiStageOperator, QueryExecutionContext> v) -> countOperators(v.getLeft()))
@@ -113,7 +117,7 @@ public class OpChainSchedulerService {
// Do not schedule the operator chain if the query has been cancelled.
if (_cancelledQueryCache.getIfPresent(requestId) != null) {
LOGGER.debug("({}): Query has been cancelled", operatorChain);
- executionContext.terminate(QueryErrorCode.QUERY_CANCELLATION, "Cancelled by user");
+ executionContext.terminate(QueryErrorCode.QUERY_CANCELLATION, "Cancelled on: " + _instanceId);
throw new QueryCancelledException(
"Query has been cancelled before op-chain: " + operatorChain.getId() + " being scheduled");
} else {
@@ -194,7 +198,7 @@ public class OpChainSchedulerService {
}
if (cancelledExecutionContext != null) {
- cancelledExecutionContext.terminate(QueryErrorCode.QUERY_CANCELLATION, "Cancelled by user");
+ cancelledExecutionContext.terminate(QueryErrorCode.QUERY_CANCELLATION, "Cancelled on: " + _instanceId);
_opChainCache.invalidateAll(cancelledOperators.keySet());
Map<Integer, MultiStageQueryStats.StageStats.Closed> statsMap = new HashMap<>();
for (Map.Entry<OpChainId, MultiStageOperator> entry : cancelledOperators.entrySet()) {
diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/MailboxSendOperator.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/MailboxSendOperator.java
index fe24b253f55..e442eab1ca9 100644
--- a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/MailboxSendOperator.java
+++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/MailboxSendOperator.java
@@ -45,7 +45,7 @@ import org.apache.pinot.query.runtime.plan.OpChainExecutionContext;
import org.apache.pinot.segment.spi.memory.DataBuffer;
import org.apache.pinot.spi.exception.QueryCancelledException;
import org.apache.pinot.spi.exception.QueryException;
-import org.apache.pinot.spi.exception.TerminationException;
+import org.apache.pinot.spi.query.QueryThreadContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -168,8 +168,8 @@ public class MailboxSendOperator extends MultiStageOperator {
List<MailboxInfo> mailboxInfos =
context.getWorkerMetadata().getMailboxInfosMap().get(receiverStageId).getMailboxInfos();
List<RoutingInfo> routingInfos =
- MailboxIdUtils.toRoutingInfos(requestId, context.getStageId(), context.getWorkerId(), receiverStageId,
- mailboxInfos);
+ MailboxIdUtils.toRoutingInfos(requestId, context.getStageId(), context.getWorkerId(), receiverStageId,
+ mailboxInfos);
List<SendingMailbox> sendingMailboxes = routingInfos.stream()
.map(v -> mailboxService.getSendingMailbox(v.getHostname(), v.getPort(), v.getMailboxId(), deadlineMs, statMap))
.collect(Collectors.toList());
@@ -214,21 +214,29 @@ public class MailboxSendOperator extends MultiStageOperator {
sendEos((MseBlock.Eos) block);
} else {
sendMseBlock(((MseBlock.Data) block));
+ checkTerminationAndSampleUsage();
}
- checkTerminationAndSampleUsage();
return block;
- } catch (QueryCancelledException e) {
- LOGGER.debug("Query was cancelled for opChain: {}", _context.getId());
- return SuccessMseBlock.INSTANCE;
- } catch (TerminationException e) {
- LOGGER.info("Query was terminated for opChain: {}", _context.getId(), e);
- return ErrorMseBlock.fromException(e);
- } catch (QueryException e) {
- return ErrorMseBlock.fromException(e);
} catch (RuntimeException e) {
- ErrorMseBlock errorBlock = ErrorMseBlock.fromException(e);
- try {
+ if (e instanceof QueryCancelledException) {
+ LOGGER.debug("Query was cancelled for opChain: {}", _context.getId());
+ // TODO: Revisit if we should return success block here.
+ return SuccessMseBlock.INSTANCE;
+ }
+ ErrorMseBlock errorBlock;
+ // First check terminate exception and use it as the results block if exists. We want to return the termination
+ // reason when query is explicitly terminated.
+ QueryException queryException = QueryThreadContext.getTerminateException();
+ if (queryException == null && e instanceof QueryException) {
+ queryException = (QueryException) e;
+ }
+ if (queryException != null) {
+ errorBlock = ErrorMseBlock.fromException(queryException);
+ } else {
LOGGER.error("Exception while transferring data on opChain: {}", _context.getId(), e);
+ errorBlock = ErrorMseBlock.fromException(e);
+ }
+ try {
sendEos(errorBlock);
} catch (RuntimeException e2) {
LOGGER.error("Exception while sending error block.", e2);
diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/utils/BlockingMultiStreamConsumer.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/utils/BlockingMultiStreamConsumer.java
index ca6e3dcc8ff..7a6f3627191 100644
--- a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/utils/BlockingMultiStreamConsumer.java
+++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/utils/BlockingMultiStreamConsumer.java
@@ -33,7 +33,7 @@ import org.apache.pinot.query.runtime.plan.MultiStageQueryStats;
import org.apache.pinot.query.runtime.plan.OpChainExecutionContext;
import org.apache.pinot.segment.spi.memory.DataBuffer;
import org.apache.pinot.spi.exception.QueryErrorCode;
-import org.apache.pinot.spi.exception.QueryException;
+import org.apache.pinot.spi.exception.TerminationException;
import org.apache.pinot.spi.query.QueryThreadContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -390,7 +390,7 @@ public abstract class BlockingMultiStreamConsumer<E> implements AutoCloseable {
@Override
protected ReceivingMailbox.MseBlockWithStats onTimeout() {
// Use the terminate exception when query is explicitly terminated.
- QueryException terminateException = QueryThreadContext.getTerminateException();
+ TerminationException terminateException = QueryThreadContext.getTerminateException();
if (terminateException != null) {
return onException(terminateException.getErrorCode(), terminateException.getMessage());
}
@@ -408,7 +408,7 @@ public abstract class BlockingMultiStreamConsumer<E> implements AutoCloseable {
@Override
protected ReceivingMailbox.MseBlockWithStats onException(Exception e) {
// Use the terminate exception when query is explicitly terminated.
- QueryException terminateException = QueryThreadContext.getTerminateException();
+ TerminationException terminateException = QueryThreadContext.getTerminateException();
if (terminateException != null) {
return onException(terminateException.getErrorCode(), terminateException.getMessage());
}
diff --git a/pinot-query-runtime/src/test/java/org/apache/pinot/query/mailbox/MailboxServiceTest.java b/pinot-query-runtime/src/test/java/org/apache/pinot/query/mailbox/MailboxServiceTest.java
index 78f21ac0569..40049f513f2 100644
--- a/pinot-query-runtime/src/test/java/org/apache/pinot/query/mailbox/MailboxServiceTest.java
+++ b/pinot-query-runtime/src/test/java/org/apache/pinot/query/mailbox/MailboxServiceTest.java
@@ -44,7 +44,10 @@ import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
-import static org.testng.Assert.*;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
public class MailboxServiceTest {
@@ -245,12 +248,10 @@ public class MailboxServiceTest {
// Send one data block, sleep until timed out, then send one more block
sendingMailbox.send(OperatorTestUtil.block(DATA_SCHEMA, new Object[]{"0"}));
Thread.sleep(deadlineMs - System.currentTimeMillis() + 10);
- try {
- sendingMailbox.send(OperatorTestUtil.block(DATA_SCHEMA, new Object[]{"1"}));
- fail("Expect exception when sending data after timing out");
- } catch (Exception e) {
- // Expected
- }
+ sendingMailbox.send(OperatorTestUtil.block(DATA_SCHEMA, new Object[]{"1"}));
+
+ // Mailbox should be terminated due to timeout
+ assertTrue(sendingMailbox.isTerminated());
// Data blocks will be cleaned up
assertEquals(numCallbacks.get(), 2);
@@ -282,14 +283,11 @@ public class MailboxServiceTest {
sendingMailbox.send(OperatorTestUtil.block(DATA_SCHEMA, new Object[]{Integer.toString(i)}));
}
- // Next send will throw exception because buffer is full
- try {
- // The block sent must be a data block, as error and eos blocks are always accepted
- sendingMailbox.send(OperatorTestUtil.block(DATA_SCHEMA, new Object[]{Integer.toString(1)}));
- fail("Except exception when sending data after buffer is full");
- } catch (Exception e) {
- // Expected
- }
+ // Next send will block until timeout because buffer is full
+ sendingMailbox.send(OperatorTestUtil.block(DATA_SCHEMA, new Object[]{Integer.toString(1)}));
+
+ // Mailbox should be terminated due to timeout
+ assertTrue(sendingMailbox.isTerminated());
// Data blocks will be cleaned up
assertEquals(numCallbacks.get(), ReceivingMailbox.DEFAULT_MAX_PENDING_BLOCKS + 1);
@@ -567,20 +565,10 @@ public class MailboxServiceTest {
receiveMailLatch.countDown();
});
- // Send one data block, RPC will timeout after deadline
+ // Send one data block, receiver will time out after deadline
sendingMailbox.send(OperatorTestUtil.block(DATA_SCHEMA, new Object[]{"0"}));
- Thread.sleep(deadlineMs - System.currentTimeMillis() + 10);
receiveMailLatch.await();
assertEquals(numCallbacks.get(), 2);
- // TODO: Currently we cannot differentiate early termination vs stream error
- assertTrue(sendingMailbox.isTerminated());
-// try {
-// sendingMailbox.send(OperatorTestUtil.block(DATA_SCHEMA, new Object[]{1}));
-// fail("Expect exception when sending data after timing out");
-// } catch (Exception e) {
-// // Expected
-// }
-// assertEquals(numCallbacks.get(), 2);
// Data blocks will be cleaned up
assertEquals(receivingMailbox.getNumPendingBlocks(), 0);
@@ -588,6 +576,9 @@ public class MailboxServiceTest {
assertNotNull(block);
assertTrue(block.isError());
+ // Sender side should be terminated after receiver side times out and closes the connection
+ TestUtils.waitForCondition(aVoid -> sendingMailbox.isTerminated(), 1000L, "Failed to terminate sender");
+
// Cancel is idempotent for both sending and receiving mailbox, so safe to call multiple times
sendingMailbox.cancel(new Exception("TEST ERROR"));
receivingMailbox.cancel();
@@ -610,18 +601,14 @@ public class MailboxServiceTest {
receiveMailLatch.countDown();
});
- // send blocks big enough to avoid block combining in the sending side
- // although fragile, this works because we know that outboundMax = inboundMax/2
- String payload = new String(new char[MAX_DATA_BLOCK_SIZE / 3]).replace('\0', 'a');
-
// Sends are non-blocking as long as channel capacity is not breached
for (int i = 0; i < ReceivingMailbox.DEFAULT_MAX_PENDING_BLOCKS; i++) {
- sendingMailbox.send(OperatorTestUtil.block(DATA_SCHEMA, new Object[]{payload}));
+ sendingMailbox.send(OperatorTestUtil.block(DATA_SCHEMA, new Object[]{"0"}));
}
// Next send will be blocked on the receiver side and cause exception after timeout
// We need to send a data block, given we don't block on EOS
- sendingMailbox.send(OperatorTestUtil.block(DATA_SCHEMA, new Object[]{payload}));
+ sendingMailbox.send(OperatorTestUtil.block(DATA_SCHEMA, new Object[]{"0"}));
receiveMailLatch.await();
assertEquals(numCallbacks.get(), ReceivingMailbox.DEFAULT_MAX_PENDING_BLOCKS + 1);
@@ -676,8 +663,7 @@ public class MailboxServiceTest {
ReceivingMailbox.MseBlockWithStats block = receivingMailbox.poll();
assertNotNull(block);
assertTrue(block.getBlock().isData());
- List<Object[]> rows = ((MseBlock.Data) block.getBlock()).asRowHeap().getRows();
- return rows;
+ return ((MseBlock.Data) block.getBlock()).asRowHeap().getRows();
}
@Nullable
diff --git a/pinot-query-runtime/src/test/java/org/apache/pinot/query/mailbox/ReceivingMailboxTest.java b/pinot-query-runtime/src/test/java/org/apache/pinot/query/mailbox/ReceivingMailboxTest.java
index b9c35169afd..063d4a1cc20 100644
--- a/pinot-query-runtime/src/test/java/org/apache/pinot/query/mailbox/ReceivingMailboxTest.java
+++ b/pinot-query-runtime/src/test/java/org/apache/pinot/query/mailbox/ReceivingMailboxTest.java
@@ -35,7 +35,10 @@ import org.mockito.Mockito;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
-import static org.testng.Assert.*;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
public class ReceivingMailboxTest {
@@ -51,27 +54,31 @@ public class ReceivingMailboxTest {
}
@Test
- public void tooManyDataBlocksTheWriter()
- throws InterruptedException, TimeoutException {
+ public void tooManyDataBlocksTheWriter() {
int size = 2;
ReceivingMailbox receivingMailbox = new ReceivingMailbox("id", size);
+ receivingMailbox.registeredReader(_reader);
// Offer up to capacity
for (int i = 0; i < size; i++) {
ReceivingMailbox.ReceivingMailboxStatus status = receivingMailbox.offer(DATA_BLOCK, List.of(), 10);
assertEquals(status, ReceivingMailbox.ReceivingMailboxStatus.SUCCESS, "Should be able to offer up to capacity");
}
- // Offer one more should be rejected
- try {
- ReceivingMailbox.ReceivingMailboxStatus status = receivingMailbox.offer(DATA_BLOCK, List.of(), 10);
- fail("Should have thrown timeout exception, but " + status + " was returned");
- } catch (TimeoutException e) {
- // expected
- }
+ // Offer one more should cause timeout
+ ReceivingMailbox.ReceivingMailboxStatus status = receivingMailbox.offer(DATA_BLOCK, List.of(), 10);
+ assertEquals(status, ReceivingMailbox.ReceivingMailboxStatus.LAST_BLOCK,
+ "Should timeout when offering over capacity");
+ ReceivingMailbox.MseBlockWithStats read = receivingMailbox.poll();
+ assertNotNull(read, "Should be able to read the timeout error block");
+ MseBlock block = read.getBlock();
+ assertTrue(block.isError(), "The block should be an error block");
+ ErrorMseBlock errorBlock = (ErrorMseBlock) block;
+ assertTrue(errorBlock.getErrorMessages().containsKey(QueryErrorCode.EXECUTION_TIMEOUT),
+ "The error block should contain timeout error");
}
@Test
- public void offerAfterEos() throws InterruptedException, TimeoutException {
+ public void offerAfterEos() {
ReceivingMailbox receivingMailbox = new ReceivingMailbox("id", 10);
ReceivingMailbox.ReceivingMailboxStatus status = receivingMailbox.offer(DATA_BLOCK, List.of(), 10);
@@ -97,7 +104,7 @@ public class ReceivingMailboxTest {
}
@Test
- public void shouldReadDataInOrder() throws InterruptedException, TimeoutException {
+ public void shouldReadDataInOrder() {
ReceivingMailbox receivingMailbox = new ReceivingMailbox("id", 10);
receivingMailbox.registeredReader(_reader);
@@ -121,7 +128,7 @@ public class ReceivingMailboxTest {
}
@Test
- public void lateEosRead() throws InterruptedException, TimeoutException {
+ public void lateEosRead() {
ReceivingMailbox receivingMailbox = new ReceivingMailbox("id", 10);
receivingMailbox.registeredReader(_reader);
@@ -161,7 +168,7 @@ public class ReceivingMailboxTest {
}
@Test
- public void bufferedDataIsKeptOnSuccess() throws InterruptedException, TimeoutException {
+ public void bufferedDataIsKeptOnSuccess() {
ReceivingMailbox receivingMailbox = new ReceivingMailbox("id", 10);
receivingMailbox.registeredReader(_reader);
@@ -189,7 +196,7 @@ public class ReceivingMailboxTest {
}
@Test
- public void bufferedDataIsDiscardedOnError() throws InterruptedException, TimeoutException {
+ public void bufferedDataIsDiscardedOnError() {
ReceivingMailbox receivingMailbox = new ReceivingMailbox("id", 10);
receivingMailbox.registeredReader(_reader);
ErrorMseBlock errorBlock = ErrorMseBlock.fromException(new RuntimeException("Test error"));
@@ -213,7 +220,7 @@ public class ReceivingMailboxTest {
}
@Test
- public void dataAfterSuccess() throws InterruptedException, TimeoutException {
+ public void dataAfterSuccess() {
ReceivingMailbox receivingMailbox = new ReceivingMailbox("id", 10);
receivingMailbox.registeredReader(_reader);
@@ -237,7 +244,7 @@ public class ReceivingMailboxTest {
}
@Test
- public void dataAfterError() throws InterruptedException, TimeoutException {
+ public void dataAfterError() {
ReceivingMailbox receivingMailbox = new ReceivingMailbox("id", 10);
receivingMailbox.registeredReader(_reader);
@@ -285,7 +292,7 @@ public class ReceivingMailboxTest {
@Test(timeOut = 10_000)
public void readingUnblocksWriters()
- throws ExecutionException, InterruptedException, TimeoutException {
+ throws ExecutionException, InterruptedException {
int maxPendingBlocks = 2;
ReceivingMailbox mailbox = new ReceivingMailbox("id", maxPendingBlocks);
mailbox.registeredReader(_reader);
@@ -319,15 +326,6 @@ public class ReceivingMailboxTest {
CompletableFuture<ReceivingMailbox.ReceivingMailboxStatus> offer(MseBlock block, ReceivingMailbox receivingMailbox,
ExecutorService executor) {
- return CompletableFuture.supplyAsync(() -> {
- try {
- return receivingMailbox.offer(block, List.of(), 10_000);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new RuntimeException(e);
- } catch (TimeoutException e) {
- throw new RuntimeException(e);
- }
- }, executor);
+ return CompletableFuture.supplyAsync(() -> receivingMailbox.offer(block, List.of(), 10_000), executor);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org