You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by si...@apache.org on 2023/05/24 17:33:11 UTC
[pinot] branch master updated: Enhance the test cases for query killing to reduce the log size and fix a flaky test (#10798)
This is an automated email from the ASF dual-hosted git repository.
siddteotia pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new b76f24cb30 Enhance the test cases for query killing to reduce the log size and fix a flaky test (#10798)
b76f24cb30 is described below
commit b76f24cb30c7c92c2839faac5ea6a97ac01f2044
Author: Jia Guo <ji...@linkedin.com>
AuthorDate: Wed May 24 10:33:01 2023 -0700
Enhance the test cases for query killing to reduce the log size and fix a flaky test (#10798)
* Enhance the test cases for query killing to reduce the log size and fix a flaky test
* Enhance the test cases for query killing to reduce the log size and fix a flaky test
---
...flineClusterMemBasedBrokerQueryKillingTest.java | 28 +++++++---------------
...flineClusterMemBasedServerQueryKillingTest.java | 26 +++++++++-----------
...fflineClusterServerCPUTimeQueryKillingTest.java | 18 +++++++-------
3 files changed, 28 insertions(+), 44 deletions(-)
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedBrokerQueryKillingTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedBrokerQueryKillingTest.java
index d67d34fc9b..4bbab9a93c 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedBrokerQueryKillingTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedBrokerQueryKillingTest.java
@@ -51,8 +51,6 @@ import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.apache.pinot.util.TestUtils;
import org.junit.Assert;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@@ -68,7 +66,6 @@ public class OfflineClusterMemBasedBrokerQueryKillingTest extends BaseClusterInt
public static final String LONG_DIM_SV1 = "longDimSV1";
public static final String DOUBLE_DIM_SV1 = "doubleDimSV1";
public static final String BOOLEAN_DIM_SV1 = "booleanDimSV1";
- private static final Logger LOGGER = LoggerFactory.getLogger(OfflineClusterMemBasedBrokerQueryKillingTest.class);
private static final int NUM_BROKERS = 1;
private static final int NUM_SERVERS = 3;
private static final String OOM_QUERY =
@@ -97,13 +94,8 @@ public class OfflineClusterMemBasedBrokerQueryKillingTest extends BaseClusterInt
@BeforeClass
public void setUp()
throws Exception {
- // Setup logging and resource accounting
- LogManager.getLogger(OfflineClusterMemBasedBrokerQueryKillingTest.class).setLevel(Level.INFO);
LogManager.getLogger(PerQueryCPUMemAccountantFactory.PerQueryCPUMemResourceUsageAccountant.class)
- .setLevel(Level.INFO);
- LogManager.getLogger(ThreadResourceUsageProvider.class).setLevel(Level.INFO);
- LogManager.getLogger(Tracing.class).setLevel(Level.INFO);
- LogManager.getLogger(ThreadResourceUsageProvider.class).setLevel(Level.INFO);
+ .setLevel(Level.ERROR);
ThreadResourceUsageProvider.setThreadCpuTimeMeasurementEnabled(true);
ThreadResourceUsageProvider.setThreadMemoryMeasurementEnabled(true);
@@ -137,6 +129,13 @@ public class OfflineClusterMemBasedBrokerQueryKillingTest extends BaseClusterInt
//Wait for all documents loaded
waitForAllDocsLoaded(10_000L);
+
+ // Setup logging and resource accounting
+ LogManager.getLogger(OfflineClusterMemBasedBrokerQueryKillingTest.class).setLevel(Level.INFO);
+ LogManager.getLogger(PerQueryCPUMemAccountantFactory.PerQueryCPUMemResourceUsageAccountant.class)
+ .setLevel(Level.INFO);
+ LogManager.getLogger(ThreadResourceUsageProvider.class).setLevel(Level.INFO);
+ LogManager.getLogger(Tracing.class).setLevel(Level.INFO);
}
protected void startBrokers()
@@ -153,7 +152,7 @@ public class OfflineClusterMemBasedBrokerQueryKillingTest extends BaseClusterInt
brokerConf.setProperty(CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + "."
+ CommonConstants.Accounting.CONFIG_OF_ALARMING_LEVEL_HEAP_USAGE_RATIO, 0.0f);
brokerConf.setProperty(CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + "."
- + CommonConstants.Accounting.CONFIG_OF_CRITICAL_LEVEL_HEAP_USAGE_RATIO, 0.60f);
+ + CommonConstants.Accounting.CONFIG_OF_CRITICAL_LEVEL_HEAP_USAGE_RATIO, 0.40f);
brokerConf.setProperty(CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + "."
+ CommonConstants.Accounting.CONFIG_OF_INSTANCE_TYPE, InstanceType.BROKER);
brokerConf.setProperty(CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + "."
@@ -206,14 +205,6 @@ public class OfflineClusterMemBasedBrokerQueryKillingTest extends BaseClusterInt
.build();
}
-// @Test
-// public void testDigestOOM()
-// throws Exception {
-// JsonNode queryResponse = postQuery(OOM_QUERY);
-// LOGGER.info("testDigestOOM: {}", queryResponse);
-// Assert.assertTrue(queryResponse.get("exceptions").toString().contains("QueryCancelledException"));
-// Assert.assertTrue(queryResponse.get("exceptions").toString().contains("got killed because"));
-// }
@Test
public void testDigestOOMMultipleQueries()
@@ -252,7 +243,6 @@ public class OfflineClusterMemBasedBrokerQueryKillingTest extends BaseClusterInt
}
);
countDownLatch.await();
- LOGGER.info("testDigestOOMMultipleQueries: {}", queryResponse1);
Assert.assertTrue(queryResponse1.get().get("exceptions").toString().contains(
"Interrupted in broker reduce phase"));
Assert.assertTrue(queryResponse1.get().get("exceptions").toString().contains("\"errorCode\":"
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedServerQueryKillingTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedServerQueryKillingTest.java
index db877ac005..7cc9a60fe6 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedServerQueryKillingTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedServerQueryKillingTest.java
@@ -50,8 +50,6 @@ import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.apache.pinot.util.TestUtils;
import org.junit.Assert;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@@ -60,7 +58,6 @@ import org.testng.annotations.Test;
* Integration test for heap size based server query killing, this works only for xmx4G
*/
public class OfflineClusterMemBasedServerQueryKillingTest extends BaseClusterIntegrationTestSet {
- private static final Logger LOGGER = LoggerFactory.getLogger(OfflineClusterMemBasedServerQueryKillingTest.class);
public static final String STRING_DIM_SV1 = "stringDimSV1";
public static final String STRING_DIM_SV2 = "stringDimSV2";
public static final String INT_DIM_SV1 = "intDimSV1";
@@ -69,7 +66,7 @@ public class OfflineClusterMemBasedServerQueryKillingTest extends BaseClusterInt
public static final String BOOLEAN_DIM_SV1 = "booleanDimSV1";
private static final int NUM_BROKERS = 1;
private static final int NUM_SERVERS = 1;
-private static final int NUM_DOCS = 3_000_000;
+ private static final int NUM_DOCS = 3_000_000;
private static final String OOM_QUERY =
"SELECT PERCENTILETDigest(doubleDimSV1, 50) AS digest, intDimSV1 FROM mytable GROUP BY intDimSV1"
@@ -77,7 +74,7 @@ private static final int NUM_DOCS = 3_000_000;
private static final String OOM_QUERY_2 =
"SELECT stringDimSV2 FROM mytable GROUP BY stringDimSV2"
- + " ORDER BY stringDimSV2 LIMIT 1000000";
+ + " ORDER BY stringDimSV2 LIMIT 3000000";
private static final String DIGEST_QUERY_1 =
"SELECT PERCENTILETDigest(doubleDimSV1, 50) AS digest FROM mytable";
@@ -101,13 +98,8 @@ private static final int NUM_DOCS = 3_000_000;
@BeforeClass
public void setUp()
throws Exception {
- // Setup logging and resource accounting
- LogManager.getLogger(OfflineClusterMemBasedServerQueryKillingTest.class).setLevel(Level.INFO);
LogManager.getLogger(PerQueryCPUMemAccountantFactory.PerQueryCPUMemResourceUsageAccountant.class)
- .setLevel(Level.INFO);
- LogManager.getLogger(ThreadResourceUsageProvider.class).setLevel(Level.INFO);
- LogManager.getLogger(Tracing.class).setLevel(Level.INFO);
- LogManager.getLogger(ThreadResourceUsageProvider.class).setLevel(Level.INFO);
+ .setLevel(Level.ERROR);
ThreadResourceUsageProvider.setThreadCpuTimeMeasurementEnabled(true);
ThreadResourceUsageProvider.setThreadMemoryMeasurementEnabled(true);
@@ -142,6 +134,13 @@ private static final int NUM_DOCS = 3_000_000;
//Wait for all documents loaded
waitForAllDocsLoaded(10_000L);
+
+ // Setup logging and resource accounting
+ LogManager.getLogger(OfflineClusterMemBasedServerQueryKillingTest.class).setLevel(Level.INFO);
+ LogManager.getLogger(PerQueryCPUMemAccountantFactory.PerQueryCPUMemResourceUsageAccountant.class)
+ .setLevel(Level.INFO);
+ LogManager.getLogger(ThreadResourceUsageProvider.class).setLevel(Level.INFO);
+ LogManager.getLogger(Tracing.class).setLevel(Level.INFO);
}
protected void startBrokers()
@@ -158,7 +157,7 @@ private static final int NUM_DOCS = 3_000_000;
serverConf.setProperty(CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + "."
+ CommonConstants.Accounting.CONFIG_OF_ALARMING_LEVEL_HEAP_USAGE_RATIO, 0.0f);
serverConf.setProperty(CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + "."
- + CommonConstants.Accounting.CONFIG_OF_CRITICAL_LEVEL_HEAP_USAGE_RATIO, 0.25f);
+ + CommonConstants.Accounting.CONFIG_OF_CRITICAL_LEVEL_HEAP_USAGE_RATIO, 0.15f);
serverConf.setProperty(
CommonConstants.PINOT_QUERY_SCHEDULER_PREFIX + "." + CommonConstants.Accounting.CONFIG_OF_FACTORY_NAME,
"org.apache.pinot.core.accounting.PerQueryCPUMemAccountantFactory");
@@ -216,7 +215,6 @@ private static final int NUM_DOCS = 3_000_000;
public void testDigestOOM()
throws Exception {
JsonNode queryResponse = postQuery(OOM_QUERY);
- LOGGER.info("testDigestOOM: {}", queryResponse);
Assert.assertTrue(queryResponse.get("exceptions").toString().contains("\"errorCode\":"
+ QueryException.QUERY_CANCELLATION_ERROR_CODE));
Assert.assertTrue(queryResponse.get("exceptions").toString().contains("QueryCancelledException"));
@@ -227,7 +225,6 @@ private static final int NUM_DOCS = 3_000_000;
public void testDigestOOM2()
throws Exception {
JsonNode queryResponse = postQuery(OOM_QUERY_2);
- LOGGER.info("testDigestOOM: {}", queryResponse);
Assert.assertTrue(queryResponse.get("exceptions").toString().contains("QueryCancelledException"));
Assert.assertTrue(queryResponse.get("exceptions").toString().contains("got killed because"));
}
@@ -269,7 +266,6 @@ private static final int NUM_DOCS = 3_000_000;
}
);
countDownLatch.await();
- LOGGER.info("testDigestOOMMultipleQueries: {}", queryResponse1);
Assert.assertTrue(queryResponse1.get().get("exceptions").toString().contains("\"errorCode\":503"));
Assert.assertTrue(queryResponse1.get().get("exceptions").toString().contains("QueryCancelledException"));
Assert.assertTrue(queryResponse1.get().get("exceptions").toString().contains("got killed because"));
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterServerCPUTimeQueryKillingTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterServerCPUTimeQueryKillingTest.java
index 67b46c3220..ce8a12a478 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterServerCPUTimeQueryKillingTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterServerCPUTimeQueryKillingTest.java
@@ -49,8 +49,6 @@ import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.apache.pinot.util.TestUtils;
import org.junit.Assert;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@@ -59,7 +57,6 @@ import org.testng.annotations.Test;
* Integration test for heap size based server query killing, this works only for xmx4G
*/
public class OfflineClusterServerCPUTimeQueryKillingTest extends BaseClusterIntegrationTestSet {
- private static final Logger LOGGER = LoggerFactory.getLogger(OfflineClusterServerCPUTimeQueryKillingTest.class);
public static final String STRING_DIM_SV1 = "stringDimSV1";
public static final String STRING_DIM_SV2 = "stringDimSV2";
public static final String INT_DIM_SV1 = "intDimSV1";
@@ -94,13 +91,8 @@ public class OfflineClusterServerCPUTimeQueryKillingTest extends BaseClusterInte
@BeforeClass
public void setUp()
throws Exception {
- // Setup logging and resource accounting
- LogManager.getLogger(OfflineClusterServerCPUTimeQueryKillingTest.class).setLevel(Level.INFO);
LogManager.getLogger(PerQueryCPUMemAccountantFactory.PerQueryCPUMemResourceUsageAccountant.class)
- .setLevel(Level.INFO);
- LogManager.getLogger(ThreadResourceUsageProvider.class).setLevel(Level.INFO);
- LogManager.getLogger(Tracing.class).setLevel(Level.INFO);
- LogManager.getLogger(ThreadResourceUsageProvider.class).setLevel(Level.INFO);
+ .setLevel(Level.ERROR);
ThreadResourceUsageProvider.setThreadCpuTimeMeasurementEnabled(true);
ThreadResourceUsageProvider.setThreadMemoryMeasurementEnabled(true);
@@ -135,6 +127,13 @@ public class OfflineClusterServerCPUTimeQueryKillingTest extends BaseClusterInte
//Wait for all documents loaded
waitForAllDocsLoaded(10_000L);
+
+ // Setup logging and resource accounting
+ LogManager.getLogger(OfflineClusterServerCPUTimeQueryKillingTest.class).setLevel(Level.INFO);
+ LogManager.getLogger(PerQueryCPUMemAccountantFactory.PerQueryCPUMemResourceUsageAccountant.class)
+ .setLevel(Level.INFO);
+ LogManager.getLogger(ThreadResourceUsageProvider.class).setLevel(Level.INFO);
+ LogManager.getLogger(Tracing.class).setLevel(Level.INFO);
}
protected void startBrokers()
@@ -252,7 +251,6 @@ public class OfflineClusterServerCPUTimeQueryKillingTest extends BaseClusterInte
}
);
countDownLatch.await();
- LOGGER.info("testDigestOOMMultipleQueries: {}", queryResponse1);
Assert.assertTrue(queryResponse1.get().get("exceptions").toString().contains("got killed on SERVER"));
Assert.assertTrue(queryResponse1.get().get("exceptions").toString().contains("CPU time exceeding limit of"));
Assert.assertFalse(StringUtils.isEmpty(queryResponse2.get().get("exceptions").toString()));
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org