You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2023/04/05 21:23:59 UTC
[pinot] branch master updated: [integration test] test query via controller (#10551)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new e3d74339a4 [integration test] test query via controller (#10551)
e3d74339a4 is described below
commit e3d74339a41c7fbeffe7904adf4fd9ea27398ad3
Author: Rong Rong <ro...@apache.org>
AuthorDate: Wed Apr 5 14:23:52 2023 -0700
[integration test] test query via controller (#10551)
Co-authored-by: Rong Rong <ro...@startree.ai>
---
.../pinot/controller/helix/ControllerTest.java | 5 +++
.../tests/BaseClusterIntegrationTest.java | 17 ++++++++
.../tests/ClusterIntegrationTestUtils.java | 48 ++++++++++++++--------
.../pinot/integration/tests/ClusterTest.java | 14 +++++++
...ultiStageEngineCustomTenantIntegrationTest.java | 9 +++-
5 files changed, 75 insertions(+), 18 deletions(-)
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/ControllerTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/ControllerTest.java
index 3259c5148f..bad59e60b8 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/ControllerTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/ControllerTest.java
@@ -210,9 +210,14 @@ public class ControllerTest {
properties.put(ControllerConf.HELIX_CLUSTER_NAME, getHelixClusterName());
// Enable groovy on the controller
properties.put(ControllerConf.DISABLE_GROOVY, false);
+ overrideControllerConf(properties);
return properties;
}
+ protected void overrideControllerConf(Map<String, Object> properties) {
+ // do nothing, to be overridden by tests if they need something specific
+ }
+
public void startController()
throws Exception {
startController(getDefaultControllerConfiguration());
diff --git a/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTest.java b/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTest.java
index 11ffabe953..64f3e6c767 100644
--- a/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTest.java
+++ b/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTest.java
@@ -650,4 +650,21 @@ public abstract class BaseClusterIntegrationTest extends ClusterTest {
ClusterIntegrationTestUtils.testQuery(pinotQuery, _brokerBaseApiUrl, getPinotConnection(), h2Query,
getH2Connection());
}
+
+ /**
+ * Run equivalent Pinot and H2 query and compare the results.
+ */
+ protected void testQueryViaController(String query)
+ throws Exception {
+ testQueryViaController(query, query);
+ }
+
+ /**
+ * Run equivalent Pinot and H2 query and compare the results.
+ */
+ protected void testQueryViaController(String pinotQuery, String h2Query)
+ throws Exception {
+ ClusterIntegrationTestUtils.testQueryViaController(pinotQuery, _controllerBaseApiUrl, getPinotConnection(), h2Query,
+ getH2Connection(), null, null);
+ }
}
diff --git a/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/ClusterIntegrationTestUtils.java b/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/ClusterIntegrationTestUtils.java
index c333f554d6..92370bf8db 100644
--- a/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/ClusterIntegrationTestUtils.java
+++ b/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/ClusterIntegrationTestUtils.java
@@ -544,55 +544,71 @@ public class ClusterIntegrationTestUtils {
/**
* Run equivalent Pinot and H2 query and compare the results.
*/
- static void testQuery(String pinotQuery, String brokerUrl, org.apache.pinot.client.Connection pinotConnection,
+ static void testQuery(String pinotQuery, String queryResourceUrl, org.apache.pinot.client.Connection pinotConnection,
String h2Query, Connection h2Connection)
throws Exception {
- testQuery(pinotQuery, brokerUrl, pinotConnection, h2Query, h2Connection, null);
+ testQuery(pinotQuery, queryResourceUrl, pinotConnection, h2Query, h2Connection, null);
}
/**
* Run equivalent Pinot and H2 query and compare the results.
*/
- static void testQuery(String pinotQuery, String brokerUrl, org.apache.pinot.client.Connection pinotConnection,
+ static void testQuery(String pinotQuery, String queryResourceUrl, org.apache.pinot.client.Connection pinotConnection,
String h2Query, Connection h2Connection, @Nullable Map<String, String> headers)
throws Exception {
- testQuery(pinotQuery, brokerUrl, pinotConnection, h2Query, h2Connection, headers, null);
+ testQuery(pinotQuery, queryResourceUrl, pinotConnection, h2Query, h2Connection, headers, null);
}
/**
* Compare # of rows in pinot and H2 only. Succeed if # of rows matches. Note this only applies to non-aggregation
* query.
*/
- static void testQueryWithMatchingRowCount(String pinotQuery, String brokerUrl,
+ static void testQueryWithMatchingRowCount(String pinotQuery, String queryResourceUrl,
org.apache.pinot.client.Connection pinotConnection, String h2Query, Connection h2Connection,
@Nullable Map<String, String> headers, @Nullable Map<String, String> extraJsonProperties)
throws Exception {
try {
- testQueryInternal(pinotQuery, brokerUrl, pinotConnection, h2Query, h2Connection, headers, extraJsonProperties,
- true);
+ testQueryInternal(pinotQuery, queryResourceUrl, pinotConnection, h2Query, h2Connection, headers,
+ extraJsonProperties, true, false);
} catch (Exception e) {
failure(pinotQuery, h2Query, "Caught exception while testing query!", e);
}
}
- static void testQuery(String pinotQuery, String brokerUrl, org.apache.pinot.client.Connection pinotConnection,
+ static void testQuery(String pinotQuery, String queryResourceUrl, org.apache.pinot.client.Connection pinotConnection,
String h2Query, Connection h2Connection, @Nullable Map<String, String> headers,
@Nullable Map<String, String> extraJsonProperties) {
try {
- testQueryInternal(pinotQuery, brokerUrl, pinotConnection, h2Query, h2Connection, headers, extraJsonProperties,
- false);
+ testQueryInternal(pinotQuery, queryResourceUrl, pinotConnection, h2Query, h2Connection, headers,
+ extraJsonProperties, false, false);
} catch (Exception e) {
failure(pinotQuery, h2Query, "Caught exception while testing query!", e);
}
}
- private static void testQueryInternal(String pinotQuery, String brokerUrl,
+ static void testQueryViaController(String pinotQuery, String queryResourceUrl,
+ org.apache.pinot.client.Connection pinotConnection, String h2Query, Connection h2Connection,
+ @Nullable Map<String, String> headers, @Nullable Map<String, String> extraJsonProperties) {
+ try {
+ testQueryInternal(pinotQuery, queryResourceUrl, pinotConnection, h2Query, h2Connection, headers,
+ extraJsonProperties, false, true);
+ } catch (Exception e) {
+ failure(pinotQuery, h2Query, "Caught exception while testing query!", e);
+ }
+ }
+
+ private static void testQueryInternal(String pinotQuery, String queryResourceUrl,
org.apache.pinot.client.Connection pinotConnection, String h2Query, Connection h2Connection,
@Nullable Map<String, String> headers, @Nullable Map<String, String> extraJsonProperties,
- boolean matchingRowCount)
+ boolean matchingRowCount, boolean viaController)
throws Exception {
// broker response
- JsonNode pinotResponse = ClusterTest.postQuery(pinotQuery, brokerUrl, headers, extraJsonProperties);
+ JsonNode pinotResponse;
+ if (viaController) {
+ pinotResponse = ClusterTest.postQueryToController(pinotQuery, queryResourceUrl, headers, extraJsonProperties);
+ } else {
+ pinotResponse = ClusterTest.postQuery(pinotQuery, queryResourceUrl, headers, extraJsonProperties);
+ }
if (!pinotResponse.get("exceptions").isEmpty()) {
throw new RuntimeException("Got Exceptions from Query Response: " + pinotResponse);
}
@@ -649,7 +665,7 @@ public class ClusterIntegrationTestUtils {
if (h2Value == null) {
if (pinotNumRecordsSelected != 0) {
throw new RuntimeException("No record selected in H2 but " + pinotNumRecordsSelected
- + " records selected in Pinot, explain plan: " + getExplainPlan(pinotQuery, brokerUrl, headers,
+ + " records selected in Pinot, explain plan: " + getExplainPlan(pinotQuery, queryResourceUrl, headers,
extraJsonProperties));
}
@@ -672,7 +688,7 @@ public class ClusterIntegrationTestUtils {
throw new RuntimeException(
"Value: " + c + " does not match, expected: " + h2Value + ", got broker value: " + brokerValue
+ ", got client value:" + connectionValue + ", explain plan: " + getExplainPlan(pinotQuery,
- brokerUrl, headers, extraJsonProperties));
+ queryResourceUrl, headers, extraJsonProperties));
}
}
} else {
@@ -696,7 +712,7 @@ public class ClusterIntegrationTestUtils {
throw new RuntimeException(
"Value: " + c + " does not match, expected: " + h2Value + ", got broker value: " + brokerValue
+ ", got client value:" + connectionValue + ", explain plan: " + getExplainPlan(pinotQuery,
- brokerUrl, headers, extraJsonProperties));
+ queryResourceUrl, headers, extraJsonProperties));
}
}
if (!h2ResultSet.next()) {
diff --git a/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/ClusterTest.java b/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/ClusterTest.java
index f8030eac78..6802736941 100644
--- a/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/ClusterTest.java
+++ b/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/ClusterTest.java
@@ -496,8 +496,22 @@ public abstract class ClusterTest extends ControllerTest {
*/
public static JsonNode postQueryToController(String query, String controllerBaseApiUrl, Map<String, String> headers)
throws Exception {
+ return postQueryToController(query, controllerBaseApiUrl, headers, null);
+ }
+
+ /**
+ * Queries the controller's sql query endpoint (/sql)
+ */
+ public static JsonNode postQueryToController(String query, String controllerBaseApiUrl, Map<String, String> headers,
+ Map<String, String> extraJsonProperties)
+ throws Exception {
ObjectNode payload = JsonUtils.newObjectNode();
payload.put("sql", query);
+ if (MapUtils.isNotEmpty(extraJsonProperties)) {
+ for (Map.Entry<String, String> extraProperty :extraJsonProperties.entrySet()) {
+ payload.put(extraProperty.getKey(), extraProperty.getValue());
+ }
+ }
return JsonUtils.stringToJsonNode(
sendPostRequest(controllerBaseApiUrl + "/sql", JsonUtils.objectToString(payload), headers));
}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineCustomTenantIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineCustomTenantIntegrationTest.java
index c77d6c4cd1..599b95f341 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineCustomTenantIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineCustomTenantIntegrationTest.java
@@ -150,12 +150,17 @@ public class MultiStageEngineCustomTenantIntegrationTest extends MultiStageEngin
serverConf.setProperty(QueryConfig.KEY_OF_QUERY_RUNNER_PORT, 8422);
}
+ @Override
+ protected void overrideControllerConf(Map<String, Object> properties) {
+ properties.put(CommonConstants.Helix.CONFIG_OF_MULTI_STAGE_ENGINE_ENABLED, true);
+ }
+
@Override
protected void testQuery(String pinotQuery, String h2Query)
throws Exception {
ClusterIntegrationTestUtils
- .testQuery(pinotQuery, _brokerBaseApiUrl, getPinotConnection(), h2Query, getH2Connection(), null,
- ImmutableMap.of("queryOptions", "useMultistageEngine=true"));
+ .testQueryViaController(pinotQuery, _controllerBaseApiUrl, getPinotConnection(), h2Query, getH2Connection(),
+ null, ImmutableMap.of("queryOptions", "useMultistageEngine=true"));
}
@AfterClass
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org