You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2023/04/07 14:41:53 UTC
[pinot] branch master updated: Turn on v2 engine by default (#10543)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 8eaeb036e3 Turn on v2 engine by default (#10543)
8eaeb036e3 is described below
commit 8eaeb036e353eb4a64570df4cd3696eaf56c2d3b
Author: Xiang Fu <xi...@gmail.com>
AuthorDate: Fri Apr 7 07:41:45 2023 -0700
Turn on v2 engine by default (#10543)
---
.../broker/broker/helix/BaseBrokerStarter.java | 33 ++++++++++++++++++++++
.../MultiStageBrokerRequestHandler.java | 3 +-
.../pinot/common/datatable/DataTableImplV4.java | 7 ++---
.../common/datatable/DataTableBuilderFactory.java | 2 +-
.../apache/pinot/queries/AllNullQueriesTest.java | 2 --
.../pinot/queries/BigDecimalQueriesTest.java | 2 --
.../queries/BooleanNullEnabledQueriesTest.java | 2 --
.../pinot/queries/NullEnabledQueriesTest.java | 2 --
...ultiStageEngineCustomTenantIntegrationTest.java | 26 -----------------
.../tests/MultiStageEngineIntegrationTest.java | 21 --------------
.../tests/NullHandlingIntegrationTest.java | 3 --
.../tests/OfflineClusterIntegrationTest.java | 3 ++
.../integration/tests/SSBQueryIntegrationTest.java | 21 --------------
.../pinot/query/runtime/QueryRunnerTest.java | 2 --
.../runtime/queries/ResourceBasedQueriesTest.java | 3 --
.../server/starter/helix/BaseServerStarter.java | 7 +++++
.../apache/pinot/spi/utils/CommonConstants.java | 2 +-
.../pinot/tools/MultistageEngineQuickStart.java | 9 ------
.../tools/admin/command/QuickstartRunner.java | 7 -----
19 files changed, 49 insertions(+), 108 deletions(-)
diff --git a/pinot-broker/src/main/java/org/apache/pinot/broker/broker/helix/BaseBrokerStarter.java b/pinot-broker/src/main/java/org/apache/pinot/broker/broker/helix/BaseBrokerStarter.java
index e9b6ec3c9a..cb24a4d5d8 100644
--- a/pinot-broker/src/main/java/org/apache/pinot/broker/broker/helix/BaseBrokerStarter.java
+++ b/pinot-broker/src/main/java/org/apache/pinot/broker/broker/helix/BaseBrokerStarter.java
@@ -67,6 +67,7 @@ import org.apache.pinot.core.query.executor.sql.SqlQueryExecutor;
import org.apache.pinot.core.transport.ListenerConfig;
import org.apache.pinot.core.transport.server.routing.stats.ServerRoutingStatsManager;
import org.apache.pinot.core.util.ListenerConfigUtil;
+import org.apache.pinot.query.service.QueryConfig;
import org.apache.pinot.spi.accounting.ThreadResourceUsageProvider;
import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.metrics.PinotMetricUtils;
@@ -132,6 +133,9 @@ public abstract class BaseBrokerStarter implements ServiceStartable {
_clusterName = brokerConf.getProperty(Helix.CONFIG_OF_CLUSTER_NAME);
ServiceStartableUtils.applyClusterConfig(_brokerConf, _zkServers, _clusterName, ServiceRole.BROKER);
+ if (_brokerConf.getProperty(QueryConfig.KEY_OF_QUERY_RUNNER_PORT, QueryConfig.DEFAULT_QUERY_RUNNER_PORT) == 0) {
+ _brokerConf.setProperty(QueryConfig.KEY_OF_QUERY_RUNNER_PORT, NetUtils.findOpenPort());
+ }
setupHelixSystemProperties();
_listenerConfigs = ListenerConfigUtil.buildBrokerConfigs(brokerConf);
_hostname = brokerConf.getProperty(Broker.CONFIG_OF_BROKER_HOSTNAME);
@@ -391,9 +395,19 @@ public abstract class BaseBrokerStarter implements ServiceStartable {
private void updateInstanceConfigAndBrokerResourceIfNeeded() {
InstanceConfig instanceConfig = HelixHelper.getInstanceConfig(_participantHelixManager, _instanceId);
boolean updated = HelixHelper.updateHostnamePort(instanceConfig, _hostname, _port);
+
+ ZNRecord znRecord = instanceConfig.getRecord();
+ Map<String, String> simpleFields = znRecord.getSimpleFields();
if (_tlsPort > 0) {
HelixHelper.updateTlsPort(instanceConfig, _tlsPort);
}
+ // Update multi-stage query engine ports
+ if (_brokerConf.getProperty(Helix.CONFIG_OF_MULTI_STAGE_ENGINE_ENABLED, Helix.DEFAULT_MULTI_STAGE_ENGINE_ENABLED)) {
+ updated |= updatePortIfNeeded(simpleFields, Helix.Instance.MULTI_STAGE_QUERY_ENGINE_MAILBOX_PORT_KEY,
+ Integer.parseInt(_brokerConf.getProperty(QueryConfig.KEY_OF_QUERY_RUNNER_PORT)));
+ } else {
+ updated |= updatePortIfNeeded(simpleFields, Helix.Instance.MULTI_STAGE_QUERY_ENGINE_MAILBOX_PORT_KEY, -1);
+ }
updated |= HelixHelper.removeDisabledPartitions(instanceConfig);
boolean shouldUpdateBrokerResource = false;
String brokerTag = null;
@@ -462,6 +476,25 @@ public abstract class BaseBrokerStarter implements ServiceStartable {
}
}
+ private boolean updatePortIfNeeded(Map<String, String> instanceConfigSimpleFields, String key, int port) {
+ String existingPortStr = instanceConfigSimpleFields.get(key);
+ if (port > 0) {
+ String portStr = Integer.toString(port);
+ if (!portStr.equals(existingPortStr)) {
+ LOGGER.info("Updating '{}' for instance: {} to: {}", key, _instanceId, port);
+ instanceConfigSimpleFields.put(key, portStr);
+ return true;
+ }
+ } else {
+ if (existingPortStr != null) {
+ LOGGER.info("Removing '{}' from instance: {}", key, _instanceId);
+ instanceConfigSimpleFields.remove(key);
+ return true;
+ }
+ }
+ return false;
+ }
+
@Override
public void stop() {
LOGGER.info("Shutting down Pinot broker");
diff --git a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/MultiStageBrokerRequestHandler.java b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/MultiStageBrokerRequestHandler.java
index 9ebc6ea6b4..a1686aa47a 100644
--- a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/MultiStageBrokerRequestHandler.java
+++ b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/MultiStageBrokerRequestHandler.java
@@ -93,7 +93,8 @@ public class MultiStageBrokerRequestHandler extends BaseBrokerRequestHandler {
reducerHostname = brokerId;
}
_reducerHostname = reducerHostname;
- _reducerPort = config.getProperty(QueryConfig.KEY_OF_QUERY_RUNNER_PORT, QueryConfig.DEFAULT_QUERY_RUNNER_PORT);
+ // This config has to be set to a valid port number.
+ _reducerPort = Integer.parseInt(config.getProperty(QueryConfig.KEY_OF_QUERY_RUNNER_PORT));
_defaultBrokerTimeoutMs = config.getProperty(CommonConstants.Broker.CONFIG_OF_BROKER_TIMEOUT_MS,
CommonConstants.Broker.DEFAULT_BROKER_TIMEOUT_MS);
_queryEnvironment = new QueryEnvironment(new TypeFactory(new TypeSystem()),
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/datatable/DataTableImplV4.java b/pinot-common/src/main/java/org/apache/pinot/common/datatable/DataTableImplV4.java
index d4d27634f9..a4d69ee714 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/datatable/DataTableImplV4.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/datatable/DataTableImplV4.java
@@ -150,15 +150,12 @@ public class DataTableImplV4 implements DataTable {
}
// Read variable size data.
+ _variableSizeDataBytes = new byte[variableSizeDataLength];
if (variableSizeDataLength != 0) {
- _variableSizeDataBytes = new byte[variableSizeDataLength];
byteBuffer.position(variableSizeDataStart);
byteBuffer.get(_variableSizeDataBytes);
- _variableSizeData = ByteBuffer.wrap(_variableSizeDataBytes);
- } else {
- _variableSizeDataBytes = null;
- _variableSizeData = null;
}
+ _variableSizeData = ByteBuffer.wrap(_variableSizeDataBytes);
// Read metadata.
int metadataLength = byteBuffer.getInt();
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/DataTableBuilderFactory.java b/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/DataTableBuilderFactory.java
index 2bf0426a78..967233b2ee 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/DataTableBuilderFactory.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/DataTableBuilderFactory.java
@@ -34,7 +34,7 @@ public class DataTableBuilderFactory {
private static final Logger LOGGER = LoggerFactory.getLogger(DataTableBuilderFactory.class);
- public static final int DEFAULT_VERSION = DataTableFactory.VERSION_3;
+ public static final int DEFAULT_VERSION = DataTableFactory.VERSION_4;
private static int _version = DEFAULT_VERSION;
diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/AllNullQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/AllNullQueriesTest.java
index 960bde9cca..f345e99f58 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/AllNullQueriesTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/AllNullQueriesTest.java
@@ -26,7 +26,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.common.datatable.DataTableFactory;
import org.apache.pinot.common.response.broker.BrokerResponseNative;
import org.apache.pinot.common.response.broker.ResultTable;
import org.apache.pinot.common.utils.DataSchema;
@@ -285,7 +284,6 @@ public class AllNullQueriesTest extends BaseQueriesTest {
public void testQueries(ColumnDataType columnDataType, File indexDir)
throws IOException {
- DataTableBuilderFactory.setDataTableVersion(DataTableFactory.VERSION_4);
Map<String, String> queryOptions = new HashMap<>();
queryOptions.put("enableNullHandling", "true");
DataType dataType = columnDataType.toDataType();
diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/BigDecimalQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/BigDecimalQueriesTest.java
index d44a266b02..42978199c9 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/BigDecimalQueriesTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/BigDecimalQueriesTest.java
@@ -28,7 +28,6 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.common.datatable.DataTableFactory;
import org.apache.pinot.common.response.broker.BrokerResponseNative;
import org.apache.pinot.common.response.broker.ResultTable;
import org.apache.pinot.common.utils.DataSchema;
@@ -160,7 +159,6 @@ public class BigDecimalQueriesTest extends BaseQueriesTest {
}
public void testQueries() {
- DataTableBuilderFactory.setDataTableVersion(DataTableFactory.VERSION_4);
Map<String, String> queryOptions = new HashMap<>();
queryOptions.put("enableNullHandling", "true");
{
diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/BooleanNullEnabledQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/BooleanNullEnabledQueriesTest.java
index 63443449d9..e707bf0fde 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/BooleanNullEnabledQueriesTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/BooleanNullEnabledQueriesTest.java
@@ -27,7 +27,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.common.datatable.DataTableFactory;
import org.apache.pinot.common.response.broker.BrokerResponseNative;
import org.apache.pinot.common.response.broker.ResultTable;
import org.apache.pinot.common.utils.DataSchema;
@@ -181,7 +180,6 @@ public class BooleanNullEnabledQueriesTest extends BaseQueriesTest {
}
public void testQueries() {
- DataTableBuilderFactory.setDataTableVersion(DataTableFactory.VERSION_4);
Map<String, String> queryOptions = new HashMap<>();
queryOptions.put("enableNullHandling", "true");
HashSet<Integer> trueIndices = new HashSet<Integer>(Arrays.asList(1, 3, 5));
diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/NullEnabledQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/NullEnabledQueriesTest.java
index a93cd88a71..f279487fa6 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/NullEnabledQueriesTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/NullEnabledQueriesTest.java
@@ -28,7 +28,6 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.common.datatable.DataTableFactory;
import org.apache.pinot.common.response.broker.BrokerResponseNative;
import org.apache.pinot.common.response.broker.ResultTable;
import org.apache.pinot.common.utils.DataSchema;
@@ -281,7 +280,6 @@ public class NullEnabledQueriesTest extends BaseQueriesTest {
}
public void testQueries(Number baseValue, ColumnDataType dataType, boolean nullValuesExist) {
- DataTableBuilderFactory.setDataTableVersion(DataTableFactory.VERSION_4);
Map<String, String> queryOptions = new HashMap<>();
queryOptions.put("enableNullHandling", "true");
{
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineCustomTenantIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineCustomTenantIntegrationTest.java
index 599b95f341..a9f204a365 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineCustomTenantIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineCustomTenantIntegrationTest.java
@@ -26,14 +26,9 @@ import java.util.Properties;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.client.Connection;
import org.apache.pinot.client.ConnectionFactory;
-import org.apache.pinot.common.datatable.DataTableFactory;
import org.apache.pinot.controller.ControllerConf;
-import org.apache.pinot.core.common.datatable.DataTableBuilderFactory;
-import org.apache.pinot.query.service.QueryConfig;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.data.Schema;
-import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.util.TestUtils;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
@@ -97,9 +92,6 @@ public class MultiStageEngineCustomTenantIntegrationTest extends MultiStageEngin
// Wait for all documents loaded
waitForAllDocsLoaded(600_000L);
-
- // Setting data table version to 4
- DataTableBuilderFactory.setDataTableVersion(DataTableFactory.VERSION_4);
}
@Test
@@ -137,24 +129,6 @@ public class MultiStageEngineCustomTenantIntegrationTest extends MultiStageEngin
return _pinotConnection;
}
- @Override
- protected void overrideBrokerConf(PinotConfiguration brokerConf) {
- brokerConf.setProperty(CommonConstants.Helix.CONFIG_OF_MULTI_STAGE_ENGINE_ENABLED, true);
- brokerConf.setProperty(QueryConfig.KEY_OF_QUERY_RUNNER_PORT, 8421);
- }
-
- @Override
- protected void overrideServerConf(PinotConfiguration serverConf) {
- serverConf.setProperty(CommonConstants.Helix.CONFIG_OF_MULTI_STAGE_ENGINE_ENABLED, true);
- serverConf.setProperty(QueryConfig.KEY_OF_QUERY_SERVER_PORT, 8842);
- serverConf.setProperty(QueryConfig.KEY_OF_QUERY_RUNNER_PORT, 8422);
- }
-
- @Override
- protected void overrideControllerConf(Map<String, Object> properties) {
- properties.put(CommonConstants.Helix.CONFIG_OF_MULTI_STAGE_ENGINE_ENABLED, true);
- }
-
@Override
protected void testQuery(String pinotQuery, String h2Query)
throws Exception {
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineIntegrationTest.java
index 6dabf814d8..fa5e51b7e2 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineIntegrationTest.java
@@ -25,13 +25,8 @@ import java.util.Properties;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.client.Connection;
import org.apache.pinot.client.ConnectionFactory;
-import org.apache.pinot.common.datatable.DataTableFactory;
-import org.apache.pinot.core.common.datatable.DataTableBuilderFactory;
-import org.apache.pinot.query.service.QueryConfig;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.data.Schema;
-import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.util.TestUtils;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
@@ -78,9 +73,6 @@ public class MultiStageEngineIntegrationTest extends BaseClusterIntegrationTestS
// Wait for all documents loaded
waitForAllDocsLoaded(600_000L);
-
- // Setting data table version to 4
- DataTableBuilderFactory.setDataTableVersion(DataTableFactory.VERSION_4);
}
@Test
@@ -129,19 +121,6 @@ public class MultiStageEngineIntegrationTest extends BaseClusterIntegrationTestS
return _pinotConnection;
}
- @Override
- protected void overrideBrokerConf(PinotConfiguration brokerConf) {
- brokerConf.setProperty(CommonConstants.Helix.CONFIG_OF_MULTI_STAGE_ENGINE_ENABLED, true);
- brokerConf.setProperty(QueryConfig.KEY_OF_QUERY_RUNNER_PORT, 8421);
- }
-
- @Override
- protected void overrideServerConf(PinotConfiguration serverConf) {
- serverConf.setProperty(CommonConstants.Helix.CONFIG_OF_MULTI_STAGE_ENGINE_ENABLED, true);
- serverConf.setProperty(QueryConfig.KEY_OF_QUERY_SERVER_PORT, 8842);
- serverConf.setProperty(QueryConfig.KEY_OF_QUERY_RUNNER_PORT, 8422);
- }
-
@Override
protected void testQuery(String pinotQuery, String h2Query)
throws Exception {
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/NullHandlingIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/NullHandlingIntegrationTest.java
index c93b37835f..16b8307ecf 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/NullHandlingIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/NullHandlingIntegrationTest.java
@@ -23,7 +23,6 @@ import java.io.File;
import java.util.List;
import javax.annotation.Nullable;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.common.datatable.DataTableFactory;
import org.apache.pinot.core.common.datatable.DataTableBuilderFactory;
import org.apache.pinot.util.TestUtils;
import org.testng.annotations.AfterClass;
@@ -196,7 +195,6 @@ public class NullHandlingIntegrationTest extends BaseClusterIntegrationTestSet {
@Test
public void testTotalCountWithNullHandlingQueryOptionEnabled()
throws Exception {
- DataTableBuilderFactory.setDataTableVersion(DataTableFactory.VERSION_4);
String pinotQuery = "SELECT COUNT(*) FROM " + getTableName() + " option(enableNullHandling=true)";
String h2Query = "SELECT COUNT(*) FROM " + getTableName();
testQuery(pinotQuery, h2Query);
@@ -210,7 +208,6 @@ public class NullHandlingIntegrationTest extends BaseClusterIntegrationTestSet {
@Test
public void testNullLiteralSelectionOnlyBroker()
throws Exception {
- DataTableBuilderFactory.setDataTableVersion(DataTableFactory.VERSION_4);
// Null literal only
String sqlQuery = "SELECT null FROM mytable OPTION(enableNullHandling=true)";
JsonNode response = postQuery(sqlQuery, _brokerBaseApiUrl);
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
index 1a77e10929..d4920c2c15 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
@@ -49,6 +49,7 @@ import org.apache.http.message.BasicHeader;
import org.apache.http.message.BasicNameValuePair;
import org.apache.pinot.client.PinotConnection;
import org.apache.pinot.client.PinotDriver;
+import org.apache.pinot.common.datatable.DataTableFactory;
import org.apache.pinot.common.exception.HttpErrorStatusException;
import org.apache.pinot.common.exception.QueryException;
import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
@@ -56,6 +57,7 @@ import org.apache.pinot.common.utils.FileUploadDownloadClient;
import org.apache.pinot.common.utils.ServiceStatus;
import org.apache.pinot.common.utils.SimpleHttpResponse;
import org.apache.pinot.common.utils.http.HttpClient;
+import org.apache.pinot.core.common.datatable.DataTableBuilderFactory;
import org.apache.pinot.core.operator.query.NonScanBasedAggregationOperator;
import org.apache.pinot.segment.spi.index.startree.AggregationFunctionColumnPair;
import org.apache.pinot.spi.config.instance.InstanceType;
@@ -165,6 +167,7 @@ public class OfflineClusterIntegrationTest extends BaseClusterIntegrationTestSet
@BeforeClass
public void setUp()
throws Exception {
+ DataTableBuilderFactory.setDataTableVersion(DataTableFactory.VERSION_3);
TestUtils.ensureDirectoriesExistAndEmpty(_tempDir, _segmentDir, _tarDir);
// Start the Pinot cluster
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/SSBQueryIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/SSBQueryIntegrationTest.java
index 5776f2c82b..5bbd692aad 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/SSBQueryIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/SSBQueryIntegrationTest.java
@@ -32,13 +32,8 @@ import org.apache.commons.io.FileUtils;
import org.apache.pinot.client.Connection;
import org.apache.pinot.client.ConnectionFactory;
import org.apache.pinot.client.ResultSetGroup;
-import org.apache.pinot.common.datatable.DataTableFactory;
-import org.apache.pinot.core.common.datatable.DataTableBuilderFactory;
-import org.apache.pinot.query.service.QueryConfig;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.data.Schema;
-import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.tools.utils.JarUtils;
import org.apache.pinot.util.TestUtils;
import org.slf4j.Logger;
@@ -103,9 +98,6 @@ public class SSBQueryIntegrationTest extends BaseClusterIntegrationTest {
// H2
ClusterIntegrationTestUtils.setUpH2TableWithAvro(Collections.singletonList(dataFile), tableName, _h2Connection);
}
-
- // Setting data table version to 4
- DataTableBuilderFactory.setDataTableVersion(DataTableFactory.VERSION_4);
}
@Test(dataProvider = "QueryDataProvider")
@@ -170,19 +162,6 @@ public class SSBQueryIntegrationTest extends BaseClusterIntegrationTest {
return _pinotConnection;
}
- @Override
- protected void overrideBrokerConf(PinotConfiguration brokerConf) {
- brokerConf.setProperty(CommonConstants.Helix.CONFIG_OF_MULTI_STAGE_ENGINE_ENABLED, true);
- brokerConf.setProperty(QueryConfig.KEY_OF_QUERY_RUNNER_PORT, 8421);
- }
-
- @Override
- protected void overrideServerConf(PinotConfiguration serverConf) {
- serverConf.setProperty(CommonConstants.Helix.CONFIG_OF_MULTI_STAGE_ENGINE_ENABLED, true);
- serverConf.setProperty(QueryConfig.KEY_OF_QUERY_SERVER_PORT, 8842);
- serverConf.setProperty(QueryConfig.KEY_OF_QUERY_RUNNER_PORT, 8422);
- }
-
@AfterClass
public void tearDown()
throws Exception {
diff --git a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/QueryRunnerTest.java b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/QueryRunnerTest.java
index de8cc2288a..649a2bee05 100644
--- a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/QueryRunnerTest.java
+++ b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/QueryRunnerTest.java
@@ -26,7 +26,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
-import org.apache.pinot.common.datatable.DataTableFactory;
import org.apache.pinot.core.common.datatable.DataTableBuilderFactory;
import org.apache.pinot.query.QueryEnvironmentTestBase;
import org.apache.pinot.query.QueryServerEnclosure;
@@ -95,7 +94,6 @@ public class QueryRunnerTest extends QueryRunnerTestBase {
@BeforeClass
public void setUp()
throws Exception {
- DataTableBuilderFactory.setDataTableVersion(DataTableFactory.VERSION_4);
MockInstanceDataManagerFactory factory1 = new MockInstanceDataManagerFactory("server1")
.registerTable(SCHEMA_BUILDER.setSchemaName("a").build(), "a_REALTIME")
.registerTable(SCHEMA_BUILDER.setSchemaName("b").build(), "b_REALTIME")
diff --git a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/queries/ResourceBasedQueriesTest.java b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/queries/ResourceBasedQueriesTest.java
index a38d7e083a..4911f3b57f 100644
--- a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/queries/ResourceBasedQueriesTest.java
+++ b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/queries/ResourceBasedQueriesTest.java
@@ -38,7 +38,6 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.pinot.common.datatable.DataTable;
-import org.apache.pinot.common.datatable.DataTableFactory;
import org.apache.pinot.common.response.broker.BrokerResponseNativeV2;
import org.apache.pinot.common.response.broker.BrokerResponseStats;
import org.apache.pinot.core.common.datatable.DataTableBuilderFactory;
@@ -75,8 +74,6 @@ public class ResourceBasedQueriesTest extends QueryRunnerTestBase {
@BeforeClass
public void setUp()
throws Exception {
- DataTableBuilderFactory.setDataTableVersion(DataTableFactory.VERSION_4);
-
// Setting up mock server factories.
// All test data are loaded upfront b/c the mock server and brokers needs to be in sync.
MockInstanceDataManagerFactory factory1 = new MockInstanceDataManagerFactory("server1");
diff --git a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/BaseServerStarter.java b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/BaseServerStarter.java
index e9fdab9511..0f77155dbe 100644
--- a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/BaseServerStarter.java
+++ b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/BaseServerStarter.java
@@ -68,6 +68,7 @@ import org.apache.pinot.core.data.manager.InstanceDataManager;
import org.apache.pinot.core.data.manager.realtime.RealtimeConsumptionRateManager;
import org.apache.pinot.core.transport.ListenerConfig;
import org.apache.pinot.core.util.ListenerConfigUtil;
+import org.apache.pinot.query.service.QueryConfig;
import org.apache.pinot.segment.local.realtime.impl.invertedindex.RealtimeLuceneIndexRefreshState;
import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.server.access.AccessControlFactory;
@@ -170,6 +171,12 @@ public abstract class BaseServerStarter implements ServiceStartable {
// NOTE: Need to add the instance id to the config because it is required in HelixInstanceDataManagerConfig
_serverConf.addProperty(Server.CONFIG_OF_INSTANCE_ID, _instanceId);
}
+ if (_serverConf.getProperty(QueryConfig.KEY_OF_QUERY_SERVER_PORT, QueryConfig.DEFAULT_QUERY_SERVER_PORT) == 0) {
+ _serverConf.setProperty(QueryConfig.KEY_OF_QUERY_SERVER_PORT, NetUtils.findOpenPort());
+ }
+ if (_serverConf.getProperty(QueryConfig.KEY_OF_QUERY_RUNNER_PORT, QueryConfig.DEFAULT_QUERY_RUNNER_PORT) == 0) {
+ _serverConf.setProperty(QueryConfig.KEY_OF_QUERY_RUNNER_PORT, NetUtils.findOpenPort());
+ }
_instanceConfigScope =
new HelixConfigScopeBuilder(ConfigScopeProperty.PARTICIPANT, _helixClusterName).forParticipant(_instanceId)
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
index 46b0474d26..7a84ef257b 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
@@ -194,7 +194,7 @@ public class CommonConstants {
public static final String CONFIG_OF_PINOT_MINION_STARTABLE_CLASS = "pinot.minion.startable.class";
public static final String CONFIG_OF_MULTI_STAGE_ENGINE_ENABLED = "pinot.multistage.engine.enabled";
- public static final boolean DEFAULT_MULTI_STAGE_ENGINE_ENABLED = false;
+ public static final boolean DEFAULT_MULTI_STAGE_ENGINE_ENABLED = true;
}
public static class Broker {
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/MultistageEngineQuickStart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/MultistageEngineQuickStart.java
index 690e99990b..226f3dc518 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/MultistageEngineQuickStart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/MultistageEngineQuickStart.java
@@ -21,7 +21,6 @@ package org.apache.pinot.tools;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.pinot.spi.utils.CommonConstants;
@@ -91,14 +90,6 @@ public class MultistageEngineQuickStart extends Quickstart {
return MULTI_STAGE_TABLE_DIRECTORIES;
}
- @Override
- public Map<String, Object> getConfigOverrides() {
- Map<String, Object> overrides = new HashMap<>(super.getConfigOverrides());
- overrides.put("pinot.multistage.engine.enabled", "true");
- overrides.put("pinot.server.instance.currentDataTableVersion", 4);
- return overrides;
- }
-
@Override
protected int getNumQuickstartRunnerServers() {
return 3;
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
index cac14a4239..e1e80e8ae0 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
@@ -55,10 +55,6 @@ public class QuickstartRunner {
private static final int DEFAULT_SERVER_GRPC_PORT = 7100;
private static final int DEFAULT_MINION_PORT = 6000;
- private static final int DEFAULT_BROKER_MULTISTAGE_RUNNER_PORT = 8421;
- private static final int DEFAULT_SERVER_MULTISTAGE_RUNNER_PORT = 8442;
- private static final int DEFAULT_SERVER_MULTISTAGE_SERVER_PORT = 8842;
-
private static final String DEFAULT_ZK_DIR = "PinotZkDir";
private static final String DEFAULT_CONTROLLER_DIR = "PinotControllerDir";
private static final String DEFAULT_SERVER_DATA_DIR = "PinotServerDataDir";
@@ -143,7 +139,6 @@ public class QuickstartRunner {
for (int i = 0; i < _numBrokers; i++) {
StartBrokerCommand brokerStarter = new StartBrokerCommand();
brokerStarter.setPort(DEFAULT_BROKER_PORT + i)
- .setBrokerMultiStageRunnerPort(DEFAULT_BROKER_MULTISTAGE_RUNNER_PORT + i)
.setZkAddress(_zkExternalAddress != null ? _zkExternalAddress : ZK_ADDRESS).setClusterName(CLUSTER_NAME)
.setConfigOverrides(_configOverrides);
if (!brokerStarter.execute()) {
@@ -159,8 +154,6 @@ public class QuickstartRunner {
StartServerCommand serverStarter = new StartServerCommand();
serverStarter.setPort(DEFAULT_SERVER_NETTY_PORT + i).setAdminPort(DEFAULT_SERVER_ADMIN_API_PORT + i)
.setGrpcPort(DEFAULT_SERVER_GRPC_PORT + i)
- .setMultiStageServerPort(DEFAULT_SERVER_MULTISTAGE_SERVER_PORT + i)
- .setMultiStageRunnerPort(DEFAULT_SERVER_MULTISTAGE_RUNNER_PORT + i)
.setZkAddress(_zkExternalAddress != null ? _zkExternalAddress : ZK_ADDRESS).setClusterName(CLUSTER_NAME)
.setDataDir(new File(_tempDir, DEFAULT_SERVER_DATA_DIR + i).getAbsolutePath())
.setSegmentDir(new File(_tempDir, DEFAULT_SERVER_SEGMENT_DIR + i).getAbsolutePath())
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org