You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ri...@apache.org on 2022/02/15 17:10:57 UTC
[pinot] branch master updated: Enable service discovery in controller too (#8193)
This is an automated email from the ASF dual-hosted git repository.
richardstartin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 561621b Enable service discovery in controller too (#8193)
561621b is described below
commit 561621babac7f91a543aee754e0081cf1bce8218
Author: Xiaoman Dong <xi...@startree.ai>
AuthorDate: Tue Feb 15 09:09:40 2022 -0800
Enable service discovery in controller too (#8193)
* enable service discovery in controller too
* fix spotless check
* extra test to show that initialization are needed twice for two services
* fix checkstyle
---
.../broker/broker/BrokerAdminApiApplication.java | 3 ++-
.../api/ControllerAdminApiApplication.java | 4 ++++
pinot-core/pom.xml | 4 ++++
.../pinot/core/api/ServiceAutoDiscoveryFeature.java | 7 ++++---
...covery.java => BrokerEchoWithAutoDiscovery.java} | 6 +++---
.../resources/ControllerEchoWithAutoDiscovery.java} | 9 +++++----
.../api/AutoLoadedServiceForTest.java} | 4 ++--
.../BrokerServiceDiscoveryIntegrationTest.java | 2 +-
... ControllerServiceDiscoveryIntegrationTest.java} | 21 ++++++++++++++++-----
.../org/apache/pinot/spi/utils/CommonConstants.java | 2 ++
10 files changed, 43 insertions(+), 19 deletions(-)
diff --git a/pinot-broker/src/main/java/org/apache/pinot/broker/broker/BrokerAdminApiApplication.java b/pinot-broker/src/main/java/org/apache/pinot/broker/broker/BrokerAdminApiApplication.java
index b0fd1d2..75f01ce 100644
--- a/pinot-broker/src/main/java/org/apache/pinot/broker/broker/BrokerAdminApiApplication.java
+++ b/pinot-broker/src/main/java/org/apache/pinot/broker/broker/BrokerAdminApiApplication.java
@@ -26,6 +26,7 @@ import java.util.List;
import org.apache.pinot.broker.requesthandler.BrokerRequestHandler;
import org.apache.pinot.broker.routing.RoutingManager;
import org.apache.pinot.common.metrics.BrokerMetrics;
+import org.apache.pinot.core.api.ServiceAutoDiscoveryFeature;
import org.apache.pinot.core.transport.ListenerConfig;
import org.apache.pinot.core.util.ListenerConfigUtil;
import org.apache.pinot.spi.env.PinotConfiguration;
@@ -51,7 +52,7 @@ public class BrokerAdminApiApplication extends ResourceConfig {
packages(RESOURCE_PACKAGE);
property(PINOT_CONFIGURATION, brokerConf);
if (brokerConf.getProperty(CommonConstants.Broker.BROKER_SERVICE_AUTO_DISCOVERY, false)) {
- register(BrokerServiceAutoDiscoveryFeature.class);
+ register(ServiceAutoDiscoveryFeature.class);
}
register(new AbstractBinder() {
@Override
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/ControllerAdminApiApplication.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/ControllerAdminApiApplication.java
index 8b2e69e..3f060de 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/ControllerAdminApiApplication.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/ControllerAdminApiApplication.java
@@ -28,6 +28,7 @@ import javax.ws.rs.container.ContainerResponseContext;
import javax.ws.rs.container.ContainerResponseFilter;
import org.apache.pinot.controller.ControllerConf;
import org.apache.pinot.controller.api.access.AuthenticationFilter;
+import org.apache.pinot.core.api.ServiceAutoDiscoveryFeature;
import org.apache.pinot.core.transport.ListenerConfig;
import org.apache.pinot.core.util.ListenerConfigUtil;
import org.apache.pinot.spi.utils.CommonConstants;
@@ -57,6 +58,9 @@ public class ControllerAdminApiApplication extends ResourceConfig {
packages(_controllerResourcePackages);
// TODO See ControllerResponseFilter
// register(new LoggingFeature());
+ if (conf.getProperty(CommonConstants.Controller.CONTROLLER_SERVICE_AUTO_DISCOVERY, false)) {
+ register(ServiceAutoDiscoveryFeature.class);
+ }
register(JacksonFeature.class);
register(MultiPartFeature.class);
registerClasses(io.swagger.jaxrs.listing.ApiListingResource.class);
diff --git a/pinot-core/pom.xml b/pinot-core/pom.xml
index 244fa30..90bb06e 100644
--- a/pinot-core/pom.xml
+++ b/pinot-core/pom.xml
@@ -179,6 +179,10 @@
<groupId>org.glassfish.grizzly</groupId>
<artifactId>grizzly-http-server</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.glassfish.hk2</groupId>
+ <artifactId>hk2-locator</artifactId>
+ </dependency>
<!-- test -->
<dependency>
diff --git a/pinot-broker/src/main/java/org/apache/pinot/broker/broker/BrokerServiceAutoDiscoveryFeature.java b/pinot-core/src/main/java/org/apache/pinot/core/api/ServiceAutoDiscoveryFeature.java
similarity index 95%
rename from pinot-broker/src/main/java/org/apache/pinot/broker/broker/BrokerServiceAutoDiscoveryFeature.java
rename to pinot-core/src/main/java/org/apache/pinot/core/api/ServiceAutoDiscoveryFeature.java
index f1a2d2d..458aa4d 100644
--- a/pinot-broker/src/main/java/org/apache/pinot/broker/broker/BrokerServiceAutoDiscoveryFeature.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/api/ServiceAutoDiscoveryFeature.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.broker.broker;
+package org.apache.pinot.core.api;
import java.io.IOException;
import javax.inject.Inject;
@@ -31,6 +31,7 @@ import org.glassfish.hk2.utilities.DuplicatePostProcessor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
/**
* Auto scan and discovery classes annotated with Service tags. This enables the feature similar to Spring's
* auto-discovery, that if a class is annotated with some tags like @Service, the class will be auto-loaded
@@ -68,8 +69,8 @@ import org.slf4j.LoggerFactory;
* pinot-integration-tests/src/main/java/org/apache/pinot/broker/integration/tests/BrokerTestAutoLoadedService.java
* </code>
*/
-public class BrokerServiceAutoDiscoveryFeature implements Feature {
- private static final Logger LOGGER = LoggerFactory.getLogger(BrokerServiceAutoDiscoveryFeature.class);
+public class ServiceAutoDiscoveryFeature implements Feature {
+ private static final Logger LOGGER = LoggerFactory.getLogger(ServiceAutoDiscoveryFeature.class);
@Inject
ServiceLocator _serviceLocator;
diff --git a/pinot-integration-tests/src/main/java/org/apache/pinot/broker/api/resources/EchoWithAutoDiscovery.java b/pinot-integration-tests/src/main/java/org/apache/pinot/broker/api/resources/BrokerEchoWithAutoDiscovery.java
similarity index 89%
copy from pinot-integration-tests/src/main/java/org/apache/pinot/broker/api/resources/EchoWithAutoDiscovery.java
copy to pinot-integration-tests/src/main/java/org/apache/pinot/broker/api/resources/BrokerEchoWithAutoDiscovery.java
index 6d67872..445048f 100644
--- a/pinot-integration-tests/src/main/java/org/apache/pinot/broker/api/resources/EchoWithAutoDiscovery.java
+++ b/pinot-integration-tests/src/main/java/org/apache/pinot/broker/api/resources/BrokerEchoWithAutoDiscovery.java
@@ -26,7 +26,7 @@ import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
-import org.apache.pinot.broker.integration.tests.BrokerTestAutoLoadedService;
+import org.apache.pinot.core.api.AutoLoadedServiceForTest;
/**
* This class is a typical "echo" service that will return whatever string you call GET with a path.
@@ -35,9 +35,9 @@ import org.apache.pinot.broker.integration.tests.BrokerTestAutoLoadedService;
*/
@Api(tags = "Test")
@Path("/test")
-public class EchoWithAutoDiscovery {
+public class BrokerEchoWithAutoDiscovery {
@Inject
- public BrokerTestAutoLoadedService _injectedService;
+ public AutoLoadedServiceForTest _injectedService;
@GET
@Path("/echo/{table}")
@Produces(MediaType.TEXT_PLAIN)
diff --git a/pinot-integration-tests/src/main/java/org/apache/pinot/broker/api/resources/EchoWithAutoDiscovery.java b/pinot-integration-tests/src/main/java/org/apache/pinot/controller/api/resources/ControllerEchoWithAutoDiscovery.java
similarity index 87%
rename from pinot-integration-tests/src/main/java/org/apache/pinot/broker/api/resources/EchoWithAutoDiscovery.java
rename to pinot-integration-tests/src/main/java/org/apache/pinot/controller/api/resources/ControllerEchoWithAutoDiscovery.java
index 6d67872..66248c3 100644
--- a/pinot-integration-tests/src/main/java/org/apache/pinot/broker/api/resources/EchoWithAutoDiscovery.java
+++ b/pinot-integration-tests/src/main/java/org/apache/pinot/controller/api/resources/ControllerEchoWithAutoDiscovery.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.pinot.broker.api.resources;
+package org.apache.pinot.controller.api.resources;
import io.swagger.annotations.Api;
import javax.inject.Inject;
@@ -26,7 +26,8 @@ import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
-import org.apache.pinot.broker.integration.tests.BrokerTestAutoLoadedService;
+import org.apache.pinot.core.api.AutoLoadedServiceForTest;
+
/**
* This class is a typical "echo" service that will return whatever string you call GET with a path.
@@ -35,9 +36,9 @@ import org.apache.pinot.broker.integration.tests.BrokerTestAutoLoadedService;
*/
@Api(tags = "Test")
@Path("/test")
-public class EchoWithAutoDiscovery {
+public class ControllerEchoWithAutoDiscovery {
@Inject
- public BrokerTestAutoLoadedService _injectedService;
+ public AutoLoadedServiceForTest _injectedService;
@GET
@Path("/echo/{table}")
@Produces(MediaType.TEXT_PLAIN)
diff --git a/pinot-integration-tests/src/main/java/org/apache/pinot/broker/integration/tests/BrokerTestAutoLoadedService.java b/pinot-integration-tests/src/main/java/org/apache/pinot/core/api/AutoLoadedServiceForTest.java
similarity index 91%
rename from pinot-integration-tests/src/main/java/org/apache/pinot/broker/integration/tests/BrokerTestAutoLoadedService.java
rename to pinot-integration-tests/src/main/java/org/apache/pinot/core/api/AutoLoadedServiceForTest.java
index fa0434b..f886176 100644
--- a/pinot-integration-tests/src/main/java/org/apache/pinot/broker/integration/tests/BrokerTestAutoLoadedService.java
+++ b/pinot-integration-tests/src/main/java/org/apache/pinot/core/api/AutoLoadedServiceForTest.java
@@ -16,14 +16,14 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.broker.integration.tests;
+package org.apache.pinot.core.api;
import javax.inject.Singleton;
import org.jvnet.hk2.annotations.Service;
@Service
@Singleton
-public class BrokerTestAutoLoadedService {
+public class AutoLoadedServiceForTest {
public String echo(String echoText) {
return echoText;
}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BrokerServiceDiscoveryIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BrokerServiceDiscoveryIntegrationTest.java
index da93bad..207d2f6 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BrokerServiceDiscoveryIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BrokerServiceDiscoveryIntegrationTest.java
@@ -28,7 +28,7 @@ import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
/**
- * Integration test that converts Avro data for 12 segments and runs queries against it.
+ * Integration test that starts one broker with auto-discovered echo service and test it
*/
public class BrokerServiceDiscoveryIntegrationTest extends BaseClusterIntegrationTestSet {
private static final String TENANT_NAME = "TestTenant";
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BrokerServiceDiscoveryIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/ControllerServiceDiscoveryIntegrationTest.java
similarity index 75%
copy from pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BrokerServiceDiscoveryIntegrationTest.java
copy to pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/ControllerServiceDiscoveryIntegrationTest.java
index da93bad..72fc961 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BrokerServiceDiscoveryIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/ControllerServiceDiscoveryIntegrationTest.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.integration.tests;
+import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.utils.CommonConstants;
@@ -27,10 +28,11 @@ import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
+
/**
- * Integration test that converts Avro data for 12 segments and runs queries against it.
+ * Integration test that starts one broker with auto-discovered echo service and test it
*/
-public class BrokerServiceDiscoveryIntegrationTest extends BaseClusterIntegrationTestSet {
+public class ControllerServiceDiscoveryIntegrationTest extends BaseClusterIntegrationTestSet {
private static final String TENANT_NAME = "TestTenant";
@Override
@@ -43,6 +45,14 @@ public class BrokerServiceDiscoveryIntegrationTest extends BaseClusterIntegratio
return TENANT_NAME;
}
+ @Override
+ public Map<String, Object> getDefaultControllerConfiguration() {
+ Map<String, Object> retVal = super.getDefaultControllerConfiguration();
+ retVal.put(CommonConstants.Controller.CONTROLLER_SERVICE_AUTO_DISCOVERY, true);
+ return retVal;
+ }
+
+ @Override
protected PinotConfiguration getDefaultBrokerConfiguration() {
PinotConfiguration config = new PinotConfiguration();
config.setProperty(CommonConstants.Broker.BROKER_SERVICE_AUTO_DISCOVERY, true);
@@ -60,7 +70,6 @@ public class BrokerServiceDiscoveryIntegrationTest extends BaseClusterIntegratio
startBrokers(1);
startServers(1);
}
-
@AfterClass
public void tearDown()
throws Exception {
@@ -73,9 +82,11 @@ public class BrokerServiceDiscoveryIntegrationTest extends BaseClusterIntegratio
}
@Test
- public void testBrokerExtraEndpointsAutoLoaded()
+ public void testControllerExtraEndpointsAutoLoaded()
throws Exception {
- String response = sendGetRequest(_brokerBaseApiUrl + "/test/echo/doge");
+ String response = sendGetRequest(_controllerBaseApiUrl + "/test/echo/doge");
+ Assert.assertEquals(response, "doge");
+ response = sendGetRequest(_brokerBaseApiUrl + "/test/echo/doge");
Assert.assertEquals(response, "doge");
}
}
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
index bf2922b..4a8c774 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
@@ -442,6 +442,8 @@ public class CommonConstants {
public static final String CONFIG_OF_INSTANCE_ID = "pinot.controller.instance.id";
public static final String CONFIG_OF_CONTROLLER_QUERY_REWRITER_CLASS_NAMES =
"pinot.controller.query.rewriter.class.names";
+ //Set to true to load all services tagged and compiled with hk2-metadata-generator. Default to False
+ public static final String CONTROLLER_SERVICE_AUTO_DISCOVERY = "pinot.controller.service.auto.discovery";
}
public static class Minion {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org