You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by ji...@apache.org on 2020/04/23 19:27:46 UTC
[helix] branch master updated (9106e88 -> 4c65bfc)
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git.
from 9106e88 Fix routing data refreshing in MSDS (#955)
new 4ece90a Add the CustomizedView Helix property (#723)
new 2189de7 add CustomizedStateAggregation config (#776)
new f0eb926 Add java API to add or remove CustomizedStateAggregationConfig (#792)
new 7175a56 Add REST API to add, remove and update CustomizedStateAggregationConfig (#797)
new f4e3a93 Implement Helix API for updating customized state (#729)
new 84d0fe7 Add basic functionalities for RoutingTableProvider for CustomizedView (#814)
new ca7af92 add listener and config for customized view aggregation (#815)
new 5074c15 Improve CustomizedStateProvider tests (#840)
new 55f3b60 Add intermediate storage for customized state (#827)
new 21a47c5 rename custmized state aggregation config to customized state config (#885)
new d6fd362 Use updater to update customized state for concurrency control (#859)
new eeeab15 update cache functions for customized view aggregation (#887)
new cacbd85 Add two stages for customized state view aggregation. (#888)
new cae7afe Complete the Routing Table Provider for CustomizedView (#834)
new fc3dfdc Add new stages in Helix generic controller for customized view aggregation. (#851)
new fc78947 minor fix for customized view aggregation (#917)
new 9c40583 Replace customized view cache with property cache (#869)
new 1c26122 Add integration test to customized view aggregation (#912)
new 15ad8a0 fix customized state provider (#928)
new a23445f Add cache update/delete in customized view aggregation stage (#934)
new 36ba93f Add registration logic for CustomizedView listeners (#944)
new 25e4726 Move routing table provider initialization (#946)
new 4c65bfc use new ZNRecord and update test
The 23 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../main/java/org/apache/helix/ConfigAccessor.java | 22 +
.../src/main/java/org/apache/helix/HelixAdmin.java | 26 +
.../main/java/org/apache/helix/HelixConstants.java | 5 +
.../main/java/org/apache/helix/HelixManager.java | 46 +-
.../main/java/org/apache/helix/PropertyKey.java | 76 +++
.../java/org/apache/helix/PropertyPathBuilder.java | 52 +-
.../main/java/org/apache/helix/PropertyType.java | 5 +-
...ner.java => CustomizedStateChangeListener.java} | 18 +-
...va => CustomizedStateConfigChangeListener.java} | 18 +-
...java => CustomizedStateRootChangeListener.java} | 18 +-
...ener.java => CustomizedViewChangeListener.java} | 12 +-
....java => CustomizedViewRootChangeListener.java} | 17 +-
.../helix/common/caches/CurrentStateCache.java | 149 +----
.../helix/common/caches/CustomizedStateCache.java | 67 +++
.../helix/common/caches/CustomizedViewCache.java | 97 ++++
.../helix/common/caches/ParticipantStateCache.java | 184 ++++++
.../helix/controller/GenericHelixController.java | 166 +++++-
.../dataproviders/BaseControllerDataProvider.java | 10 +-
.../ResourceControllerDataProvider.java | 126 +++++
.../helix/controller/pipeline/AsyncWorkerType.java | 3 +-
.../helix/controller/stages/AttributeName.java | 1 +
.../helix/controller/stages/ClusterEventType.java | 3 +
.../stages/CustomizedStateComputationStage.java | 93 ++++
.../controller/stages/CustomizedStateOutput.java | 120 ++++
.../stages/CustomizedViewAggregationStage.java | 157 ++++++
.../customizedstate/CustomizedStateProvider.java | 118 ++++
.../CustomizedStateProviderFactory.java | 70 +++
.../apache/helix/manager/zk/CallbackHandler.java | 217 +++++---
.../helix/manager/zk/ControllerManagerHelper.java | 2 +
.../org/apache/helix/manager/zk/ZKHelixAdmin.java | 85 +++
.../apache/helix/manager/zk/ZKHelixManager.java | 46 ++
.../org/apache/helix/model/CustomizedState.java | 155 ++++++
.../apache/helix/model/CustomizedStateConfig.java | 146 +++++
.../org/apache/helix/model/CustomizedView.java | 99 ++++
.../org/apache/helix/model/HelixConfigScope.java | 8 +-
.../model/builder/HelixConfigScopeBuilder.java | 3 +
.../spectator/CustomizedViewRoutingTable.java | 112 ++++
.../apache/helix/spectator/RoutingDataCache.java | 118 ++--
.../org/apache/helix/spectator/RoutingTable.java | 52 +-
.../helix/spectator/RoutingTableProvider.java | 614 +++++++++++++++------
.../helix/spectator/RoutingTableSnapshot.java | 37 ++
.../org/apache/helix/TestPropertyPathBuilder.java | 5 +
.../test/java/org/apache/helix/TestZKCallback.java | 29 +-
.../controller/stages/DummyClusterManager.java | 37 ++
.../TestCustomizedStateComputationStage.java | 102 ++++
...ViewStage.java => TestCustomizedViewStage.java} | 63 ++-
.../TestComputeAndCleanupCustomizedView.java | 291 ++++++++++
.../integration/TestCustomizedViewAggregation.java | 449 +++++++++++++++
.../integration/TestZkCallbackHandlerLeak.java | 16 +-
.../TestControllerDataProviderSelectiveUpdate.java | 4 +-
.../manager/TestParticipantManager.java | 2 +-
.../paticipant/TestCustomizedStateUpdate.java | 319 +++++++++++
.../spectator/TestRoutingTableProvider.java | 164 ++++++
.../TestRoutingTableProviderPeriodicRefresh.java | 26 +-
.../apache/helix/manager/zk/TestZkHelixAdmin.java | 101 +++-
.../java/org/apache/helix/mock/MockHelixAdmin.java | 29 +
.../java/org/apache/helix/mock/MockManager.java | 36 ++
.../helix/model/TestCustomizedStateConfig.java | 125 +++++
.../helix/participant/MockZKHelixManager.java | 36 ++
.../rest/server/resources/AbstractResource.java | 1 +
.../server/resources/helix/ClusterAccessor.java | 109 ++++
.../helix/rest/server/TestClusterAccessor.java | 155 +++++-
62 files changed, 4944 insertions(+), 528 deletions(-)
copy helix-core/src/main/java/org/apache/helix/api/listeners/{CurrentStateChangeListener.java => CustomizedStateChangeListener.java} (71%)
copy helix-core/src/main/java/org/apache/helix/api/listeners/{InstanceConfigChangeListener.java => CustomizedStateConfigChangeListener.java} (69%)
copy helix-core/src/main/java/org/apache/helix/api/listeners/{CurrentStateChangeListener.java => CustomizedStateRootChangeListener.java} (69%)
copy helix-core/src/main/java/org/apache/helix/api/listeners/{ExternalViewChangeListener.java => CustomizedViewChangeListener.java} (75%)
copy helix-core/src/main/java/org/apache/helix/api/listeners/{LiveInstanceChangeListener.java => CustomizedViewRootChangeListener.java} (72%)
create mode 100644 helix-core/src/main/java/org/apache/helix/common/caches/CustomizedStateCache.java
create mode 100644 helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java
create mode 100644 helix-core/src/main/java/org/apache/helix/common/caches/ParticipantStateCache.java
create mode 100644 helix-core/src/main/java/org/apache/helix/controller/stages/CustomizedStateComputationStage.java
create mode 100644 helix-core/src/main/java/org/apache/helix/controller/stages/CustomizedStateOutput.java
create mode 100644 helix-core/src/main/java/org/apache/helix/controller/stages/CustomizedViewAggregationStage.java
create mode 100644 helix-core/src/main/java/org/apache/helix/customizedstate/CustomizedStateProvider.java
create mode 100644 helix-core/src/main/java/org/apache/helix/customizedstate/CustomizedStateProviderFactory.java
create mode 100644 helix-core/src/main/java/org/apache/helix/model/CustomizedState.java
create mode 100644 helix-core/src/main/java/org/apache/helix/model/CustomizedStateConfig.java
create mode 100644 helix-core/src/main/java/org/apache/helix/model/CustomizedView.java
create mode 100644 helix-core/src/main/java/org/apache/helix/spectator/CustomizedViewRoutingTable.java
create mode 100644 helix-core/src/test/java/org/apache/helix/controller/stages/TestCustomizedStateComputationStage.java
copy helix-core/src/test/java/org/apache/helix/controller/stages/{TestExternalViewStage.java => TestCustomizedViewStage.java} (54%)
create mode 100644 helix-core/src/test/java/org/apache/helix/integration/TestComputeAndCleanupCustomizedView.java
create mode 100644 helix-core/src/test/java/org/apache/helix/integration/TestCustomizedViewAggregation.java
create mode 100644 helix-core/src/test/java/org/apache/helix/integration/paticipant/TestCustomizedStateUpdate.java
create mode 100644 helix-core/src/test/java/org/apache/helix/model/TestCustomizedStateConfig.java
[helix] 04/23: Add REST API to add,
remove and update CustomizedStateAggregationConfig (#797)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 7175a56645e66d421b93f1fc6bae90a27d43b1a7
Author: Ali Reza Zamani Zadeh Najari <an...@linkedin.com>
AuthorDate: Tue Feb 25 14:05:10 2020 -0800
Add REST API to add, remove and update CustomizedStateAggregationConfig (#797)
In this commit the below REST APIs have been added.
1- addCustomizedStateAggregationConfig
2- removeCustomizedStateAggregationConfig
3- getCustomizedStateAggregationConfig
4- updateCustomizedStateAggregationConfig
Tests have been added to check the functionality of these REST APIs.
Also some of the depricated calls have been updated.
---
.../apache/helix/manager/zk/TestZkHelixAdmin.java | 1 +
.../TestCustomizedStateAggregationConfig.java | 12 +-
.../rest/server/resources/AbstractResource.java | 1 +
.../server/resources/helix/ClusterAccessor.java | 109 +++++++++++++++
.../helix/rest/server/TestClusterAccessor.java | 147 ++++++++++++++++++++-
5 files changed, 262 insertions(+), 8 deletions(-)
diff --git a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
index 61eb9e2..da2145b 100644
--- a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
+++ b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
@@ -600,6 +600,7 @@ public class TestZkHelixAdmin extends ZkUnitTestBase {
// OK since resourceConfig is empty
}
+<<<<<<< HEAD
// Set PARTITION_CAPACITY_MAP
Map<String, String> capacityDataMap =
ImmutableMap.of("WCU", "1", "RCU", "2", "STORAGE", "3");
diff --git a/helix-core/src/test/java/org/apache/helix/model/TestCustomizedStateAggregationConfig.java b/helix-core/src/test/java/org/apache/helix/model/TestCustomizedStateAggregationConfig.java
index 2b9752a..bda6a50 100644
--- a/helix-core/src/test/java/org/apache/helix/model/TestCustomizedStateAggregationConfig.java
+++ b/helix-core/src/test/java/org/apache/helix/model/TestCustomizedStateAggregationConfig.java
@@ -38,7 +38,7 @@ public class TestCustomizedStateAggregationConfig extends ZkUnitTestBase {
String className = getShortClassName();
String clusterName = "CLUSTER_" + className;
// Read CustomizedStateAggregationConfig from Zookeeper and get exception since cluster in not setup yet
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CustomizedStateAggregationConfig customizedStateAggregationConfig =
_configAccessor.getCustomizedStateAggregationConfig(clusterName);
}
@@ -49,7 +49,7 @@ public class TestCustomizedStateAggregationConfig extends ZkUnitTestBase {
String clusterName = "CLUSTER_" + className;
TestHelper.setupEmptyCluster(_gZkClient, clusterName);
// Read CustomizedStateAggregationConfig from Zookeeper
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CustomizedStateAggregationConfig customizedStateAggregationConfigFromZk =
_configAccessor.getCustomizedStateAggregationConfig(clusterName);
Assert.assertNull(customizedStateAggregationConfigFromZk);
@@ -73,13 +73,13 @@ public class TestCustomizedStateAggregationConfig extends ZkUnitTestBase {
// Write the CustomizedStateAggregationConfig to Zookeeper
ZKHelixDataAccessor accessor =
- new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor(_gZkClient));
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor(ZK_ADDR));
Builder keyBuilder = accessor.keyBuilder();
accessor.setProperty(keyBuilder.customizedStateAggregationConfig(),
customizedStateAggregationConfig);
// Read CustomizedStateAggregationConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CustomizedStateAggregationConfig customizedStateAggregationConfigFromZk =
_configAccessor.getCustomizedStateAggregationConfig(clusterName);
Assert.assertEquals(customizedStateAggregationConfigFromZk.getAggregationEnabledTypes().size(),
@@ -107,13 +107,13 @@ public class TestCustomizedStateAggregationConfig extends ZkUnitTestBase {
CustomizedStateAggregationConfig customizedStateAggregationConfig = builder.build();
ZKHelixDataAccessor accessor =
- new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor(_gZkClient));
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor(ZK_ADDR));
Builder keyBuilder = accessor.keyBuilder();
accessor.setProperty(keyBuilder.customizedStateAggregationConfig(),
customizedStateAggregationConfig);
// Read CustomizedStateAggregationConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CustomizedStateAggregationConfig customizedStateAggregationConfigFromZk =
_configAccessor.getCustomizedStateAggregationConfig(clusterName);
List<String> aggregationEnabledTypesFromZk =
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/AbstractResource.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/AbstractResource.java
index 8e47b77..1b9111e 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/AbstractResource.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/AbstractResource.java
@@ -62,6 +62,7 @@ public class AbstractResource {
enablePartitions,
disablePartitions,
update,
+ add,
delete,
stoppable,
rebalance,
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
index 046e286..07838e3 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
@@ -48,6 +48,7 @@ import org.apache.helix.manager.zk.ZKUtil;
import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.ControllerHistory;
+import org.apache.helix.model.CustomizedStateAggregationConfig;
import org.apache.helix.model.HelixConfigScope;
import org.apache.helix.model.LiveInstance;
import org.apache.helix.model.MaintenanceSignal;
@@ -296,6 +297,114 @@ public class ClusterAccessor extends AbstractHelixResource {
return JSONRepresentation(config.getRecord());
}
+
+ @PUT
+ @Path("{clusterId}/customized-state-aggregation-config")
+ public Response addCustomizedStateAggregationConfig(@PathParam("clusterId") String clusterId,
+ String content) {
+ if (!doesClusterExist(clusterId)) {
+ return notFound(String.format("Cluster %s does not exist", clusterId));
+ }
+
+ HelixAdmin admin = getHelixAdmin();
+ ZNRecord record;
+ try {
+ record = toZNRecord(content);
+ } catch (IOException e) {
+ return badRequest("Input is not a vaild ZNRecord!");
+ }
+
+ try {
+ CustomizedStateAggregationConfig customizedStateAggregationConfig =
+ new CustomizedStateAggregationConfig.Builder(record).build();
+ admin.addCustomizedStateAggregationConfig(clusterId, customizedStateAggregationConfig);
+ } catch (Exception ex) {
+ _logger.error("Cannot add CustomizedStateAggregationConfig to cluster: {} Exception: {}",
+ clusterId, ex);
+ return serverError(ex);
+ }
+
+ return OK();
+ }
+
+ @DELETE
+ @Path("{clusterId}/customized-state-aggregation-config")
+ public Response removeCustomizedStateAggregationConfig(@PathParam("clusterId") String clusterId) {
+ if (!doesClusterExist(clusterId)) {
+ return notFound(String.format("Cluster %s does not exist", clusterId));
+ }
+
+ HelixAdmin admin = getHelixAdmin();
+ try {
+ admin.removeCustomizedStateAggregationConfig(clusterId);
+ } catch (Exception ex) {
+ _logger.error(
+ "Cannot remove CustomizedStateAggregationConfig from cluster: {}, Exception: {}",
+ clusterId, ex);
+ return serverError(ex);
+ }
+
+ return OK();
+ }
+
+ @GET
+ @Path("{clusterId}/customized-state-aggregation-config")
+ public Response getCustomizedStateAggregationConfig(@PathParam("clusterId") String clusterId) {
+ if (!doesClusterExist(clusterId)) {
+ return notFound(String.format("Cluster %s does not exist", clusterId));
+ }
+
+ ConfigAccessor configAccessor = getConfigAccessor();
+ CustomizedStateAggregationConfig customizedStateAggregationConfig =
+ configAccessor.getCustomizedStateAggregationConfig(clusterId);
+
+ if (customizedStateAggregationConfig != null) {
+ return JSONRepresentation(customizedStateAggregationConfig.getRecord());
+ }
+
+ return notFound();
+ }
+
+ @POST
+ @Path("{clusterId}/customized-state-aggregation-config")
+ public Response updateCustomizedStateAggregationConfig(@PathParam("clusterId") String clusterId,
+ @QueryParam("command") String commandStr, @QueryParam("type") String type) {
+ if (!doesClusterExist(clusterId)) {
+ return notFound(String.format("Cluster %s does not exist", clusterId));
+ }
+
+ Command command;
+ if (commandStr == null || commandStr.isEmpty()) {
+ command = Command.add; // Default behavior
+ } else {
+ try {
+ command = getCommand(commandStr);
+ } catch (HelixException ex) {
+ return badRequest(ex.getMessage());
+ }
+ }
+
+ HelixAdmin admin = getHelixAdmin();
+
+ try {
+ switch (command) {
+ case delete:
+ admin.removeTypeFromCustomizedStateAggregationConfig(clusterId, type);
+ break;
+ case add:
+ admin.addTypeToCustomizedStateAggregationConfig(clusterId, type);
+ break;
+ default:
+ return badRequest("Unsupported command " + commandStr);
+ }
+ } catch (Exception ex) {
+ _logger.error("Failed to {} CustomizedStateAggregationConfig for cluster {} new type: {}, Exception: {}", command, clusterId, type, ex);
+ return serverError(ex);
+ }
+ return OK();
+ }
+
+
@GET
@Path("{clusterId}/topology")
public Response getClusterTopology(@PathParam("clusterId") String clusterId) throws IOException {
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
index 9e862a8..fd343ae 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
@@ -47,6 +47,7 @@ import org.apache.helix.integration.manager.ClusterDistributedController;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
import org.apache.helix.manager.zk.ZKUtil;
import org.apache.helix.model.ClusterConfig;
+import org.apache.helix.model.CustomizedStateAggregationConfig;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.InstanceConfig;
@@ -783,7 +784,6 @@ public class TestClusterAccessor extends AbstractTestClass {
Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
}
-
@Test(dependsOnMethods = "testAddClusterWithCloudConfigDisabledCloud")
public void testAddCloudConfigNonExistedCluster() throws IOException {
System.out.println("Start test :" + TestHelper.getTestMethodName());
@@ -898,7 +898,6 @@ public class TestClusterAccessor extends AbstractTestClass {
System.out.println("End test :" + TestHelper.getTestMethodName());
}
-
@Test(dependsOnMethods = "testDeleteCloudConfig")
public void testPartialDeleteCloudConfig() throws IOException {
System.out.println("Start test :" + TestHelper.getTestMethodName());
@@ -1004,6 +1003,150 @@ public class TestClusterAccessor extends AbstractTestClass {
System.out.println("End test :" + TestHelper.getTestMethodName());
}
+ @Test(dependsOnMethods = "testUpdateCloudConfig")
+ public void testAddCustomizedConfigNonExistedCluster() throws IOException {
+ System.out.println("Start test :" + TestHelper.getTestMethodName());
+ String urlBase = "clusters/TestCluster/customizedstateconfig/";
+ ZNRecord record = new ZNRecord("TestCustomizedStateConfig");
+ List<String> testList = new ArrayList<String>();
+ testList.add("mockType1");
+ record.setListField(
+ CustomizedStateConfig.CustomizedStateProperty.AGGREGATION_ENABLED_TYPES
+ .name(),
+ testList);
+
+ // Expecting not found response since the cluster is not setup yet.
+ put(urlBase, null,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.NOT_FOUND.getStatusCode());
+ System.out.println("End test :" + TestHelper.getTestMethodName());
+ }
+
+ @Test(dependsOnMethods = "testAddCustomizedConfigNonExistedCluster")
+ public void testAddCustomizedConfig() throws Exception {
+ System.out.println("Start test :" + TestHelper.getTestMethodName());
+ _gSetupTool.addCluster("TestClusterCustomized", true);
+ String urlBase = "clusters/TestClusterCustomized/customizedstateconfig/";
+ ZNRecord record = new ZNRecord("TestCustomizedStateConfig");
+ List<String> testList = new ArrayList<String>();
+ testList.add("mockType1");
+ testList.add("mockType2");
+ record.setListField(
+ CustomizedStateConfig.CustomizedStateProperty.AGGREGATION_ENABLED_TYPES
+ .name(),
+ testList);
+
+ put(urlBase, null,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.OK.getStatusCode());
+
+ // Read CustomizedStateConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
+ CustomizedStateConfig customizedConfigFromZk = _configAccessor.getCustomizedStateConfig("TestClusterCustomized");
+ List<String> listTypesFromZk = customizedConfigFromZk.getAggregationEnabledTypes();
+ Assert.assertEquals(listTypesFromZk.get(0), "mockType1");
+ Assert.assertEquals(listTypesFromZk.get(1), "mockType2");
+
+ // Now test the getCustomizedStateConfig method.
+ String body = get(urlBase, null, Response.Status.OK.getStatusCode(), true);
+
+ ZNRecord recordFromRest = new ObjectMapper().reader(ZNRecord.class).readValue(body);
+ CustomizedStateConfig customizedConfigRest = new CustomizedStateConfig.Builder(recordFromRest).build();
+ CustomizedStateConfig customizedConfigZk = _configAccessor.getCustomizedStateConfig("TestClusterCustomized");
+
+ // Check that the CustomizedStateConfig from Zk and REST get method are equal
+ Assert.assertEquals(customizedConfigRest, customizedConfigZk);
+
+ // Check the fields individually
+ List<String> listUrlFromRest = customizedConfigRest.getAggregationEnabledTypes();
+ Assert.assertEquals(listUrlFromRest.get(0), "mockType1");
+ Assert.assertEquals(listUrlFromRest.get(1), "mockType2");
+
+ System.out.println("End test :" + TestHelper.getTestMethodName());
+ }
+
+ @Test(dependsOnMethods = "testAddCustomizedConfig")
+ public void testDeleteCustomizedConfig() throws IOException {
+ System.out.println("Start test :" + TestHelper.getTestMethodName());
+ _gSetupTool.addCluster("TestClusterCustomized", true);
+ String urlBase = "clusters/TestClusterCustomized/customizedstateconfig/";
+ ZNRecord record = new ZNRecord("TestCustomizedStateConfig");
+ List<String> testList = new ArrayList<String>();
+ testList.add("mockType1");
+ record.setListField(
+ CustomizedStateConfig.CustomizedStateProperty.AGGREGATION_ENABLED_TYPES
+ .name(),
+ testList);
+
+ put(urlBase, null,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.OK.getStatusCode());
+
+ // Read CustomizedStateConfig from Zookeeper and make sure it exists
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
+ CustomizedStateConfig customizedConfigFromZk = _configAccessor.getCustomizedStateConfig("TestClusterCustomized");
+ Assert.assertNotNull(customizedConfigFromZk);
+
+ delete(urlBase, Response.Status.OK.getStatusCode());
+
+ customizedConfigFromZk = _configAccessor.getCustomizedStateConfig("TestClusterCustomized");
+ Assert.assertNull(customizedConfigFromZk);
+
+ System.out.println("End test :" + TestHelper.getTestMethodName());
+ }
+
+ @Test(dependsOnMethods = "testDeleteCustomizedConfig")
+ public void testUpdateCustomizedConfig() throws IOException {
+ System.out.println("Start test :" + TestHelper.getTestMethodName());
+ _gSetupTool.addCluster("TestClusterCustomized", true);
+ String urlBase = "clusters/TestClusterCustomized/customizedstateconfig/";
+ ZNRecord record = new ZNRecord("TestCustomizedStateConfig");
+ List<String> testList = new ArrayList<String>();
+ testList.add("mockType1");
+ record.setListField(
+ CustomizedStateConfig.CustomizedStateProperty.AGGREGATION_ENABLED_TYPES
+ .name(),
+ testList);
+
+ put(urlBase, null,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.OK.getStatusCode());
+
+ // Read CustomizedStateConfig from Zookeeper and make sure it exists
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
+ CustomizedStateConfig customizedConfigFromZk = _configAccessor.getCustomizedStateConfig("TestClusterCustomized");
+ Assert.assertNotNull(customizedConfigFromZk);
+
+ // Add new type to CustomizedStateConfig
+ Map<String, String> map1 = new HashMap<>();
+ map1.put("command", Command.add.name());
+ map1.put("type", "mockType2");
+
+ post(urlBase, map1, Entity.entity("", MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.OK.getStatusCode());
+
+ customizedConfigFromZk =
+ _configAccessor.getCustomizedStateConfig("TestClusterCustomized");
+ List<String> listTypesFromZk = customizedConfigFromZk.getAggregationEnabledTypes();
+ Assert.assertEquals(listTypesFromZk.get(0), "mockType1");
+ Assert.assertEquals(listTypesFromZk.get(1), "mockType2");
+
+ // Remove a type to CustomizedStateConfig
+ Map<String, String> map2 = new HashMap<>();
+ map2.put("command", Command.delete.name());
+ map2.put("type", "mockType1");
+
+ post(urlBase, map2, Entity.entity("", MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.OK.getStatusCode());
+
+ customizedConfigFromZk =
+ _configAccessor.getCustomizedStateConfig("TestClusterCustomized");
+ listTypesFromZk = customizedConfigFromZk.getAggregationEnabledTypes();
+ Assert.assertEquals(listTypesFromZk.get(0), "mockType2");
+ Assert.assertFalse(listTypesFromZk.contains("mockType1"));
+ System.out.println("End test :" + TestHelper.getTestMethodName());
+ }
+
private ClusterConfig getClusterConfigFromRest(String cluster) throws IOException {
String body = get("clusters/" + cluster + "/configs", null, Response.Status.OK.getStatusCode(), true);
[helix] 11/23: Use updater to update customized state for
concurrency control (#859)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit d6fd3622aa175db0f213885954ded314a9deba02
Author: Molly Gao <31...@users.noreply.github.com>
AuthorDate: Wed Mar 11 10:45:58 2020 -0700
Use updater to update customized state for concurrency control (#859)
Currently the update customized state method is made synchronized for concurrency control. This commit modifies the implementation of update to leave the responsibility of concurrency control to ZooKeeper by using updater to update the customize state. With delete method already implemented with updater, we can prevent unexpected change of the customize state data.
---
.../customizedstate/CustomizedStateProvider.java | 23 +++++-----------------
.../paticipant/TestCustomizedStateUpdate.java | 21 ++++++++++----------
2 files changed, 16 insertions(+), 28 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/customizedstate/CustomizedStateProvider.java b/helix-core/src/main/java/org/apache/helix/customizedstate/CustomizedStateProvider.java
index 3807ea6..80f4d91 100644
--- a/helix-core/src/main/java/org/apache/helix/customizedstate/CustomizedStateProvider.java
+++ b/helix-core/src/main/java/org/apache/helix/customizedstate/CustomizedStateProvider.java
@@ -50,7 +50,7 @@ public class CustomizedStateProvider {
* Update a specific customized state based on the resource name and partition name. The
* customized state is input as a single string
*/
- public synchronized void updateCustomizedState(String customizedStateName, String resourceName,
+ public void updateCustomizedState(String customizedStateName, String resourceName,
String partitionName, String customizedState) {
Map<String, String> customizedStateMap = new HashMap<>();
customizedStateMap.put(CustomizedState.CustomizedStateProperty.CURRENT_STATE.name(), customizedState);
@@ -61,29 +61,16 @@ public class CustomizedStateProvider {
* Update a specific customized state based on the resource name and partition name. The
* customized state is input as a map
*/
- public synchronized void updateCustomizedState(String customizedStateName, String resourceName,
+ public void updateCustomizedState(String customizedStateName, String resourceName,
String partitionName, Map<String, String> customizedStateMap) {
PropertyKey.Builder keyBuilder = _helixDataAccessor.keyBuilder();
PropertyKey propertyKey =
keyBuilder.customizedState(_instanceName, customizedStateName, resourceName);
ZNRecord record = new ZNRecord(resourceName);
- Map<String, Map<String, String>> mapFields = new HashMap<>();
- CustomizedState existingState = getCustomizedState(customizedStateName, resourceName);
- if (existingState != null
- && existingState.getRecord().getMapFields().containsKey(partitionName)) {
- Map<String, String> existingMap = new HashMap<>();
- for (String key : customizedStateMap.keySet()) {
- existingMap.put(key, customizedStateMap.get(key));
- }
-
- mapFields.put(partitionName, existingMap);
- } else {
- mapFields.put(partitionName, customizedStateMap);
- }
- record.setMapFields(mapFields);
+ record.setMapField(partitionName, customizedStateMap);
if (!_helixDataAccessor.updateProperty(propertyKey, new CustomizedState(record))) {
- throw new HelixException(
- String.format("Failed to persist customized state %s to zk for instance %s, resource %s",
+ throw new HelixException(String
+ .format("Failed to persist customized state %s to zk for instance %s, resource %s",
customizedStateName, _instanceName, record.getId()));
}
}
diff --git a/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestCustomizedStateUpdate.java b/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestCustomizedStateUpdate.java
index bc086ba..740f2fd 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestCustomizedStateUpdate.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestCustomizedStateUpdate.java
@@ -57,6 +57,8 @@ public class TestCustomizedStateUpdate extends ZkStandAloneCMTestBase {
private final String PARTITION_STATE = "partitionState";
private static HelixManager _manager;
private static CustomizedStateProvider _mockProvider;
+ private PropertyKey _propertyKey;
+ private HelixDataAccessor _dataAccessor;
@BeforeClass
public void beforeClass() throws Exception {
@@ -67,6 +69,9 @@ public class TestCustomizedStateUpdate extends ZkStandAloneCMTestBase {
_participants[0].connect();
_mockProvider = CustomizedStateProviderFactory.getInstance()
.buildCustomizedStateProvider(_manager, _participants[0].getInstanceName());
+ _dataAccessor = _manager.getHelixDataAccessor();
+ _propertyKey = _dataAccessor.keyBuilder()
+ .customizedStates(_participants[0].getInstanceName(), CUSTOMIZE_STATE_NAME);
}
@AfterClass
@@ -77,11 +82,8 @@ public class TestCustomizedStateUpdate extends ZkStandAloneCMTestBase {
@BeforeMethod
public void beforeMethod() {
- HelixDataAccessor dataAccessor = _manager.getHelixDataAccessor();
- PropertyKey propertyKey = dataAccessor.keyBuilder()
- .customizedStates(_participants[0].getInstanceName(), CUSTOMIZE_STATE_NAME);
- dataAccessor.removeProperty(propertyKey);
- CustomizedState customizedStates = dataAccessor.getProperty(propertyKey);
+ _dataAccessor.removeProperty(_propertyKey);
+ CustomizedState customizedStates = _dataAccessor.getProperty(_propertyKey);
Assert.assertNull(customizedStates);
}
@@ -278,15 +280,14 @@ public class TestCustomizedStateUpdate extends ZkStandAloneCMTestBase {
@Test
public void testSimultaneousUpdateCustomizedState() {
- int n = 10;
-
List<Callable<Boolean>> threads = new ArrayList<>();
- for (int i = 0; i < n; i++) {
+ int threadCount = 10;
+ for (int i = 0; i < threadCount; i++) {
threads.add(new TestSimultaneousUpdate());
}
Map<String, Boolean> resultMap = TestHelper.startThreadsConcurrently(threads, 1000);
- Assert.assertEquals(resultMap.size(), n);
- Boolean[] results = new Boolean[n];
+ Assert.assertEquals(resultMap.size(), threadCount);
+ Boolean[] results = new Boolean[threadCount];
Arrays.fill(results, true);
Assert.assertEqualsNoOrder(resultMap.values().toArray(), results);
}
[helix] 05/23: Implement Helix API for updating customized state
(#729)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit f4e3a93d3a60a3fb243815f4c33b7d35ea6b85c5
Author: zhangmeng916 <56...@users.noreply.github.com>
AuthorDate: Wed Feb 26 10:05:55 2020 -0800
Implement Helix API for updating customized state (#729)
Implement Helix APIs in CustomizedStateProvider for customers to operate on their own customized state. The available operations include update, get, and delete. To use CustomizedStateProvider, Helix user should initialize its factory and pass required parameters.
---
.../main/java/org/apache/helix/PropertyKey.java | 26 ++++
.../java/org/apache/helix/PropertyPathBuilder.java | 23 +++
.../main/java/org/apache/helix/PropertyType.java | 2 +
.../customizedstate/CustomizedStateProvider.java | 130 +++++++++++++++++
.../CustomizedStateProviderFactory.java | 79 +++++++++++
.../org/apache/helix/manager/zk/ZKHelixAdmin.java | 1 +
.../org/apache/helix/model/CustomizedState.java | 155 +++++++++++++++++++++
.../org/apache/helix/TestPropertyPathBuilder.java | 5 +
.../paticipant/TestCustomizedStateUpdate.java | 152 ++++++++++++++++++++
.../java/org/apache/helix/mock/MockHelixAdmin.java | 3 +
10 files changed, 576 insertions(+)
diff --git a/helix-core/src/main/java/org/apache/helix/PropertyKey.java b/helix-core/src/main/java/org/apache/helix/PropertyKey.java
index 1df56ff..39f18c9 100644
--- a/helix-core/src/main/java/org/apache/helix/PropertyKey.java
+++ b/helix-core/src/main/java/org/apache/helix/PropertyKey.java
@@ -28,6 +28,7 @@ import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ControllerHistory;
import org.apache.helix.model.CurrentState;
import org.apache.helix.model.CustomizedStateAggregationConfig;
+import org.apache.helix.model.CustomizedState;
import org.apache.helix.model.Error;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.HealthStat;
@@ -53,6 +54,7 @@ import org.slf4j.LoggerFactory;
import static org.apache.helix.PropertyType.CONFIGS;
import static org.apache.helix.PropertyType.CONTROLLER;
import static org.apache.helix.PropertyType.CURRENTSTATES;
+import static org.apache.helix.PropertyType.CUSTOMIZEDSTATES;
import static org.apache.helix.PropertyType.ERRORS;
import static org.apache.helix.PropertyType.ERRORS_CONTROLLER;
import static org.apache.helix.PropertyType.EXTERNALVIEW;
@@ -481,6 +483,30 @@ public class PropertyKey {
}
/**
+ * Get a property key associated with {@link CustomizedState} of an instance and customized state
+ * @param instanceName
+ * @param customizedStateName
+ * @return {@link PropertyKey}
+ */
+ public PropertyKey customizedStates(String instanceName, String customizedStateName) {
+ return new PropertyKey(CUSTOMIZEDSTATES, CustomizedState.class, _clusterName, instanceName,
+ customizedStateName);
+ }
+
+ /**
+ * Get a property key associated with {@link CustomizedState} of an instance, customized state, and resource
+ * @param instanceName
+ * @param customizedStateName
+ * @param resourceName
+ * @return {@link PropertyKey}
+ */
+ public PropertyKey customizedState(String instanceName, String customizedStateName,
+ String resourceName) {
+ return new PropertyKey(CUSTOMIZEDSTATES, CustomizedState.class, _clusterName, instanceName,
+ customizedStateName, resourceName);
+ }
+
+ /**
* Get a property key associated with {@link StatusUpdate} of an instance, session, resource,
* and partition
* @param instanceName
diff --git a/helix-core/src/main/java/org/apache/helix/PropertyPathBuilder.java b/helix-core/src/main/java/org/apache/helix/PropertyPathBuilder.java
index 52bf9f7..48db47a 100644
--- a/helix-core/src/main/java/org/apache/helix/PropertyPathBuilder.java
+++ b/helix-core/src/main/java/org/apache/helix/PropertyPathBuilder.java
@@ -27,6 +27,7 @@ import java.util.regex.Pattern;
import org.apache.helix.model.ControllerHistory;
import org.apache.helix.model.CurrentState;
+import org.apache.helix.model.CustomizedState;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.InstanceConfig;
@@ -42,6 +43,7 @@ import org.slf4j.LoggerFactory;
import static org.apache.helix.PropertyType.CONFIGS;
import static org.apache.helix.PropertyType.CURRENTSTATES;
+import static org.apache.helix.PropertyType.CUSTOMIZEDSTATES;
import static org.apache.helix.PropertyType.EXTERNALVIEW;
import static org.apache.helix.PropertyType.HISTORY;
import static org.apache.helix.PropertyType.IDEALSTATES;
@@ -53,6 +55,7 @@ import static org.apache.helix.PropertyType.STATEMODELDEFS;
import static org.apache.helix.PropertyType.STATUSUPDATES;
import static org.apache.helix.PropertyType.WORKFLOWCONTEXT;
+
/**
* Utility mapping properties to their Zookeeper locations
*/
@@ -112,6 +115,12 @@ public class PropertyPathBuilder {
"/{clusterName}/INSTANCES/{instanceName}/CURRENTSTATES/{sessionId}/{resourceName}");
addEntry(PropertyType.CURRENTSTATES, 5,
"/{clusterName}/INSTANCES/{instanceName}/CURRENTSTATES/{sessionId}/{resourceName}/{bucketName}");
+ addEntry(PropertyType.CUSTOMIZEDSTATES, 2,
+ "/{clusterName}/INSTANCES/{instanceName}/CUSTOMIZEDSTATES");
+ addEntry(PropertyType.CUSTOMIZEDSTATES, 3,
+ "/{clusterName}/INSTANCES/{instanceName}/CUSTOMIZEDSTATES/{customizedStateName}");
+ addEntry(PropertyType.CUSTOMIZEDSTATES, 4,
+ "/{clusterName}/INSTANCES/{instanceName}/CUSTOMIZEDSTATES/{customizedStateName}/{resourceName}");
addEntry(PropertyType.STATUSUPDATES, 2,
"/{clusterName}/INSTANCES/{instanceName}/STATUSUPDATES");
addEntry(PropertyType.STATUSUPDATES, 3,
@@ -312,6 +321,20 @@ public class PropertyPathBuilder {
sessionId, resourceName);
}
+ public static String instanceCustomizedState(String clusterName, String instanceName) {
+ return String.format("/%s/INSTANCES/%s/CUSTOMIZEDSTATES", clusterName, instanceName);
+ }
+
+ public static String instanceCustomizedState(String clusterName, String instanceName,
+ String customizedStateName) {
+ return String.format("/%s/INSTANCES/%s/CUSTOMIZEDSTATES/%s", clusterName, instanceName, customizedStateName);
+ }
+
+ public static String instanceCustomizedState(String clusterName, String instanceName,
+ String customizedStateName, String resourceName) {
+ return String.format("/%s/INSTANCES/%s/CUSTOMIZEDSTATES/%s/%s", clusterName, instanceName,
+ customizedStateName, resourceName);
+ }
public static String instanceError(String clusterName, String instanceName) {
return String.format("/%s/INSTANCES/%s/ERRORS", clusterName, instanceName);
}
diff --git a/helix-core/src/main/java/org/apache/helix/PropertyType.java b/helix-core/src/main/java/org/apache/helix/PropertyType.java
index f879249..e076322 100644
--- a/helix-core/src/main/java/org/apache/helix/PropertyType.java
+++ b/helix-core/src/main/java/org/apache/helix/PropertyType.java
@@ -55,6 +55,8 @@ public enum PropertyType {
ERRORS(Type.INSTANCE, true, true),
INSTANCE_HISTORY(Type.INSTANCE, true, true, true),
HEALTHREPORT(Type.INSTANCE, true, false, false, false, false, true),
+ CUSTOMIZEDSTATES(Type.INSTANCE, true, false, false, true, true),
+
// CONTROLLER PROPERTY
LEADER(Type.CONTROLLER, false, false, true, true),
diff --git a/helix-core/src/main/java/org/apache/helix/customizedstate/CustomizedStateProvider.java b/helix-core/src/main/java/org/apache/helix/customizedstate/CustomizedStateProvider.java
new file mode 100644
index 0000000..3807ea6
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/customizedstate/CustomizedStateProvider.java
@@ -0,0 +1,130 @@
+package org.apache.helix.customizedstate;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.HashMap;
+import java.util.Map;
+import org.I0Itec.zkclient.DataUpdater;
+import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.HelixException;
+import org.apache.helix.HelixManager;
+import org.apache.helix.PropertyKey;
+import org.apache.helix.ZNRecord;
+import org.apache.helix.model.CustomizedState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Class for Helix customers to operate on customized state
+ */
+public class CustomizedStateProvider {
+ private static final Logger LOG = LoggerFactory.getLogger(CustomizedStateProvider.class);
+ private final HelixManager _helixManager;
+ private final HelixDataAccessor _helixDataAccessor;
+ private String _instanceName;
+
+ public CustomizedStateProvider(HelixManager helixManager, String instanceName) {
+ _helixManager = helixManager;
+ _instanceName = instanceName;
+ _helixDataAccessor = _helixManager.getHelixDataAccessor();
+ }
+
+ /**
+ * Update a specific customized state based on the resource name and partition name. The
+ * customized state is input as a single string
+ */
+ public synchronized void updateCustomizedState(String customizedStateName, String resourceName,
+ String partitionName, String customizedState) {
+ Map<String, String> customizedStateMap = new HashMap<>();
+ customizedStateMap.put(CustomizedState.CustomizedStateProperty.CURRENT_STATE.name(), customizedState);
+ updateCustomizedState(customizedStateName, resourceName, partitionName, customizedStateMap);
+ }
+
+ /**
+ * Update a specific customized state based on the resource name and partition name. The
+ * customized state is input as a map
+ */
+ public synchronized void updateCustomizedState(String customizedStateName, String resourceName,
+ String partitionName, Map<String, String> customizedStateMap) {
+ PropertyKey.Builder keyBuilder = _helixDataAccessor.keyBuilder();
+ PropertyKey propertyKey =
+ keyBuilder.customizedState(_instanceName, customizedStateName, resourceName);
+ ZNRecord record = new ZNRecord(resourceName);
+ Map<String, Map<String, String>> mapFields = new HashMap<>();
+ CustomizedState existingState = getCustomizedState(customizedStateName, resourceName);
+ if (existingState != null
+ && existingState.getRecord().getMapFields().containsKey(partitionName)) {
+ Map<String, String> existingMap = new HashMap<>();
+ for (String key : customizedStateMap.keySet()) {
+ existingMap.put(key, customizedStateMap.get(key));
+ }
+
+ mapFields.put(partitionName, existingMap);
+ } else {
+ mapFields.put(partitionName, customizedStateMap);
+ }
+ record.setMapFields(mapFields);
+ if (!_helixDataAccessor.updateProperty(propertyKey, new CustomizedState(record))) {
+ throw new HelixException(
+ String.format("Failed to persist customized state %s to zk for instance %s, resource %s",
+ customizedStateName, _instanceName, record.getId()));
+ }
+ }
+
+ /**
+ * Get the customized state for a specified resource
+ */
+ public CustomizedState getCustomizedState(String customizedStateName, String resourceName) {
+ HelixDataAccessor accessor = _helixManager.getHelixDataAccessor();
+ PropertyKey.Builder keyBuilder = accessor.keyBuilder();
+ return (CustomizedState) accessor
+ .getProperty(keyBuilder.customizedState(_instanceName, customizedStateName, resourceName));
+ }
+
+ /**
+ * Get the customized state for a specified resource and a specified partition
+ */
+ public Map<String, String> getPerPartitionCustomizedState(String customizedStateName,
+ String resourceName, String partitionName) {
+ PropertyKey.Builder keyBuilder = _helixDataAccessor.keyBuilder();
+ Map<String, Map<String, String>> mapView = _helixDataAccessor
+ .getProperty(keyBuilder.customizedState(_instanceName, customizedStateName, resourceName))
+ .getRecord().getMapFields();
+ return mapView.get(partitionName);
+ }
+
+ /**
+ * Delete the customized state for a specified resource and a specified partition
+ */
+ public void deletePerPartitionCustomizedState(String customizedStateName, String resourceName,
+ String partitionName) {
+ PropertyKey.Builder keyBuilder = _helixDataAccessor.keyBuilder();
+ PropertyKey propertyKey =
+ keyBuilder.customizedState(_instanceName, customizedStateName, resourceName);
+ CustomizedState existingState = getCustomizedState(customizedStateName, resourceName);
+ _helixDataAccessor.updateProperty(propertyKey, new DataUpdater<ZNRecord>() {
+ @Override
+ public ZNRecord update(ZNRecord current) {
+ current.getMapFields().remove(partitionName);
+ return current;
+ }
+ }, existingState);
+ }
+}
diff --git a/helix-core/src/main/java/org/apache/helix/customizedstate/CustomizedStateProviderFactory.java b/helix-core/src/main/java/org/apache/helix/customizedstate/CustomizedStateProviderFactory.java
new file mode 100644
index 0000000..3e60522
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/customizedstate/CustomizedStateProviderFactory.java
@@ -0,0 +1,79 @@
+package org.apache.helix.customizedstate;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.HashMap;
+import org.apache.helix.HelixException;
+import org.apache.helix.HelixManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Singleton factory that build customized state provider.
+ */
+public class CustomizedStateProviderFactory {
+ private static Logger LOG = LoggerFactory.getLogger(CustomizedStateProvider.class);
+ private final HashMap<String, CustomizedStateProvider> _customizedStateProviderMap =
+ new HashMap<>();
+ private HelixManager _helixManager;
+
+ protected CustomizedStateProviderFactory() {
+ }
+
+ private static class SingletonHelper {
+ private static final CustomizedStateProviderFactory INSTANCE =
+ new CustomizedStateProviderFactory();
+ }
+
+ public static CustomizedStateProviderFactory getInstance() {
+ return SingletonHelper.INSTANCE;
+ }
+
+ public CustomizedStateProvider buildCustomizedStateProvider(String instanceName) {
+ if (_helixManager == null) {
+ throw new HelixException("Helix Manager has not been set yet.");
+ }
+ return buildCustomizedStateProvider(_helixManager, instanceName);
+ }
+
+ /**
+ * Build a customized state provider based on the specified input. If the instance already has a
+ * provider, return it. Otherwise, build a new one and put it in the map.
+ * @param helixManager The helix manager that belongs to the instance
+ * @param instanceName The name of the instance
+ * @return CustomizedStateProvider
+ */
+ public CustomizedStateProvider buildCustomizedStateProvider(HelixManager helixManager,
+ String instanceName) {
+ synchronized (_customizedStateProviderMap) {
+ if (_customizedStateProviderMap.get(instanceName) != null) {
+ return _customizedStateProviderMap.get(instanceName);
+ }
+ CustomizedStateProvider customizedStateProvider =
+ new CustomizedStateProvider(helixManager, instanceName);
+ _customizedStateProviderMap.put(instanceName, customizedStateProvider);
+ return customizedStateProvider;
+ }
+ }
+
+ public void setHelixManager(HelixManager helixManager) {
+ _helixManager = helixManager;
+ }
+}
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
index 24e0a60..3eb4d55 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
@@ -189,6 +189,7 @@ public class ZKHelixAdmin implements HelixAdmin {
_zkClient.createPersistent(PropertyPathBuilder.instanceMessage(clusterName, nodeId), true);
_zkClient.createPersistent(PropertyPathBuilder.instanceCurrentState(clusterName, nodeId), true);
+ _zkClient.createPersistent(PropertyPathBuilder.instanceCustomizedState(clusterName, nodeId), true);
_zkClient.createPersistent(PropertyPathBuilder.instanceError(clusterName, nodeId), true);
_zkClient.createPersistent(PropertyPathBuilder.instanceStatusUpdate(clusterName, nodeId), true);
_zkClient.createPersistent(PropertyPathBuilder.instanceHistory(clusterName, nodeId), true);
diff --git a/helix-core/src/main/java/org/apache/helix/model/CustomizedState.java b/helix-core/src/main/java/org/apache/helix/model/CustomizedState.java
new file mode 100644
index 0000000..7b0b97d
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/model/CustomizedState.java
@@ -0,0 +1,155 @@
+package org.apache.helix.model;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.helix.HelixProperty;
+import org.apache.helix.ZNRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Customized states of partitions in a resource for an instance.
+ */
+public class CustomizedState extends HelixProperty {
+ private static Logger LOG = LoggerFactory.getLogger(CustomizedState.class);
+
+ /**
+ * Lookup keys for the customized state
+ */
+ public enum CustomizedStateProperty {
+ PREVIOUS_STATE,
+ CURRENT_STATE,
+ START_TIME,
+ END_TIME
+ }
+
+ /**
+ * Instantiate a customized state with a resource
+ * @param resourceName name identifying the resource
+ */
+ public CustomizedState(String resourceName) {
+ super(resourceName);
+ }
+
+ /**
+ * Instantiate a customized state with a pre-populated ZNRecord
+ * @param record a ZNRecord corresponding to the customized state
+ */
+ public CustomizedState(ZNRecord record) {
+ super(record);
+ }
+
+ /**
+ * Get the name of the resource
+ * @return String resource identifier
+ */
+ public String getResourceName() {
+ return _record.getId();
+ }
+
+ /**
+ * Get the partitions on this instance and the specified property that each partition is currently having.
+ * @return (partition, property) pairs
+ */
+ public Map<String, String> getPartitionStateMap(CustomizedStateProperty property) {
+ Map<String, String> map = new HashMap<String, String>();
+ Map<String, Map<String, String>> mapFields = _record.getMapFields();
+ for (String partitionName : mapFields.keySet()) {
+ Map<String, String> tempMap = mapFields.get(partitionName);
+ if (tempMap != null) {
+ map.put(partitionName, tempMap.get(property.name()));
+ }
+ }
+ return map;
+ }
+
+ /**
+ * Get the state of a partition on this instance
+ * @param partitionName the name of the partition
+ * @return the state, or null if the partition is not present
+ */
+ public String getState(String partitionName) {
+ return getProperty(partitionName, CustomizedStateProperty.CURRENT_STATE);
+ }
+
+ public long getStartTime(String partitionName) {
+ String startTime = getProperty(partitionName, CustomizedStateProperty.START_TIME);
+ return startTime == null ? -1L : Long.parseLong(startTime);
+ }
+
+ public long getEndTime(String partitionName) {
+ String endTime = getProperty(partitionName, CustomizedStateProperty.END_TIME);
+ return endTime == null ? -1L : Long.parseLong(endTime);
+ }
+
+
+ public String getPreviousState(String partitionName) {
+ return getProperty(partitionName, CustomizedStateProperty.PREVIOUS_STATE);
+ }
+
+ private String getProperty(String partitionName, CustomizedStateProperty property) {
+ Map<String, String> mapField = _record.getMapField(partitionName);
+ if (mapField != null) {
+ return mapField.get(property.name());
+ }
+ return null;
+ }
+
+ /**
+ * Set the state that a partition is currently in on this instance
+ * @param partitionName the name of the partition
+ * @param state the state of the partition
+ */
+ public void setState(String partitionName, String state) {
+ setProperty(partitionName, CustomizedStateProperty.CURRENT_STATE, state);
+ }
+
+ public void setStartTime(String partitionName, long startTime) {
+ setProperty(partitionName, CustomizedStateProperty.START_TIME, String.valueOf(startTime));
+ }
+
+ public void setEndTime(String partitionName, long endTime) {
+ setProperty(partitionName, CustomizedStateProperty.END_TIME, String.valueOf(endTime));
+ }
+
+ public void setPreviousState(String partitionName, String state) {
+ setProperty(partitionName, CustomizedStateProperty.PREVIOUS_STATE, state);
+ }
+
+ private void setProperty(String partitionName, CustomizedStateProperty property, String value) {
+ Map<String, Map<String, String>> mapFields = _record.getMapFields();
+ mapFields.putIfAbsent(partitionName, new TreeMap<String, String>());
+ mapFields.get(partitionName).put(property.name(), value);
+ }
+
+ @Override
+ public boolean isValid() {
+ if (getPartitionStateMap(CustomizedStateProperty.CURRENT_STATE) == null) {
+ LOG.error("Customized state does not contain state map. id:" + getResourceName());
+ return false;
+ }
+ return true;
+ }
+}
diff --git a/helix-core/src/test/java/org/apache/helix/TestPropertyPathBuilder.java b/helix-core/src/test/java/org/apache/helix/TestPropertyPathBuilder.java
index 0ff2e87..d414d54 100644
--- a/helix-core/src/test/java/org/apache/helix/TestPropertyPathBuilder.java
+++ b/helix-core/src/test/java/org/apache/helix/TestPropertyPathBuilder.java
@@ -40,6 +40,11 @@ public class TestPropertyPathBuilder {
actual = PropertyPathBuilder.instanceCurrentState("test_cluster", "instanceName1", "sessionId");
AssertJUnit.assertEquals(actual, "/test_cluster/INSTANCES/instanceName1/CURRENTSTATES/sessionId");
+ actual = PropertyPathBuilder.instanceCustomizedState("test_cluster", "instanceName1");
+ AssertJUnit.assertEquals(actual, "/test_cluster/INSTANCES/instanceName1/CUSTOMIZEDSTATES");
+ actual = PropertyPathBuilder.instanceCustomizedState("test_cluster", "instanceName1", "customizedState1");
+ AssertJUnit.assertEquals(actual, "/test_cluster/INSTANCES/instanceName1/CUSTOMIZEDSTATES/customizedState1");
+
actual = PropertyPathBuilder.controller("test_cluster");
AssertJUnit.assertEquals(actual, "/test_cluster/CONTROLLER");
actual = PropertyPathBuilder.controllerMessage("test_cluster");
diff --git a/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestCustomizedStateUpdate.java b/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestCustomizedStateUpdate.java
new file mode 100644
index 0000000..e0553cd
--- /dev/null
+++ b/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestCustomizedStateUpdate.java
@@ -0,0 +1,152 @@
+package org.apache.helix.integration.paticipant;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.HelixManager;
+import org.apache.helix.HelixManagerFactory;
+import org.apache.helix.InstanceType;
+import org.apache.helix.PropertyKey;
+import org.apache.helix.customizedstate.CustomizedStateProvider;
+import org.apache.helix.customizedstate.CustomizedStateProviderFactory;
+import org.apache.helix.integration.common.ZkStandAloneCMTestBase;
+import org.apache.helix.model.CustomizedState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class TestCustomizedStateUpdate extends ZkStandAloneCMTestBase {
+ private static Logger LOG = LoggerFactory.getLogger(TestCustomizedStateUpdate.class);
+ private final String CUSTOMIZE_STATE_NAME = "testState1";
+ private final String PARTITION_NAME1 = "testPartition1";
+ private final String PARTITION_NAME2 = "testPartition2";
+ private final String RESOURCE_NAME = "testResource1";
+
+ @Test
+ public void testUpdateCustomizedState() throws Exception {
+ HelixManager manager = HelixManagerFactory.getZKHelixManager(CLUSTER_NAME, "admin",
+ InstanceType.ADMINISTRATOR, ZK_ADDR);
+ manager.connect();
+ _participants[0].connect();
+
+ HelixDataAccessor dataAccessor = manager.getHelixDataAccessor();
+ PropertyKey propertyKey = dataAccessor.keyBuilder()
+ .customizedStates(_participants[0].getInstanceName(), CUSTOMIZE_STATE_NAME);
+ CustomizedState customizedStates = manager.getHelixDataAccessor().getProperty(propertyKey);
+ Assert.assertNull(customizedStates);
+
+ CustomizedStateProvider mockProvider = CustomizedStateProviderFactory.getInstance()
+ .buildCustomizedStateProvider(manager, _participants[0].getInstanceName());
+
+ // test adding customized state for a partition
+ Map<String, String> customizedStateMap = new HashMap<>();
+ customizedStateMap.put("PREVIOUS_STATE", "STARTED");
+ customizedStateMap.put("CURRENT_STATE", "END_OF_PUSH_RECEIVED");
+ mockProvider.updateCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME1,
+ customizedStateMap);
+
+ CustomizedState customizedState =
+ mockProvider.getCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME);
+ Assert.assertNotNull(customizedState);
+ Assert.assertEquals(customizedState.getId(), RESOURCE_NAME);
+ Map<String, Map<String, String>> mapView = customizedState.getRecord().getMapFields();
+ Assert.assertEquals(mapView.keySet().size(), 1);
+ Assert.assertEquals(mapView.keySet().iterator().next(), PARTITION_NAME1);
+ Assert.assertEquals(mapView.get(PARTITION_NAME1).keySet().size(), 2);
+ Assert.assertEquals(mapView.get(PARTITION_NAME1).get("PREVIOUS_STATE"), "STARTED");
+ Assert.assertEquals(mapView.get(PARTITION_NAME1).get("CURRENT_STATE"), "END_OF_PUSH_RECEIVED");
+
+ // test partial update customized state for previous partition
+ Map<String, String> stateMap1 = new HashMap<>();
+ stateMap1.put("PREVIOUS_STATE", "END_OF_PUSH_RECEIVED");
+ mockProvider.updateCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME1,
+ stateMap1);
+
+ customizedState = mockProvider.getCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME);
+ Assert.assertNotNull(customizedState);
+ Assert.assertEquals(customizedState.getId(), RESOURCE_NAME);
+ mapView = customizedState.getRecord().getMapFields();
+ Assert.assertEquals(mapView.keySet().size(), 1);
+ Assert.assertEquals(mapView.keySet().iterator().next(), PARTITION_NAME1);
+ Assert.assertEquals(mapView.get(PARTITION_NAME1).keySet().size(), 2);
+ Assert.assertEquals(mapView.get(PARTITION_NAME1).get("PREVIOUS_STATE"), "END_OF_PUSH_RECEIVED");
+ Assert.assertEquals(mapView.get(PARTITION_NAME1).get("CURRENT_STATE"), "END_OF_PUSH_RECEIVED");
+
+ // test full update customized state for previous partition
+ stateMap1 = new HashMap<>();
+ stateMap1.put("PREVIOUS_STATE", "END_OF_PUSH_RECEIVED");
+ stateMap1.put("CURRENT_STATE", "COMPLETED");
+ mockProvider.updateCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME1,
+ stateMap1);
+
+ customizedState = mockProvider.getCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME);
+ Assert.assertNotNull(customizedState);
+ Assert.assertEquals(customizedState.getId(), RESOURCE_NAME);
+ mapView = customizedState.getRecord().getMapFields();
+ Assert.assertEquals(mapView.keySet().size(), 1);
+ Assert.assertEquals(mapView.keySet().iterator().next(), PARTITION_NAME1);
+ Assert.assertEquals(mapView.get(PARTITION_NAME1).keySet().size(), 2);
+ Assert.assertEquals(mapView.get(PARTITION_NAME1).get("PREVIOUS_STATE"), "END_OF_PUSH_RECEIVED");
+ Assert.assertEquals(mapView.get(PARTITION_NAME1).get("CURRENT_STATE"), "COMPLETED");
+
+ // test adding adding customized state for a new partition in the same resource
+ Map<String, String> stateMap2 = new HashMap<>();
+ stateMap2.put("PREVIOUS_STATE", "STARTED");
+ stateMap2.put("CURRENT_STATE", "END_OF_PUSH_RECEIVED");
+ mockProvider.updateCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME2,
+ stateMap2);
+
+ customizedState = mockProvider.getCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME);
+ Assert.assertNotNull(customizedState);
+ Assert.assertEquals(customizedState.getId(), RESOURCE_NAME);
+ mapView = customizedState.getRecord().getMapFields();
+ Assert.assertEquals(mapView.keySet().size(), 2);
+ Assert.assertEqualsNoOrder(mapView.keySet().toArray(), new String[] {
+ PARTITION_NAME1, PARTITION_NAME2
+ });
+
+ Map<String, String> partitionMap1 = mockProvider
+ .getPerPartitionCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME1);
+ Assert.assertEquals(partitionMap1.keySet().size(), 2);
+ Assert.assertEquals(partitionMap1.get("PREVIOUS_STATE"), "END_OF_PUSH_RECEIVED");
+ Assert.assertEquals(partitionMap1.get("CURRENT_STATE"), "COMPLETED");
+
+ Map<String, String> partitionMap2 = mockProvider
+ .getPerPartitionCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME2);
+ Assert.assertEquals(partitionMap2.keySet().size(), 2);
+ Assert.assertEquals(partitionMap2.get("PREVIOUS_STATE"), "STARTED");
+ Assert.assertEquals(partitionMap2.get("CURRENT_STATE"), "END_OF_PUSH_RECEIVED");
+
+ // test delete customized state for a partition
+ mockProvider.deletePerPartitionCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME,
+ PARTITION_NAME1);
+ customizedState = mockProvider.getCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME);
+ Assert.assertNotNull(customizedState);
+ Assert.assertEquals(customizedState.getId(), RESOURCE_NAME);
+ mapView = customizedState.getRecord().getMapFields();
+ Assert.assertEquals(mapView.keySet().size(), 1);
+ Assert.assertEquals(mapView.keySet().iterator().next(), PARTITION_NAME2);
+
+ manager.disconnect();
+ }
+}
diff --git a/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java b/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java
index 595e435..02bd639 100644
--- a/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java
+++ b/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java
@@ -217,6 +217,9 @@ public class MockHelixAdmin implements HelixAdmin {
.set(PropertyPathBuilder.instanceCurrentState(clusterName, nodeId), new ZNRecord(nodeId),
0);
_baseDataAccessor
+ .set(PropertyPathBuilder.instanceCustomizedState(clusterName, nodeId), new ZNRecord(nodeId),
+ 0);
+ _baseDataAccessor
.set(PropertyPathBuilder.instanceError(clusterName, nodeId), new ZNRecord(nodeId), 0);
_baseDataAccessor
.set(PropertyPathBuilder.instanceStatusUpdate(clusterName, nodeId), new ZNRecord(nodeId),
[helix] 20/23: Add cache update/delete in customized view
aggregation stage (#934)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit a23445f2e3484078502f9ef89227c6640bc3274f
Author: Meng Zhang <mn...@linkedin.com>
AuthorDate: Fri Apr 10 00:16:47 2020 -0700
Add cache update/delete in customized view aggregation stage (#934)
Add customized view cache update in customized view aggregation stage. This is to ensure that the cache does not have stale data
---
.../helix/common/caches/CustomizedViewCache.java | 8 +++++
.../ResourceControllerDataProvider.java | 35 +++++++++++++++++++++-
.../stages/CustomizedViewAggregationStage.java | 15 ++++++++--
3 files changed, 54 insertions(+), 4 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java b/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java
index cdbc0f8..a62e34b 100644
--- a/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java
+++ b/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java
@@ -86,4 +86,12 @@ public class CustomizedViewCache extends AbstractDataCache<CustomizedView> {
public Map<String, CustomizedView> getCustomizedViewMap() {
return Collections.unmodifiableMap(_customizedViewCache.getPropertyMap());
}
+
+ /**
+ * Return customized view cache as a property cache.
+ * @return
+ */
+ public PropertyCache<CustomizedView> getCustomizedViewCache() {
+ return _customizedViewCache;
+ }
}
diff --git a/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java b/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java
index 3625630..a904e80 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java
@@ -39,6 +39,7 @@ import org.apache.helix.controller.pipeline.Pipeline;
import org.apache.helix.controller.stages.MissingTopStateRecord;
import org.apache.helix.model.CustomizedState;
import org.apache.helix.model.CustomizedStateConfig;
+import org.apache.helix.model.CustomizedView;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.ResourceAssignment;
@@ -281,6 +282,22 @@ public class ResourceControllerDataProvider extends BaseControllerDataProvider {
}
/**
+ * Update the cached customized view map
+ * @param customizedViews
+ */
+ public void updateCustomizedViews(String customizedStateType,
+ List<CustomizedView> customizedViews) {
+ if (!_customizedViewCacheMap.containsKey(customizedStateType)) {
+ CustomizedViewCache customizedViewCache =
+ new CustomizedViewCache(getClusterName(), customizedStateType);
+ _customizedViewCacheMap.put(customizedStateType, customizedViewCache);
+ }
+ for (CustomizedView cv : customizedViews) {
+ _customizedViewCacheMap.get(customizedStateType).getCustomizedViewCache().setProperty(cv);
+ }
+ }
+
+ /**
* Get local cached customized view map
* @return
*/
@@ -304,12 +321,28 @@ public class ResourceControllerDataProvider extends BaseControllerDataProvider {
* @param stateTypeNames
*/
- private void removeCustomizedViewTypes(Set<String> stateTypeNames) {
+ public void removeCustomizedViewTypes(Set<String> stateTypeNames) {
for (String stateType : stateTypeNames) {
_customizedViewCacheMap.remove(stateType);
}
}
+ /**
+ * Remove dead customized views for a certain state type from customized view cache
+ * @param stateType a specific customized state type
+ * @param resourceNames the names of resources whose customized view is stale
+ */
+ public void removeCustomizedViews(String stateType, List<String> resourceNames) {
+ if (!_customizedViewCacheMap.containsKey(stateType)) {
+ logger.warn(String.format("The customized state type : %s is not in the cache", stateType));
+ return;
+ }
+ for (String resourceName : resourceNames) {
+ _customizedViewCacheMap.get(stateType).getCustomizedViewCache()
+ .deletePropertyByName(resourceName);
+ }
+ }
+
public Map<String, Map<String, MissingTopStateRecord>> getMissingTopStateMap() {
return _missingTopStateMap;
}
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/CustomizedViewAggregationStage.java b/helix-core/src/main/java/org/apache/helix/controller/stages/CustomizedViewAggregationStage.java
index cdcdfd2..58888ed 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/stages/CustomizedViewAggregationStage.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/stages/CustomizedViewAggregationStage.java
@@ -21,9 +21,11 @@ package org.apache.helix.controller.stages;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.helix.HelixDataAccessor;
import org.apache.helix.HelixException;
@@ -72,13 +74,16 @@ public class CustomizedViewAggregationStage extends AbstractAsyncBaseStage {
Map<String, CustomizedViewCache> customizedViewCacheMap = cache.getCustomizedViewCacheMap();
- // remove stale customized view type from ZK
+ // remove stale customized view type from ZK and cache
+ Set<String> customizedTypesToRemove = new HashSet<>();
for (String stateType : customizedViewCacheMap.keySet()) {
if (!customizedStateOutput.getAllStateTypes().contains(stateType)) {
LogUtil.logInfo(LOG, _eventId, "Remove customizedView for stateType: " + stateType);
dataAccessor.removeProperty(keyBuilder.customizedView(stateType));
+ customizedTypesToRemove.add(stateType);
}
}
+ cache.removeCustomizedViewTypes(customizedTypesToRemove);
// update customized view
for (String stateType : customizedStateOutput.getAllStateTypes()) {
@@ -100,18 +105,22 @@ public class CustomizedViewAggregationStage extends AbstractAsyncBaseStage {
String resourceName = view.getResourceName();
keys.add(keyBuilder.customizedView(stateType, resourceName));
}
- // add/update customized-views
+ // add/update customized-views from zk and cache
if (updatedCustomizedViews.size() > 0) {
dataAccessor.setChildren(keys, updatedCustomizedViews);
+ cache.updateCustomizedViews(stateType, updatedCustomizedViews);
}
- // remove stale customized views
+ // remove stale customized views from zk and cache
+ List<String> customizedViewToRemove = new ArrayList<>();
for (String resourceName : curCustomizedViews.keySet()) {
if (!resourceMap.keySet().contains(resourceName)) {
LogUtil.logInfo(LOG, _eventId, "Remove customizedView for resource: " + resourceName);
dataAccessor.removeProperty(keyBuilder.customizedView(stateType, resourceName));
+ customizedViewToRemove.add(resourceName);
}
}
+ cache.removeCustomizedViews(stateType, customizedViewToRemove);
} catch (HelixException ex) {
LogUtil.logError(LOG, _eventId,
"Failed to calculate customized view for resource " + resource.getResourceName(), ex);
[helix] 02/23: add CustomizedStateAggregation config (#776)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 2189de73936f1d2b7166bb407019b88c4faffde2
Author: zhangmeng916 <56...@users.noreply.github.com>
AuthorDate: Thu Feb 20 11:53:44 2020 -0800
add CustomizedStateAggregation config (#776)
Add CustomizedViewAggregation as a cluster level config. This config defines the types of customized states that will be aggregated by Helix controller to generate a customized view. If Helix customers would like to have an aggregated view generated for their own states, they will need to add the type of the state to the list view in this config.
---
.../main/java/org/apache/helix/ConfigAccessor.java | 22 ++++
.../main/java/org/apache/helix/PropertyKey.java | 11 ++
.../model/CustomizedStateAggregationConfig.java | 144 +++++++++++++++++++++
.../org/apache/helix/model/HelixConfigScope.java | 5 +-
.../model/builder/HelixConfigScopeBuilder.java | 2 +
.../TestCustomizedStateAggregationConfig.java | 124 ++++++++++++++++++
6 files changed, 307 insertions(+), 1 deletion(-)
diff --git a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
index 41f7365..5a41241 100644
--- a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
+++ b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
@@ -33,6 +33,7 @@ import org.apache.helix.manager.zk.ZKUtil;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ConfigScope;
+import org.apache.helix.model.CustomizedStateAggregationConfig;
import org.apache.helix.model.HelixConfigScope;
import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
import org.apache.helix.model.InstanceConfig;
@@ -588,6 +589,27 @@ public class ConfigAccessor {
}
/**
+ * Get CustomizedStateAggregationConfig of the given cluster.
+ * @param clusterName
+ * @return The instance of {@link CustomizedStateAggregationConfig}
+ */
+ public CustomizedStateAggregationConfig getCustomizedStateAggregationConfig(String clusterName) {
+ if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
+ throw new HelixException(String.format("Failed to get config. cluster: %s is not setup.", clusterName));
+ }
+ HelixConfigScope scope =
+ new HelixConfigScopeBuilder(ConfigScopeProperty.CUSTOMIZED_STATE_AGGREGATION).forCluster(clusterName).build();
+ ZNRecord record = getConfigZnRecord(scope);
+
+ if (record == null) {
+ LOG.warn("No customized state aggregation config found at {}.", scope.getZkPath());
+ return null;
+ }
+
+ return new CustomizedStateAggregationConfig(record);
+ }
+
+ /**
* Get ClusterConfig of the given cluster.
*
* @param clusterName
diff --git a/helix-core/src/main/java/org/apache/helix/PropertyKey.java b/helix-core/src/main/java/org/apache/helix/PropertyKey.java
index 73cc3f0..1df56ff 100644
--- a/helix-core/src/main/java/org/apache/helix/PropertyKey.java
+++ b/helix-core/src/main/java/org/apache/helix/PropertyKey.java
@@ -27,6 +27,7 @@ import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ControllerHistory;
import org.apache.helix.model.CurrentState;
+import org.apache.helix.model.CustomizedStateAggregationConfig;
import org.apache.helix.model.Error;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.HealthStat;
@@ -245,6 +246,16 @@ public class PropertyKey {
}
/**
+ * Get a property key associated with this customized state aggregation configuration
+ * @return {@link PropertyKey}
+ */
+ public PropertyKey customizedStateAggregationConfig() {
+ return new PropertyKey(CONFIGS, ConfigScopeProperty.CUSTOMIZED_STATE_AGGREGATION,
+ CustomizedStateAggregationConfig.class, _clusterName,
+ ConfigScopeProperty.CUSTOMIZED_STATE_AGGREGATION.name(), _clusterName);
+ }
+
+ /**
* Get a property key associated with {@link InstanceConfig}
* @return {@link PropertyKey}
*/
diff --git a/helix-core/src/main/java/org/apache/helix/model/CustomizedStateAggregationConfig.java b/helix-core/src/main/java/org/apache/helix/model/CustomizedStateAggregationConfig.java
new file mode 100644
index 0000000..0e4a065
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/model/CustomizedStateAggregationConfig.java
@@ -0,0 +1,144 @@
+package org.apache.helix.model;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.helix.HelixProperty;
+import org.apache.helix.ZNRecord;
+
+/**
+ * CustomizedStateAggregation configurations
+ */
+public class CustomizedStateAggregationConfig extends HelixProperty {
+ /**
+ * Indicate which customized states will be aggregated.
+ * NOTE: Do NOT use this field name directly, use its corresponding getter/setter in the
+ * CustomizedStateAggregationConfig.
+ */
+ public enum CustomizedStateAggregationProperty {
+ AGGREGATION_ENABLED_TYPES,
+ }
+
+ /**
+ * Instantiate the CustomizedStateAggregationConfig
+ * @param cluster
+ */
+ public CustomizedStateAggregationConfig(String cluster) {
+ super(cluster);
+ }
+
+ /**
+ * Instantiate with a pre-populated record
+ * @param record a ZNRecord corresponding to a CustomizedStateAggregationConfig
+ */
+ public CustomizedStateAggregationConfig(ZNRecord record) {
+ super(record);
+ }
+
+ /**
+ * Instantiate the config using each field individually.
+ * Users should use CustomizedStateAggregationConfig.Builder to create
+ * CustomizedStateAggregationConfig.
+ * @param cluster
+ * @param aggregationEnabledTypes
+ */
+ public CustomizedStateAggregationConfig(String cluster, List<String> aggregationEnabledTypes) {
+ super(cluster);
+ _record.setListField(CustomizedStateAggregationProperty.AGGREGATION_ENABLED_TYPES.name(),
+ aggregationEnabledTypes);
+
+ }
+
+ /**
+ * Set the AGGREGATION_ENABLED_STATES field.
+ * @param aggregationEnabledTypes
+ */
+ public void setAggregationEnabledTypes(List<String> aggregationEnabledTypes) {
+ _record.setListField(CustomizedStateAggregationProperty.AGGREGATION_ENABLED_TYPES.name(),
+ aggregationEnabledTypes);
+ }
+
+ /**
+ * Get the AGGREGATION_ENABLED_STATES field.
+ * @return AGGREGATION_ENABLED_STATES field.
+ */
+ public List<String> getAggregationEnabledTypes() {
+ return _record
+ .getListField(CustomizedStateAggregationProperty.AGGREGATION_ENABLED_TYPES.name());
+ }
+
+ public static class Builder {
+ private String _clusterName = null;
+ private List<String> _aggregationEnabledTypes;
+
+ public CustomizedStateAggregationConfig build() {
+ return new CustomizedStateAggregationConfig(_clusterName, _aggregationEnabledTypes);
+ }
+
+ /**
+ * Default constructor
+ */
+ public Builder() {
+ }
+
+ /**
+ * Constructor with Cluster Name as input
+ * @param clusterName
+ */
+ public Builder(String clusterName) {
+ _clusterName = clusterName;
+ }
+
+ /**
+ * Constructor with CustomizedStateAggregationConfig as input
+ * @param customizedStateAggregationConfig
+ */
+ public Builder(CustomizedStateAggregationConfig customizedStateAggregationConfig) {
+ _aggregationEnabledTypes = customizedStateAggregationConfig.getAggregationEnabledTypes();
+ }
+
+ public Builder setClusterName(String v) {
+ _clusterName = v;
+ return this;
+ }
+
+ public Builder setAggregationEnabledTypes(List<String> v) {
+ _aggregationEnabledTypes = v;
+ return this;
+ }
+
+ public Builder addAggregationEnabledType(String v) {
+ if (_aggregationEnabledTypes == null) {
+ _aggregationEnabledTypes = new ArrayList<String>();
+ }
+ _aggregationEnabledTypes.add(v);
+ return this;
+ }
+
+ public String getClusterName() {
+ return _clusterName;
+ }
+
+ public List<String> getAggregationEnabledTypes() {
+ return _aggregationEnabledTypes;
+ }
+ }
+}
diff --git a/helix-core/src/main/java/org/apache/helix/model/HelixConfigScope.java b/helix-core/src/main/java/org/apache/helix/model/HelixConfigScope.java
index 8d814c5..cf49ccd 100644
--- a/helix-core/src/main/java/org/apache/helix/model/HelixConfigScope.java
+++ b/helix-core/src/main/java/org/apache/helix/model/HelixConfigScope.java
@@ -37,7 +37,8 @@ public class HelixConfigScope {
PARTITION(2, 1),
CONSTRAINT(2, 0),
REST(2, 0),
- CLOUD(2, 0);
+ CLOUD(2, 0),
+ CUSTOMIZED_STATE_AGGREGATION(2, 0);
final int _zkPathArgNum;
final int _mapKeyArgNum;
@@ -88,6 +89,8 @@ public class HelixConfigScope {
template.addEntry(ConfigScopeProperty.RESOURCE, 1, "/{clusterName}/CONFIGS/RESOURCE");
template.addEntry(ConfigScopeProperty.REST, 2, "/{clusterName}/CONFIGS/REST/{clusterName}");
template.addEntry(ConfigScopeProperty.CLOUD, 1, "/{clusterName}/CONFIGS/CLOUD");
+ template.addEntry(ConfigScopeProperty.CUSTOMIZED_STATE_AGGREGATION, 2,
+ "/{clusterName}/CONFIGS/CUSTOMIZED_STATE_AGGREGATION/{clusterName}");
}
final ConfigScopeProperty _type;
diff --git a/helix-core/src/main/java/org/apache/helix/model/builder/HelixConfigScopeBuilder.java b/helix-core/src/main/java/org/apache/helix/model/builder/HelixConfigScopeBuilder.java
index 78ed074..98de1e7 100644
--- a/helix-core/src/main/java/org/apache/helix/model/builder/HelixConfigScopeBuilder.java
+++ b/helix-core/src/main/java/org/apache/helix/model/builder/HelixConfigScopeBuilder.java
@@ -129,6 +129,8 @@ public class HelixConfigScopeBuilder {
case CLOUD:
scope = new HelixConfigScope(_type, Arrays.asList(_clusterName, _clusterName), null);
break;
+ case CUSTOMIZED_STATE_AGGREGATION:
+ scope = new HelixConfigScope(_type, Arrays.asList(_clusterName, _clusterName), null);
default:
break;
}
diff --git a/helix-core/src/test/java/org/apache/helix/model/TestCustomizedStateAggregationConfig.java b/helix-core/src/test/java/org/apache/helix/model/TestCustomizedStateAggregationConfig.java
new file mode 100644
index 0000000..5cdc333
--- /dev/null
+++ b/helix-core/src/test/java/org/apache/helix/model/TestCustomizedStateAggregationConfig.java
@@ -0,0 +1,124 @@
+package org.apache.helix.model;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.ArrayList;
+import org.apache.helix.ConfigAccessor;
+import org.apache.helix.HelixException;
+import org.apache.helix.PropertyKey.Builder;
+import org.apache.helix.TestHelper;
+import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.manager.zk.ZKHelixDataAccessor;
+import java.util.List;
+import org.apache.helix.manager.zk.ZkBaseDataAccessor;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class TestCustomizedStateAggregationConfig extends ZkUnitTestBase {
+
+ @Test(expectedExceptions = HelixException.class)
+ public void TestCustomizedStateAggregationConfigNonExistentCluster() {
+ String className = getShortClassName();
+ String clusterName = "CLUSTER_" + className;
+ // Read CustomizedStateAggregationConfig from Zookeeper and get exception since cluster in not setup yet
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CustomizedStateAggregationConfig customizedStateAggregationConfig =
+ _configAccessor.getCustomizedStateAggregationConfig(clusterName);
+ }
+
+ @Test(dependsOnMethods = "TestCustomizedStateAggregationConfigNonExistentCluster")
+ public void testCustomizedStateAggregationConfigNull() {
+ String className = getShortClassName();
+ String clusterName = "CLUSTER_" + className;
+ TestHelper.setupEmptyCluster(_gZkClient, clusterName);
+ // Read CustomizedStateAggregationConfig from Zookeeper
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CustomizedStateAggregationConfig customizedStateAggregationConfigFromZk =
+ _configAccessor.getCustomizedStateAggregationConfig(clusterName);
+ Assert.assertNull(customizedStateAggregationConfigFromZk);
+ }
+
+ @Test(dependsOnMethods = "testCustomizedStateAggregationConfigNull")
+ public void testCustomizedStateAggregationConfig() {
+ String className = getShortClassName();
+ String clusterName = "CLUSTER_" + className;
+ TestHelper.setupEmptyCluster(_gZkClient, clusterName);
+
+ // Create dummy CustomizedStateAggregationConfig object
+ CustomizedStateAggregationConfig customizedStateAggregationConfig =
+ new CustomizedStateAggregationConfig(clusterName);
+ List<String> aggregationEnabledTypes = new ArrayList<String>();
+ aggregationEnabledTypes.add("mockState1");
+ aggregationEnabledTypes.add("mockState2");
+ customizedStateAggregationConfig.setAggregationEnabledTypes(aggregationEnabledTypes);
+
+ // Write the CustomizedStateAggregationConfig to Zookeeper
+ ZKHelixDataAccessor accessor =
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor(_gZkClient));
+ Builder keyBuilder = accessor.keyBuilder();
+ accessor.setProperty(keyBuilder.customizedStateAggregationConfig(),
+ customizedStateAggregationConfig);
+
+ // Read CustomizedStateAggregationConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CustomizedStateAggregationConfig customizedStateAggregationConfigFromZk =
+ _configAccessor.getCustomizedStateAggregationConfig(clusterName);
+ Assert.assertEquals(customizedStateAggregationConfigFromZk.getAggregationEnabledTypes().size(),
+ 2);
+ Assert.assertEquals(aggregationEnabledTypes.get(0), "mockType1");
+ Assert.assertEquals(aggregationEnabledTypes.get(1), "mockType2");
+ }
+
+ @Test(dependsOnMethods = "testCustomizedStateAggregationConfig")
+ public void testCustomizedStateAggregationConfigBuilder() {
+ String className = getShortClassName();
+ String clusterName = "CLUSTER_" + className;
+ TestHelper.setupEmptyCluster(_gZkClient, clusterName);
+ CustomizedStateAggregationConfig.Builder builder =
+ new CustomizedStateAggregationConfig.Builder(clusterName);
+ builder.addAggregationEnabledType("mockType1");
+ builder.addAggregationEnabledType("mockType2");
+
+ // Check builder getter methods
+ Assert.assertEquals(builder.getClusterName(), clusterName);
+ List<String> aggregationEnabledTypes = builder.getAggregationEnabledTypes();
+ Assert.assertEquals(aggregationEnabledTypes.size(), 2);
+ Assert.assertEquals(aggregationEnabledTypes.get(0), "mockType1");
+ Assert.assertEquals(aggregationEnabledTypes.get(1), "mockType2");
+
+ CustomizedStateAggregationConfig customizedStateAggregationConfig = builder.build();
+
+ ZKHelixDataAccessor accessor =
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor(_gZkClient));
+ Builder keyBuilder = accessor.keyBuilder();
+ accessor.setProperty(keyBuilder.customizedStateAggregationConfig(),
+ customizedStateAggregationConfig);
+
+ // Read CustomizedStateAggregationConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CustomizedStateAggregationConfig customizedStateAggregationConfigFromZk =
+ _configAccessor.getCustomizedStateAggregationConfig(clusterName);
+ List<String> aggregationEnabledTypesFromZk =
+ customizedStateAggregationConfigFromZk.getAggregationEnabledTypes();
+ Assert.assertEquals(aggregationEnabledTypesFromZk.get(0), "mockType1");
+ Assert.assertEquals(aggregationEnabledTypesFromZk.get(1), "mockType2");
+ }
+
+}
[helix] 03/23: Add java API to add or remove
CustomizedStateAggregationConfig (#792)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit f0eb926de81ac5b6357744588d9b0d0b16de0c9c
Author: Ali Reza Zamani Zadeh Najari <an...@linkedin.com>
AuthorDate: Fri Feb 21 11:40:28 2020 -0800
Add java API to add or remove CustomizedStateAggregationConfig (#792)
In this commit the below APIs have been added.
1- addCustomizedStateAggregationConfig
2- removeCustomizedStateAggregationConfig.
3- addTypeToCustomizedStateAggregationConfig
4- removeTypeFromCustomizedStateAggregationConfig
Tests have been added to check the functionality of these APIs.
---
.../main/java/org/apache/helix/ConfigAccessor.java | 2 +-
.../src/main/java/org/apache/helix/HelixAdmin.java | 26 ++++++
.../org/apache/helix/manager/zk/ZKHelixAdmin.java | 82 +++++++++++++++++++
.../model/CustomizedStateAggregationConfig.java | 82 ++++++++++---------
.../apache/helix/manager/zk/TestZkHelixAdmin.java | 95 +++++++++++++++++++++-
.../java/org/apache/helix/mock/MockHelixAdmin.java | 22 +++++
.../TestCustomizedStateAggregationConfig.java | 15 ++--
7 files changed, 274 insertions(+), 50 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
index 5a41241..63f232e 100644
--- a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
+++ b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
@@ -606,7 +606,7 @@ public class ConfigAccessor {
return null;
}
- return new CustomizedStateAggregationConfig(record);
+ return new CustomizedStateAggregationConfig.Builder(record).build();
}
/**
diff --git a/helix-core/src/main/java/org/apache/helix/HelixAdmin.java b/helix-core/src/main/java/org/apache/helix/HelixAdmin.java
index 511a763..985d00f 100644
--- a/helix-core/src/main/java/org/apache/helix/HelixAdmin.java
+++ b/helix-core/src/main/java/org/apache/helix/HelixAdmin.java
@@ -26,6 +26,7 @@ import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ClusterConstraints.ConstraintType;
import org.apache.helix.model.ConstraintItem;
+import org.apache.helix.model.CustomizedStateAggregationConfig;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.HelixConfigScope;
import org.apache.helix.model.IdealState;
@@ -105,6 +106,31 @@ public interface HelixAdmin {
*/
void addClusterToGrandCluster(String clusterName, String grandCluster);
+ /** Add a CustomizedStateAggregationConfig to a cluster
+ * @param clusterName
+ * @param customizedStateAggregationConfig
+ */
+ void addCustomizedStateAggregationConfig(String clusterName,
+ CustomizedStateAggregationConfig customizedStateAggregationConfig);
+
+ /**
+ * Remove CustomizedStateAggregationConfig from specific cluster
+ * @param clusterName
+ */
+ void removeCustomizedStateAggregationConfig(String clusterName);
+
+ /**
+ * Add a type to CustomizedStateAggregationConfig of specific cluster
+ * @param clusterName
+ */
+ void addTypeToCustomizedStateAggregationConfig(String clusterName, String type);
+
+ /**
+ * Remove a type from CustomizedStateAggregationConfig of specific cluster
+ * @param clusterName
+ */
+ void removeTypeFromCustomizedStateAggregationConfig(String clusterName, String type);
+
/**
* Add a resource to a cluster, using the default ideal state mode AUTO
* @param clusterName
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
index 14f73f9..24e0a60 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
@@ -63,6 +63,7 @@ import org.apache.helix.model.ClusterConstraints.ConstraintType;
import org.apache.helix.model.ConstraintItem;
import org.apache.helix.model.ControllerHistory;
import org.apache.helix.model.CurrentState;
+import org.apache.helix.model.CustomizedStateAggregationConfig;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.HelixConfigScope;
import org.apache.helix.model.IdealState;
@@ -1215,6 +1216,87 @@ public class ZKHelixAdmin implements HelixAdmin {
}
@Override
+ public void addCustomizedStateAggregationConfig(String clusterName,
+ CustomizedStateAggregationConfig customizedStateAggregationConfig) {
+ logger.info(
+ "Add CustomizedStateAggregationConfig to cluster {}, CustomizedStateAggregationConfig is {}",
+ clusterName, customizedStateAggregationConfig.toString());
+
+ if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
+ throw new HelixException("cluster " + clusterName + " is not setup yet");
+ }
+
+ CustomizedStateAggregationConfig.Builder builder =
+ new CustomizedStateAggregationConfig.Builder(customizedStateAggregationConfig);
+ CustomizedStateAggregationConfig customizedStateAggregationConfigFromBuilder = builder.build();
+
+ ZKHelixDataAccessor accessor =
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
+ Builder keyBuilder = accessor.keyBuilder();
+ accessor.setProperty(keyBuilder.customizedStateAggregationConfig(),
+ customizedStateAggregationConfigFromBuilder);
+ }
+
+ @Override
+ public void removeCustomizedStateAggregationConfig(String clusterName) {
+ logger.info(
+ "Remove CustomizedStateAggregationConfig from cluster {}.", clusterName);
+
+ ZKHelixDataAccessor accessor =
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
+ Builder keyBuilder = accessor.keyBuilder();
+ accessor.removeProperty(keyBuilder.customizedStateAggregationConfig());
+
+ }
+
+ @Override
+ public void addTypeToCustomizedStateAggregationConfig(String clusterName, String type) {
+ logger.info("Add type {} to CustomizedStateAggregationConfig of cluster {}", type, clusterName);
+
+ if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
+ throw new HelixException("cluster " + clusterName + " is not setup yet");
+ }
+ CustomizedStateAggregationConfig.Builder builder =
+ new CustomizedStateAggregationConfig.Builder();
+
+ builder.addAggregationEnabledType(type);
+ CustomizedStateAggregationConfig customizedStateAggregationConfigFromBuilder = builder.build();
+
+ ZKHelixDataAccessor accessor =
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
+ Builder keyBuilder = accessor.keyBuilder();
+ accessor.updateProperty(keyBuilder.customizedStateAggregationConfig(),
+ customizedStateAggregationConfigFromBuilder);
+ }
+
+
+ @Override
+ public void removeTypeFromCustomizedStateAggregationConfig(String clusterName, String type) {
+ logger.info("Remove type {} to CustomizedStateAggregationConfig of cluster {}", type,
+ clusterName);
+
+ if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
+ throw new HelixException("cluster " + clusterName + " is not setup yet");
+ }
+
+ CustomizedStateAggregationConfig.Builder builder = new CustomizedStateAggregationConfig.Builder(
+ _configAccessor.getCustomizedStateAggregationConfig(clusterName));
+
+ if (!builder.getAggregationEnabledTypes().contains(type)) {
+ throw new HelixException("Type " + type
+ + " is missing from the CustomizedStateAggregationConfig of cluster " + clusterName);
+ }
+
+ builder.removeAggregationEnabledType(type);
+ CustomizedStateAggregationConfig customizedStateAggregationConfigFromBuilder = builder.build();
+ ZKHelixDataAccessor accessor =
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
+ Builder keyBuilder = accessor.keyBuilder();
+ accessor.setProperty(keyBuilder.customizedStateAggregationConfig(),
+ customizedStateAggregationConfigFromBuilder);
+ }
+
+ @Override
public List<String> getConfigKeys(HelixConfigScope scope) {
return _configAccessor.getKeys(scope);
}
diff --git a/helix-core/src/main/java/org/apache/helix/model/CustomizedStateAggregationConfig.java b/helix-core/src/main/java/org/apache/helix/model/CustomizedStateAggregationConfig.java
index 0e4a065..8d13fda 100644
--- a/helix-core/src/main/java/org/apache/helix/model/CustomizedStateAggregationConfig.java
+++ b/helix-core/src/main/java/org/apache/helix/model/CustomizedStateAggregationConfig.java
@@ -21,13 +21,20 @@ package org.apache.helix.model;
import java.util.ArrayList;
import java.util.List;
+
+import org.apache.helix.HelixException;
import org.apache.helix.HelixProperty;
-import org.apache.helix.ZNRecord;
+import org.apache.helix.zookeeper.datamodel.ZNRecord;
+
/**
* CustomizedStateAggregation configurations
*/
public class CustomizedStateAggregationConfig extends HelixProperty {
+
+ public static final String CUSTOMIZED_STATE_AGGREGATION_CONFIG_KW =
+ "CustomizedStateAggregationConfig";
+
/**
* Indicate which customized states will be aggregated.
* NOTE: Do NOT use this field name directly, use its corresponding getter/setter in the
@@ -39,10 +46,9 @@ public class CustomizedStateAggregationConfig extends HelixProperty {
/**
* Instantiate the CustomizedStateAggregationConfig
- * @param cluster
*/
- public CustomizedStateAggregationConfig(String cluster) {
- super(cluster);
+ public CustomizedStateAggregationConfig() {
+ super(CUSTOMIZED_STATE_AGGREGATION_CONFIG_KW);
}
/**
@@ -50,21 +56,10 @@ public class CustomizedStateAggregationConfig extends HelixProperty {
* @param record a ZNRecord corresponding to a CustomizedStateAggregationConfig
*/
public CustomizedStateAggregationConfig(ZNRecord record) {
- super(record);
- }
-
- /**
- * Instantiate the config using each field individually.
- * Users should use CustomizedStateAggregationConfig.Builder to create
- * CustomizedStateAggregationConfig.
- * @param cluster
- * @param aggregationEnabledTypes
- */
- public CustomizedStateAggregationConfig(String cluster, List<String> aggregationEnabledTypes) {
- super(cluster);
- _record.setListField(CustomizedStateAggregationProperty.AGGREGATION_ENABLED_TYPES.name(),
- aggregationEnabledTypes);
-
+ super(CUSTOMIZED_STATE_AGGREGATION_CONFIG_KW);
+ _record.setSimpleFields(record.getSimpleFields());
+ _record.setListFields(record.getListFields());
+ _record.setMapFields(record.getMapFields());
}
/**
@@ -86,25 +81,26 @@ public class CustomizedStateAggregationConfig extends HelixProperty {
}
public static class Builder {
- private String _clusterName = null;
- private List<String> _aggregationEnabledTypes;
+ private ZNRecord _record;
+
public CustomizedStateAggregationConfig build() {
- return new CustomizedStateAggregationConfig(_clusterName, _aggregationEnabledTypes);
+ return new CustomizedStateAggregationConfig(_record);
}
/**
* Default constructor
*/
public Builder() {
+ _record = new ZNRecord(CUSTOMIZED_STATE_AGGREGATION_CONFIG_KW);
}
/**
- * Constructor with Cluster Name as input
- * @param clusterName
+ * Instantiate with a pre-populated record
+ * @param record a ZNRecord corresponding to a Customized State Aggregation configuration
*/
- public Builder(String clusterName) {
- _clusterName = clusterName;
+ public Builder(ZNRecord record) {
+ _record = record;
}
/**
@@ -112,33 +108,39 @@ public class CustomizedStateAggregationConfig extends HelixProperty {
* @param customizedStateAggregationConfig
*/
public Builder(CustomizedStateAggregationConfig customizedStateAggregationConfig) {
- _aggregationEnabledTypes = customizedStateAggregationConfig.getAggregationEnabledTypes();
+ _record = customizedStateAggregationConfig.getRecord();
}
- public Builder setClusterName(String v) {
- _clusterName = v;
+ public Builder setAggregationEnabledTypes(List<String> aggregationEnabledTypes) {
+ _record.setListField(CustomizedStateAggregationProperty.AGGREGATION_ENABLED_TYPES.name(), aggregationEnabledTypes);
return this;
}
- public Builder setAggregationEnabledTypes(List<String> v) {
- _aggregationEnabledTypes = v;
+ public Builder addAggregationEnabledType(String type) {
+ if (_record.getListField(
+ CustomizedStateAggregationProperty.AGGREGATION_ENABLED_TYPES.name()) == null) {
+ _record.setListField(CustomizedStateAggregationProperty.AGGREGATION_ENABLED_TYPES.name(), new ArrayList<String>());
+ }
+ List<String> aggregationEnabledTypes = _record.getListField(CustomizedStateAggregationProperty.AGGREGATION_ENABLED_TYPES.name());
+ aggregationEnabledTypes.add(type);
+ _record.setListField(CustomizedStateAggregationProperty.AGGREGATION_ENABLED_TYPES.name(), aggregationEnabledTypes);
return this;
}
- public Builder addAggregationEnabledType(String v) {
- if (_aggregationEnabledTypes == null) {
- _aggregationEnabledTypes = new ArrayList<String>();
+ public Builder removeAggregationEnabledType(String type) {
+ if (!_record.getListField(CustomizedStateAggregationProperty.AGGREGATION_ENABLED_TYPES.name())
+ .contains(type)) {
+ throw new HelixException(
+ "Type " + type + " is missing from the CustomizedStateAggregationConfig");
}
- _aggregationEnabledTypes.add(v);
+ _record.getListField(CustomizedStateAggregationProperty.AGGREGATION_ENABLED_TYPES.name())
+ .remove(type);
return this;
}
- public String getClusterName() {
- return _clusterName;
- }
-
public List<String> getAggregationEnabledTypes() {
- return _aggregationEnabledTypes;
+ return _record
+ .getListField(CustomizedStateAggregationProperty.AGGREGATION_ENABLED_TYPES.name());
}
}
}
diff --git a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
index 639a4b1..61eb9e2 100644
--- a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
+++ b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
@@ -54,6 +54,7 @@ import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ClusterConstraints.ConstraintAttribute;
import org.apache.helix.model.ClusterConstraints.ConstraintType;
import org.apache.helix.model.ConstraintItem;
+import org.apache.helix.model.CustomizedStateAggregationConfig;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.HelixConfigScope;
import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
@@ -684,7 +685,6 @@ public class TestZkHelixAdmin extends ZkUnitTestBase {
Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor");
}
-
@Test
public void testRemoveCloudConfig() throws Exception {
String className = TestHelper.getTestClassName();
@@ -719,4 +719,95 @@ public class TestZkHelixAdmin extends ZkUnitTestBase {
cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertNull(cloudConfigFromZk);
}
-}
+
+ @Test
+ public void testAddCustomizedStateConfig() {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ HelixAdmin admin = new ZKHelixAdmin(ZK_ADDR);
+ admin.addCluster(clusterName, true);
+ CustomizedStateConfig.Builder builder =
+ new CustomizedStateConfig.Builder();
+ builder.addAggregationEnabledType("mockType1");
+ CustomizedStateConfig customizedStateConfig = builder.build();
+
+ admin.addCustomizedStateConfig(clusterName, customizedStateConfig);
+
+ // Read CustomizedStateConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
+ CustomizedStateConfig configFromZk =
+ _configAccessor.getCustomizedStateConfig(clusterName);
+ List<String> listTypesFromZk = configFromZk.getAggregationEnabledTypes();
+ Assert.assertEquals(listTypesFromZk.get(0), "mockType1");
+ }
+
+ @Test
+ public void testRemoveCustomizedStateConfig() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ HelixAdmin admin = new ZKHelixAdmin(ZK_ADDR);
+ admin.addCluster(clusterName, true);
+ CustomizedStateConfig.Builder builder =
+ new CustomizedStateConfig.Builder();
+ builder.addAggregationEnabledType("mockType1");
+ CustomizedStateConfig customizedStateConfig = builder.build();
+
+ admin.addCustomizedStateConfig(clusterName, customizedStateConfig);
+
+ // Read CustomizedStateConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
+ CustomizedStateConfig configFromZk =
+ _configAccessor.getCustomizedStateConfig(clusterName);
+ List<String> listTypesFromZk = configFromZk.getAggregationEnabledTypes();
+ Assert.assertEquals(listTypesFromZk.get(0), "mockType1");
+
+ // Remove CustomizedStateConfig Config and make sure it has been removed from
+ // Zookeeper
+ admin.removeCustomizedStateConfig(clusterName);
+ configFromZk = _configAccessor.getCustomizedStateConfig(clusterName);
+ Assert.assertNull(configFromZk);
+ }
+
+ @Test
+ public void testUpdateCustomizedStateConfig() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ HelixAdmin admin = new ZKHelixAdmin(ZK_ADDR);
+ admin.addCluster(clusterName, true);
+ CustomizedStateConfig.Builder builder =
+ new CustomizedStateConfig.Builder();
+ builder.addAggregationEnabledType("mockType1");
+ CustomizedStateConfig customizedStateConfig = builder.build();
+
+ admin.addCustomizedStateConfig(clusterName, customizedStateConfig);
+
+ // Read CustomizedStateConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
+ CustomizedStateConfig configFromZk =
+ _configAccessor.getCustomizedStateConfig(clusterName);
+ List<String> listTypesFromZk = configFromZk.getAggregationEnabledTypes();
+ Assert.assertEquals(listTypesFromZk.get(0), "mockType1");
+
+ admin.addTypeToCustomizedStateConfig(clusterName, "mockType2");
+ admin.addTypeToCustomizedStateConfig(clusterName, "mockType3");
+ configFromZk =
+ _configAccessor.getCustomizedStateConfig(clusterName);
+ listTypesFromZk = configFromZk.getAggregationEnabledTypes();
+ Assert.assertEquals(listTypesFromZk.get(0), "mockType1");
+ Assert.assertEquals(listTypesFromZk.get(1), "mockType2");
+ Assert.assertEquals(listTypesFromZk.get(2), "mockType3");
+
+ admin.removeTypeFromCustomizedStateConfig(clusterName, "mockType1");
+ configFromZk =
+ _configAccessor.getCustomizedStateConfig(clusterName);
+ listTypesFromZk = configFromZk.getAggregationEnabledTypes();
+ Assert.assertEquals(listTypesFromZk.get(0), "mockType2");
+ Assert.assertEquals(listTypesFromZk.get(1), "mockType3");
+ }
+}
\ No newline at end of file
diff --git a/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java b/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java
index 69fdb7b..595e435 100644
--- a/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java
+++ b/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java
@@ -35,6 +35,7 @@ import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ConstraintItem;
+import org.apache.helix.model.CustomizedStateAggregationConfig;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.HelixConfigScope;
import org.apache.helix.model.IdealState;
@@ -143,6 +144,27 @@ public class MockHelixAdmin implements HelixAdmin {
}
+ @Override
+ public void addCustomizedStateAggregationConfig(String clusterName,
+ CustomizedStateAggregationConfig customizedStateAggregationConfig) {
+
+ }
+
+ @Override
+ public void removeCustomizedStateAggregationConfig(String clusterName) {
+
+ }
+
+ @Override
+ public void addTypeToCustomizedStateAggregationConfig(String clusterName, String type) {
+
+ }
+
+ @Override
+ public void removeTypeFromCustomizedStateAggregationConfig(String clusterName, String type) {
+
+ }
+
@Override public void addResource(String clusterName, String resourceName, int numPartitions,
String stateModelRef) {
diff --git a/helix-core/src/test/java/org/apache/helix/model/TestCustomizedStateAggregationConfig.java b/helix-core/src/test/java/org/apache/helix/model/TestCustomizedStateAggregationConfig.java
index 5cdc333..2b9752a 100644
--- a/helix-core/src/test/java/org/apache/helix/model/TestCustomizedStateAggregationConfig.java
+++ b/helix-core/src/test/java/org/apache/helix/model/TestCustomizedStateAggregationConfig.java
@@ -62,12 +62,14 @@ public class TestCustomizedStateAggregationConfig extends ZkUnitTestBase {
TestHelper.setupEmptyCluster(_gZkClient, clusterName);
// Create dummy CustomizedStateAggregationConfig object
- CustomizedStateAggregationConfig customizedStateAggregationConfig =
- new CustomizedStateAggregationConfig(clusterName);
+ CustomizedStateAggregationConfig.Builder customizedStateAggregationConfigBuilder =
+ new CustomizedStateAggregationConfig.Builder();
List<String> aggregationEnabledTypes = new ArrayList<String>();
- aggregationEnabledTypes.add("mockState1");
- aggregationEnabledTypes.add("mockState2");
- customizedStateAggregationConfig.setAggregationEnabledTypes(aggregationEnabledTypes);
+ aggregationEnabledTypes.add("mockType1");
+ aggregationEnabledTypes.add("mockType2");
+ customizedStateAggregationConfigBuilder.setAggregationEnabledTypes(aggregationEnabledTypes);
+ CustomizedStateAggregationConfig customizedStateAggregationConfig =
+ customizedStateAggregationConfigBuilder.build();
// Write the CustomizedStateAggregationConfig to Zookeeper
ZKHelixDataAccessor accessor =
@@ -92,12 +94,11 @@ public class TestCustomizedStateAggregationConfig extends ZkUnitTestBase {
String clusterName = "CLUSTER_" + className;
TestHelper.setupEmptyCluster(_gZkClient, clusterName);
CustomizedStateAggregationConfig.Builder builder =
- new CustomizedStateAggregationConfig.Builder(clusterName);
+ new CustomizedStateAggregationConfig.Builder();
builder.addAggregationEnabledType("mockType1");
builder.addAggregationEnabledType("mockType2");
// Check builder getter methods
- Assert.assertEquals(builder.getClusterName(), clusterName);
List<String> aggregationEnabledTypes = builder.getAggregationEnabledTypes();
Assert.assertEquals(aggregationEnabledTypes.size(), 2);
Assert.assertEquals(aggregationEnabledTypes.get(0), "mockType1");
[helix] 09/23: Add intermediate storage for customized state (#827)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 55f3b60e55a9a450f3dba11856d392802cafc349
Author: Meng Zhang <mn...@linkedin.com>
AuthorDate: Thu Mar 5 10:24:17 2020 -0800
Add intermediate storage for customized state (#827)
1. Implement a participant state cache for generalizing functions in both current state cache and customized state cache.
2. Implement an intermediate data structure to store the result of customized state computation and prepare the data to be the format that can be used by customized view computation later.
---
.../helix/common/caches/CurrentStateCache.java | 149 ++---------------
.../helix/common/caches/CustomizedStateCache.java | 66 ++++++++
.../helix/common/caches/ParticipantStateCache.java | 184 +++++++++++++++++++++
.../dataproviders/BaseControllerDataProvider.java | 4 +-
.../controller/stages/CustomizedStateOutput.java | 120 ++++++++++++++
.../model/CustomizedStateAggregationConfig.java | 6 +-
.../apache/helix/spectator/RoutingDataCache.java | 2 +-
7 files changed, 390 insertions(+), 141 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/common/caches/CurrentStateCache.java b/helix-core/src/main/java/org/apache/helix/common/caches/CurrentStateCache.java
index d266196..fc7fa78 100644
--- a/helix-core/src/main/java/org/apache/helix/common/caches/CurrentStateCache.java
+++ b/helix-core/src/main/java/org/apache/helix/common/caches/CurrentStateCache.java
@@ -19,33 +19,26 @@ package org.apache.helix.common.caches;
* under the License.
*/
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import com.google.common.collect.Maps;
import org.apache.helix.HelixDataAccessor;
import org.apache.helix.PropertyKey;
import org.apache.helix.common.controllers.ControlContextProvider;
-import org.apache.helix.controller.LogUtil;
import org.apache.helix.model.CurrentState;
import org.apache.helix.model.LiveInstance;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
/**
* Cache to hold all CurrentStates of a cluster.
*/
-public class CurrentStateCache extends AbstractDataCache<CurrentState> {
+public class CurrentStateCache extends ParticipantStateCache<CurrentState> {
private static final Logger LOG = LoggerFactory.getLogger(CurrentStateCache.class.getName());
-
- private Map<String, Map<String, Map<String, CurrentState>>> _currentStateMap;
- private Map<PropertyKey, CurrentState> _currentStateCache = Maps.newHashMap();
- // If the cache is already refreshed with current state data.
+ // If the snapshot is already refreshed with current state data.
private boolean _initialized = false;
private CurrentStateSnapshot _snapshot;
@@ -55,149 +48,35 @@ public class CurrentStateCache extends AbstractDataCache<CurrentState> {
public CurrentStateCache(ControlContextProvider contextProvider) {
super(contextProvider);
- _currentStateMap = Collections.emptyMap();
- _snapshot = new CurrentStateSnapshot(_currentStateCache);
+ _snapshot = new CurrentStateSnapshot(_participantStateCache);
}
- /**
- * This refreshes the CurrentStates data by re-fetching the data from zookeeper in an efficient
- * way
- *
- * @param accessor
- * @param liveInstanceMap map of all liveInstances in cluster
- *
- * @return
- */
- public boolean refresh(HelixDataAccessor accessor,
- Map<String, LiveInstance> liveInstanceMap) {
- long startTime = System.currentTimeMillis();
-
- refreshCurrentStatesCache(accessor, liveInstanceMap);
- Map<String, Map<String, Map<String, CurrentState>>> allCurStateMap = new HashMap<>();
- for (PropertyKey key : _currentStateCache.keySet()) {
- CurrentState currentState = _currentStateCache.get(key);
- String[] params = key.getParams();
- if (currentState != null && params.length >= 4) {
- String instanceName = params[1];
- String sessionId = params[2];
- String stateName = params[3];
- Map<String, Map<String, CurrentState>> instanceCurStateMap =
- allCurStateMap.get(instanceName);
- if (instanceCurStateMap == null) {
- instanceCurStateMap = Maps.newHashMap();
- allCurStateMap.put(instanceName, instanceCurStateMap);
- }
- Map<String, CurrentState> sessionCurStateMap = instanceCurStateMap.get(sessionId);
- if (sessionCurStateMap == null) {
- sessionCurStateMap = Maps.newHashMap();
- instanceCurStateMap.put(sessionId, sessionCurStateMap);
- }
- sessionCurStateMap.put(stateName, currentState);
- }
- }
-
- for (String instance : allCurStateMap.keySet()) {
- allCurStateMap.put(instance, Collections.unmodifiableMap(allCurStateMap.get(instance)));
- }
- _currentStateMap = Collections.unmodifiableMap(allCurStateMap);
-
- long endTime = System.currentTimeMillis();
- LogUtil.logInfo(LOG, genEventInfo(),
- "END: CurrentStateCache.refresh() for cluster " + _controlContextProvider.getClusterName()
- + ", started at : " + startTime + ", took " + (endTime - startTime) + " ms");
- if (LOG.isDebugEnabled()) {
- LogUtil.logDebug(LOG, genEventInfo(),
- String.format("Current State refreshed : %s", _currentStateMap.toString()));
- }
- return true;
- }
-
- // reload current states that has been changed from zk to local cache.
- private void refreshCurrentStatesCache(HelixDataAccessor accessor,
+ @Override
+ protected Set<PropertyKey> PopulateParticipantKeys(HelixDataAccessor accessor,
Map<String, LiveInstance> liveInstanceMap) {
-
- long start = System.currentTimeMillis();
+ Set<PropertyKey> participantStateKeys = new HashSet<>();
PropertyKey.Builder keyBuilder = accessor.keyBuilder();
-
- Set<PropertyKey> currentStateKeys = new HashSet<>();
for (String instanceName : liveInstanceMap.keySet()) {
LiveInstance liveInstance = liveInstanceMap.get(instanceName);
String sessionId = liveInstance.getEphemeralOwner();
List<String> currentStateNames =
accessor.getChildNames(keyBuilder.currentStates(instanceName, sessionId));
for (String currentStateName : currentStateNames) {
- currentStateKeys.add(keyBuilder.currentState(instanceName, sessionId, currentStateName));
+ participantStateKeys
+ .add(keyBuilder.currentState(instanceName, sessionId, currentStateName));
}
}
- // All new entries from zk not cached locally yet should be read from ZK.
- Set<PropertyKey> reloadKeys = new HashSet<>(currentStateKeys);
- reloadKeys.removeAll(_currentStateCache.keySet());
-
- Set<PropertyKey> cachedKeys = new HashSet<>(_currentStateCache.keySet());
- cachedKeys.retainAll(currentStateKeys);
-
- Set<PropertyKey> reloadedKeys = new HashSet<>();
- Map<PropertyKey, CurrentState> newStateCache = Collections.unmodifiableMap(
- refreshProperties(accessor, reloadKeys, new ArrayList<>(cachedKeys),
- _currentStateCache, reloadedKeys));
+ return participantStateKeys;
+ }
- // if the cache was not initialized, the previous state should not be included in the snapshot
+ protected void refreshSnapshot(Map<PropertyKey, CurrentState> newStateCache,
+ Map<PropertyKey, CurrentState> participantStateCache, Set<PropertyKey> reloadedKeys) {
if (_initialized) {
- _snapshot = new CurrentStateSnapshot(newStateCache, _currentStateCache, reloadedKeys);
+ _snapshot = new CurrentStateSnapshot(newStateCache, participantStateCache, reloadedKeys);
} else {
_snapshot = new CurrentStateSnapshot(newStateCache);
_initialized = true;
}
-
- _currentStateCache = newStateCache;
-
- if (LOG.isDebugEnabled()) {
- LogUtil.logDebug(LOG, genEventInfo(),
- "# of CurrentStates reload: " + reloadKeys.size() + ", skipped:" + (
- currentStateKeys.size() - reloadKeys.size()) + ". took " + (System.currentTimeMillis()
- - start) + " ms to reload new current states for cluster: " + _controlContextProvider
- .getClusterName());
- }
- }
-
- /**
- * Return CurrentStates map for all instances.
- *
- * @return
- */
- public Map<String, Map<String, Map<String, CurrentState>>> getCurrentStatesMap() {
- return Collections.unmodifiableMap(_currentStateMap);
- }
-
- /**
- * Return all CurrentState on the given instance.
- *
- * @param instance
- *
- * @return
- */
- public Map<String, Map<String, CurrentState>> getCurrentStates(String instance) {
- if (!_currentStateMap.containsKey(instance)) {
- return Collections.emptyMap();
- }
- return Collections.unmodifiableMap(_currentStateMap.get(instance));
- }
-
- /**
- * Provides the current state of the node for a given session id, the sessionid can be got from
- * LiveInstance
- *
- * @param instance
- * @param clientSessionId
- *
- * @return
- */
- public Map<String, CurrentState> getCurrentState(String instance, String clientSessionId) {
- if (!_currentStateMap.containsKey(instance) || !_currentStateMap.get(instance)
- .containsKey(clientSessionId)) {
- return Collections.emptyMap();
- }
- return Collections.unmodifiableMap(_currentStateMap.get(instance).get(clientSessionId));
}
@Override
diff --git a/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedStateCache.java b/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedStateCache.java
new file mode 100644
index 0000000..1f33948
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedStateCache.java
@@ -0,0 +1,66 @@
+package org.apache.helix.common.caches;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.PropertyKey;
+import org.apache.helix.common.controllers.ControlContextProvider;
+import org.apache.helix.model.CustomizedState;
+import org.apache.helix.model.CustomizedStateAggregationConfig;
+import org.apache.helix.model.LiveInstance;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class CustomizedStateCache extends ParticipantStateCache<CustomizedState> {
+ private static final Logger LOG = LoggerFactory.getLogger(CurrentStateCache.class.getName());
+
+ public CustomizedStateCache(String clusterName) {
+ this(createDefaultControlContextProvider(clusterName));
+ }
+
+ public CustomizedStateCache(ControlContextProvider contextProvider) {
+ super(contextProvider);
+ }
+
+ @Override
+ protected Set<PropertyKey> PopulateParticipantKeys(HelixDataAccessor accessor,
+ Map<String, LiveInstance> liveInstanceMap) {
+ Set<PropertyKey> participantStateKeys = new HashSet<>();
+ PropertyKey.Builder keyBuilder = accessor.keyBuilder();
+ Set<String> restrictedKeys = new HashSet<>(
+ accessor.getProperty(accessor.keyBuilder().customizedStateAggregationConfig()).getRecord()
+ .getListFields().get(
+ CustomizedStateAggregationConfig.CustomizedStateAggregationProperty.AGGREGATION_ENABLED_TYPES
+ .name()));
+ for (String instanceName : liveInstanceMap.keySet()) {
+ for (String customizedStateType : restrictedKeys) {
+ accessor.getChildNames(keyBuilder.customizedStates(instanceName, customizedStateType))
+ .stream().forEach(resourceName -> participantStateKeys
+ .add(keyBuilder.customizedState(instanceName, customizedStateType, resourceName)));
+ }
+ }
+ return participantStateKeys;
+ }
+}
diff --git a/helix-core/src/main/java/org/apache/helix/common/caches/ParticipantStateCache.java b/helix-core/src/main/java/org/apache/helix/common/caches/ParticipantStateCache.java
new file mode 100644
index 0000000..61556b1
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/common/caches/ParticipantStateCache.java
@@ -0,0 +1,184 @@
+package org.apache.helix.common.caches;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import com.google.common.collect.Maps;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.PropertyKey;
+import org.apache.helix.common.controllers.ControlContextProvider;
+import org.apache.helix.controller.LogUtil;
+import org.apache.helix.model.LiveInstance;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Represent a cache that holds a certain participant side state of for the whole cluster.
+ */
+public abstract class ParticipantStateCache<T> extends AbstractDataCache {
+ private static Logger LOG = LoggerFactory.getLogger(ParticipantStateCache.class);
+ protected Map<String, Map<String, Map<String, T>>> _participantStateMap;
+
+ protected Map<PropertyKey, T> _participantStateCache = Maps.newHashMap();
+
+ public ParticipantStateCache(ControlContextProvider controlContextProvider) {
+ super(controlContextProvider);
+ _participantStateMap = new HashMap<>();
+ }
+
+ /**
+ * This refreshes the participant state cache data by re-fetching the data from zookeeper in an
+ * efficient way
+ * @param accessor
+ * @param liveInstanceMap map of all liveInstances in cluster
+ * @return
+ */
+ public boolean refresh(HelixDataAccessor accessor, Map<String, LiveInstance> liveInstanceMap) {
+ long startTime = System.currentTimeMillis();
+
+ refreshParticipantStatesCacheFromZk(accessor, liveInstanceMap);
+ Map<String, Map<String, Map<String, T>>> allParticipantStateMap = new HashMap<>();
+ // There should be 4 levels of keys. The first one is the cluster name, the second one is the
+ // instance name, the third one is a customized key (could be session Id or customized state
+ // type), the fourth one is the resourceName
+ for (PropertyKey key : _participantStateCache.keySet()) {
+ T participantState = _participantStateCache.get(key);
+ String[] params = key.getParams();
+ if (participantState != null && params.length >= 4) {
+ String instanceName = params[1];
+ String customizedName = params[2];
+ String resourceName = params[3];
+ Map<String, Map<String, T>> instanceMap = allParticipantStateMap.get(instanceName);
+ if (instanceMap == null) {
+ instanceMap = Maps.newHashMap();
+ allParticipantStateMap.put(instanceName, instanceMap);
+ }
+ Map<String, T> customizedMap = instanceMap.get(customizedName);
+ if (customizedMap == null) {
+ customizedMap = Maps.newHashMap();
+ instanceMap.put(customizedName, customizedMap);
+ }
+ customizedMap.put(resourceName, participantState);
+ } else {
+ LogUtil.logError(LOG, genEventInfo(),
+ "Invalid key found in the participant state cache" + key);
+ }
+ }
+
+ _participantStateMap = Collections.unmodifiableMap(allParticipantStateMap);
+
+ long endTime = System.currentTimeMillis();
+ LogUtil.logInfo(LOG, genEventInfo(),
+ "END: participantStateCache.refresh() for cluster " + _controlContextProvider
+ .getClusterName() + ", started at : " + startTime + ", took " + (endTime - startTime)
+ + " ms");
+ if (LOG.isDebugEnabled()) {
+ LogUtil.logDebug(LOG, genEventInfo(),
+ String.format("Participant State refreshed : %s", _participantStateMap.toString()));
+ }
+ return true;
+ }
+
+ // reload participant states that has been changed from zk to local cache.
+ private void refreshParticipantStatesCacheFromZk(HelixDataAccessor accessor,
+ Map<String, LiveInstance> liveInstanceMap) {
+
+ long start = System.currentTimeMillis();
+ Set<PropertyKey> participantStateKeys = PopulateParticipantKeys(accessor, liveInstanceMap);
+
+ // All new entries from zk not cached locally yet should be read from ZK.
+ Set<PropertyKey> reloadKeys = new HashSet<>(participantStateKeys);
+ reloadKeys.removeAll(_participantStateCache.keySet());
+
+ Set<PropertyKey> cachedKeys = new HashSet<>(_participantStateCache.keySet());
+ cachedKeys.retainAll(participantStateKeys);
+
+ Set<PropertyKey> reloadedKeys = new HashSet<>();
+ Map<PropertyKey, T> newStateCache = Collections.unmodifiableMap(
+ refreshProperties(accessor, reloadKeys, new ArrayList<>(cachedKeys), _participantStateCache,
+ reloadedKeys));
+
+ refreshSnapshot(newStateCache, _participantStateCache, reloadedKeys);
+
+ _participantStateCache = newStateCache;
+
+ if (LOG.isDebugEnabled()) {
+ LogUtil.logDebug(LOG, genEventInfo(),
+ "# of participant state reload: " + reloadKeys.size() + ", skipped:" + (
+ participantStateKeys.size() - reloadKeys.size()) + ". took " + (
+ System.currentTimeMillis() - start)
+ + " ms to reload new participant states for cluster: " + _controlContextProvider
+ .getClusterName() + "and state: " + this.getClass().getName());
+ }
+ }
+
+ protected abstract Set<PropertyKey> PopulateParticipantKeys(HelixDataAccessor accessor,
+ Map<String, LiveInstance> liveInstanceMap);
+
+ /**
+ * Refresh the snapshot of the cache. This method is optional for child class to extend. If the
+ * child class does not need to refresh snapshot, it just does nothing.
+ * @return
+ */
+ protected void refreshSnapshot(Map<PropertyKey, T> newStateCache,
+ Map<PropertyKey, T> participantStateCache, Set<PropertyKey> reloadedKeys) {
+ }
+
+ /**
+ * Return the whole participant state map.
+ * @return
+ */
+ public Map<String, Map<String, Map<String, T>>> getParticipantStatesMap() {
+ return Collections.unmodifiableMap(_participantStateMap);
+ }
+
+ /**
+ * Return all participant states for a certain instance.
+ * @param instanceName
+ * @return
+ */
+ public Map<String, Map<String, T>> getParticipantStates(String instanceName) {
+ if (!_participantStateMap.containsKey(instanceName)) {
+ return Collections.emptyMap();
+ }
+ return Collections.unmodifiableMap(_participantStateMap.get(instanceName));
+ }
+
+ /**
+ * Provides the participant state map for a certain instance and a customized key
+ * @param instanceName
+ * @param customizedKey
+ * @return
+ */
+ public Map<String, T> getParticipantState(String instanceName, String customizedKey) {
+ if (!_participantStateMap.containsKey(instanceName) || !_participantStateMap.get(instanceName)
+ .containsKey(customizedKey)) {
+ return Collections.emptyMap();
+ }
+ return Collections.unmodifiableMap(_participantStateMap.get(instanceName).get(customizedKey));
+ }
+}
diff --git a/helix-core/src/main/java/org/apache/helix/controller/dataproviders/BaseControllerDataProvider.java b/helix-core/src/main/java/org/apache/helix/controller/dataproviders/BaseControllerDataProvider.java
index 405af47..0cd9355 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/dataproviders/BaseControllerDataProvider.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/dataproviders/BaseControllerDataProvider.java
@@ -337,7 +337,7 @@ public class BaseControllerDataProvider implements ControlContextProvider {
// current state must be refreshed before refreshing relay messages
// because we need to use current state to validate all relay messages.
_instanceMessagesCache.updateRelayMessages(_liveInstanceCache.getPropertyMap(),
- _currentStateCache.getCurrentStatesMap());
+ _currentStateCache.getParticipantStatesMap());
updateIdealRuleMap();
updateDisabledInstances();
@@ -526,7 +526,7 @@ public class BaseControllerDataProvider implements ControlContextProvider {
* @return
*/
public Map<String, CurrentState> getCurrentState(String instanceName, String clientSessionId) {
- return _currentStateCache.getCurrentState(instanceName, clientSessionId);
+ return _currentStateCache.getParticipantState(instanceName, clientSessionId);
}
/**
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/CustomizedStateOutput.java b/helix-core/src/main/java/org/apache/helix/controller/stages/CustomizedStateOutput.java
new file mode 100644
index 0000000..3863743
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/controller/stages/CustomizedStateOutput.java
@@ -0,0 +1,120 @@
+package org.apache.helix.controller.stages;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.helix.model.Partition;
+
+
+public class CustomizedStateOutput {
+ // stateType -> (resourceName -> (Partition -> (instanceName -> customizedState)))
+ private final Map<String, Map<String, Map<Partition, Map<String, String>>>> _customizedStateMap;
+
+ public CustomizedStateOutput() {
+ _customizedStateMap = new HashMap<>();
+ }
+
+ public void setCustomizedState(String stateType, String resourceName, Partition partition,
+ String instanceName, String state) {
+ if (!_customizedStateMap.containsKey(stateType)) {
+ _customizedStateMap
+ .put(stateType, new HashMap<String, Map<Partition, Map<String, String>>>());
+ }
+ if (!_customizedStateMap.get(stateType).containsKey(resourceName)) {
+ _customizedStateMap.get(stateType)
+ .put(resourceName, new HashMap<Partition, Map<String, String>>());
+ }
+ if (!_customizedStateMap.get(stateType).get(resourceName).containsKey(partition)) {
+ _customizedStateMap.get(stateType).get(resourceName)
+ .put(partition, new HashMap<String, String>());
+ }
+ _customizedStateMap.get(stateType).get(resourceName).get(partition).put(instanceName, state);
+ }
+
+ /**
+ * Given stateType, returns resource customized state map (resource -> parition -> instance ->
+ * customizedState)
+ * @param stateType
+ * @return
+ */
+ public Map<String, Map<Partition, Map<String, String>>> getCustomizedStateMap(String stateType) {
+ if (_customizedStateMap.containsKey(stateType)) {
+ return Collections.unmodifiableMap(_customizedStateMap.get(stateType));
+ }
+ return Collections.emptyMap();
+ }
+
+ /**
+ * given (stateType, resource), returns (partition -> instance-> customizedState) map
+ * @param stateType
+ * @param resourceName
+ * @return
+ */
+ public Map<Partition, Map<String, String>> getResourceCustomizedStateMap(String stateType,
+ String resourceName) {
+ if (getCustomizedStateMap(stateType).containsKey(resourceName)) {
+ return Collections.unmodifiableMap(getCustomizedStateMap(stateType).get(resourceName));
+ }
+ return Collections.emptyMap();
+ }
+
+ /**
+ * given (stateType, resource, partition), returns (instance-> customizedState) map
+ * @param stateType
+ * @param resourceName
+ * @param partition
+ * @return
+ */
+ public Map<String, String> getPartitionCustomizedStateMap(String stateType, String resourceName,
+ Partition partition) {
+ if (getCustomizedStateMap(stateType).containsKey(resourceName) && getResourceCustomizedStateMap(
+ stateType, resourceName).containsKey(partition)) {
+ return Collections
+ .unmodifiableMap(getResourceCustomizedStateMap(stateType, resourceName).get(partition));
+ }
+ return Collections.emptyMap();
+ }
+
+ /**
+ * given (stateType, resource, partition, instance), returns customized state
+ * @param stateType
+ * @param resourceName
+ * @param partition
+ * @param instanceName
+ * @return
+ */
+ public String getPartitionCustomizedState(String stateType, String resourceName,
+ Partition partition, String instanceName) {
+ if (getCustomizedStateMap(stateType).containsKey(resourceName) && getResourceCustomizedStateMap(
+ stateType, resourceName).containsKey(partition) && getPartitionCustomizedStateMap(stateType,
+ resourceName, partition).containsKey(instanceName)) {
+ return getPartitionCustomizedStateMap(stateType, resourceName, partition).get(instanceName);
+ }
+ return null;
+ }
+
+ public Set<String> getAllStateTypes() {
+ return _customizedStateMap.keySet();
+ }
+}
diff --git a/helix-core/src/main/java/org/apache/helix/model/CustomizedStateAggregationConfig.java b/helix-core/src/main/java/org/apache/helix/model/CustomizedStateAggregationConfig.java
index 8d13fda..4acd0e6 100644
--- a/helix-core/src/main/java/org/apache/helix/model/CustomizedStateAggregationConfig.java
+++ b/helix-core/src/main/java/org/apache/helix/model/CustomizedStateAggregationConfig.java
@@ -63,7 +63,7 @@ public class CustomizedStateAggregationConfig extends HelixProperty {
}
/**
- * Set the AGGREGATION_ENABLED_STATES field.
+ * Set the AGGREGATION_ENABLED_TYPES field.
* @param aggregationEnabledTypes
*/
public void setAggregationEnabledTypes(List<String> aggregationEnabledTypes) {
@@ -72,8 +72,8 @@ public class CustomizedStateAggregationConfig extends HelixProperty {
}
/**
- * Get the AGGREGATION_ENABLED_STATES field.
- * @return AGGREGATION_ENABLED_STATES field.
+ * Get the AGGREGATION_ENABLED_TYPES field.
+ * @return AGGREGATION_ENABLED_TYPES field.
*/
public List<String> getAggregationEnabledTypes() {
return _record
diff --git a/helix-core/src/main/java/org/apache/helix/spectator/RoutingDataCache.java b/helix-core/src/main/java/org/apache/helix/spectator/RoutingDataCache.java
index 7af0d28..4896455 100644
--- a/helix-core/src/main/java/org/apache/helix/spectator/RoutingDataCache.java
+++ b/helix-core/src/main/java/org/apache/helix/spectator/RoutingDataCache.java
@@ -121,7 +121,7 @@ class RoutingDataCache extends BasicClusterDataCache {
* @return
*/
public Map<String, Map<String, Map<String, CurrentState>>> getCurrentStatesMap() {
- return _currentStateCache.getCurrentStatesMap();
+ return _currentStateCache.getParticipantStatesMap();
}
public CurrentStateSnapshot getCurrentStateSnapshot() {
[helix] 06/23: Add basic functionalities for RoutingTableProvider
for CustomizedView (#814)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 84d0fe72c0948500e8ee69a4466ee4fadd516e9e
Author: Ali Reza Zamani Zadeh Najari <an...@linkedin.com>
AuthorDate: Wed Feb 26 16:29:27 2020 -0800
Add basic functionalities for RoutingTableProvider for CustomizedView (#814)
This commit contains the basic functionalities for
CustomizedView RoutingTableProvider.
Here are the new added functionalities:
1- CustomizedViewChangeListener
2- Addition of CustomizedView to helix PropertyType and PropertyKey
3- Implementation of CustomizedView cache.
4- Registering CallbackHandler for CustomizedView.
---
.../main/java/org/apache/helix/HelixConstants.java | 1 +
.../main/java/org/apache/helix/HelixManager.java | 7 +
.../main/java/org/apache/helix/PropertyKey.java | 30 +++++
.../main/java/org/apache/helix/PropertyType.java | 1 +
.../listeners/CustomizedViewChangeListener.java} | 37 +++---
.../helix/common/caches/CustomizedViewCache.java | 146 +++++++++++++++++++++
.../helix/controller/stages/ClusterEventType.java | 1 +
.../apache/helix/manager/zk/CallbackHandler.java | 13 ++
.../apache/helix/manager/zk/ZKHelixManager.java | 10 ++
.../controller/stages/DummyClusterManager.java | 8 ++
.../manager/TestParticipantManager.java | 2 +-
.../java/org/apache/helix/mock/MockManager.java | 8 ++
.../helix/participant/MockZKHelixManager.java | 8 ++
13 files changed, 253 insertions(+), 19 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/HelixConstants.java b/helix-core/src/main/java/org/apache/helix/HelixConstants.java
index f84173b..b0783bf 100644
--- a/helix-core/src/main/java/org/apache/helix/HelixConstants.java
+++ b/helix-core/src/main/java/org/apache/helix/HelixConstants.java
@@ -35,6 +35,7 @@ public interface HelixConstants {
CURRENT_STATE (PropertyType.CURRENTSTATES),
MESSAGE (PropertyType.MESSAGES),
EXTERNAL_VIEW (PropertyType.EXTERNALVIEW),
+ CUSTOMIZED_VIEW (PropertyType.CUSTOMIZEDVIEW),
TARGET_EXTERNAL_VIEW (PropertyType.TARGETEXTERNALVIEW),
CONTROLLER (PropertyType.CONTROLLER),
MESSAGES_CONTROLLER (PropertyType.MESSAGES_CONTROLLER),
diff --git a/helix-core/src/main/java/org/apache/helix/HelixManager.java b/helix-core/src/main/java/org/apache/helix/HelixManager.java
index 2191153..2be5ce4 100644
--- a/helix-core/src/main/java/org/apache/helix/HelixManager.java
+++ b/helix-core/src/main/java/org/apache/helix/HelixManager.java
@@ -27,6 +27,7 @@ import org.apache.helix.api.listeners.ConfigChangeListener;
import org.apache.helix.api.listeners.ControllerChangeListener;
import org.apache.helix.api.listeners.CurrentStateChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
+import org.apache.helix.api.listeners.CustomizedViewChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
import org.apache.helix.api.listeners.InstanceConfigChangeListener;
import org.apache.helix.api.listeners.LiveInstanceChangeListener;
@@ -220,6 +221,12 @@ public interface HelixManager {
void addExternalViewChangeListener(ExternalViewChangeListener listener) throws Exception;
/**
+ * @see CustomizedViewChangeListener#onCustomizedViewChange(List, NotificationContext)
+ * @param listener
+ */
+ void addCustomizedViewChangeListener(CustomizedViewChangeListener listener, String aggregationType) throws Exception;
+
+ /**
* @see ExternalViewChangeListener#onExternalViewChange(List, NotificationContext)
* @param listener
*/
diff --git a/helix-core/src/main/java/org/apache/helix/PropertyKey.java b/helix-core/src/main/java/org/apache/helix/PropertyKey.java
index 39f18c9..fc343a8 100644
--- a/helix-core/src/main/java/org/apache/helix/PropertyKey.java
+++ b/helix-core/src/main/java/org/apache/helix/PropertyKey.java
@@ -29,6 +29,7 @@ import org.apache.helix.model.ControllerHistory;
import org.apache.helix.model.CurrentState;
import org.apache.helix.model.CustomizedStateAggregationConfig;
import org.apache.helix.model.CustomizedState;
+import org.apache.helix.model.CustomizedView;
import org.apache.helix.model.Error;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.HealthStat;
@@ -58,6 +59,7 @@ import static org.apache.helix.PropertyType.CUSTOMIZEDSTATES;
import static org.apache.helix.PropertyType.ERRORS;
import static org.apache.helix.PropertyType.ERRORS_CONTROLLER;
import static org.apache.helix.PropertyType.EXTERNALVIEW;
+import static org.apache.helix.PropertyType.CUSTOMIZEDVIEW;
import static org.apache.helix.PropertyType.HISTORY;
import static org.apache.helix.PropertyType.IDEALSTATES;
import static org.apache.helix.PropertyType.INSTANCE_HISTORY;
@@ -73,6 +75,7 @@ import static org.apache.helix.PropertyType.STATUSUPDATES;
import static org.apache.helix.PropertyType.STATUSUPDATES_CONTROLLER;
import static org.apache.helix.PropertyType.TARGETEXTERNALVIEW;
+
/**
* Key allowing for type-safe lookups of and conversions to {@link HelixProperty} objects.
*/
@@ -639,6 +642,33 @@ public class PropertyKey {
}
/**
+ * Get a property key associated with all {@link CustomizedView}
+ * @return {@link PropertyKey}
+ */
+ public PropertyKey customizedView() {
+ return new PropertyKey(CUSTOMIZEDVIEW, CustomizedView.class, _clusterName);
+ }
+
+ /**
+ * Get a property key associated with an {@link CustomizedView} of a type
+ * @param aggregationType
+ * @return {@link PropertyKey}
+ */
+ public PropertyKey customizedView(String aggregationType) {
+ return new PropertyKey(CUSTOMIZEDVIEW, CustomizedView.class, _clusterName, aggregationType);
+ }
+
+ /**
+ * Get a property key associated with an {@link CustomizedView} of a type and resource
+ * @param aggregationType
+ * @return {@link PropertyKey}
+ */
+ public PropertyKey customizedView(String aggregationType, String resourceName) {
+ return new PropertyKey(CUSTOMIZEDVIEW, CustomizedView.class, _clusterName, aggregationType,
+ resourceName);
+ }
+
+ /**
* Get a property key associated with all target external view
* @return {@link PropertyKey}
*/
diff --git a/helix-core/src/main/java/org/apache/helix/PropertyType.java b/helix-core/src/main/java/org/apache/helix/PropertyType.java
index e076322..363db21 100644
--- a/helix-core/src/main/java/org/apache/helix/PropertyType.java
+++ b/helix-core/src/main/java/org/apache/helix/PropertyType.java
@@ -43,6 +43,7 @@ public enum PropertyType {
INSTANCES(Type.CLUSTER, true, false),
IDEALSTATES(Type.CLUSTER, true, false, false, false, true),
EXTERNALVIEW(Type.CLUSTER, true, false),
+ CUSTOMIZEDVIEW(Type.CLUSTER, true, false),
TARGETEXTERNALVIEW(Type.CLUSTER, true, false),
STATEMODELDEFS(Type.CLUSTER, true, false, false, false, true),
CONTROLLER(Type.CLUSTER, true, false),
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterEventType.java b/helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedViewChangeListener.java
similarity index 59%
copy from helix-core/src/main/java/org/apache/helix/controller/stages/ClusterEventType.java
copy to helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedViewChangeListener.java
index 7e9ab15..4cd4e7a 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterEventType.java
+++ b/helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedViewChangeListener.java
@@ -1,4 +1,4 @@
-package org.apache.helix.controller.stages;
+package org.apache.helix.api.listeners;
/*
* Licensed to the Apache Software Foundation (ASF) under one
@@ -19,21 +19,22 @@ package org.apache.helix.controller.stages;
* under the License.
*/
-public enum ClusterEventType {
- IdealStateChange,
- CurrentStateChange,
- ConfigChange,
- ClusterConfigChange,
- ResourceConfigChange,
- InstanceConfigChange,
- LiveInstanceChange,
- MessageChange,
- ExternalViewChange,
- TargetExternalViewChange,
- Resume,
- PeriodicalRebalance,
- OnDemandRebalance,
- RetryRebalance,
- StateVerifier,
- Unknown
+import java.util.List;
+
+import org.apache.helix.NotificationContext;
+import org.apache.helix.model.CustomizedView;
+
+/**
+ * Interface to implement to respond to changes in the CustomizedView
+ */
+public interface CustomizedViewChangeListener {
+
+ /**
+ * Invoked when customized view changes
+ * @param customizedViewList a list of CustomizedViews
+ * @param changeContext the change event and state
+ */
+ void onCustomizedViewChange(List<CustomizedView> customizedViewList,
+ NotificationContext changeContext);
+
}
diff --git a/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java b/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java
new file mode 100644
index 0000000..493ca69
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java
@@ -0,0 +1,146 @@
+package org.apache.helix.common.caches;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.HelixException;
+import org.apache.helix.PropertyKey;
+import org.apache.helix.PropertyType;
+import org.apache.helix.model.CustomizedView;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Maps;
+
+/**
+ * Cache to hold all CustomizedView of a specific type.
+ */
+public class CustomizedViewCache extends AbstractDataCache<CustomizedView> {
+ private static final Logger LOG = LoggerFactory.getLogger(CustomizedViewCache.class.getName());
+
+ protected Map<String, CustomizedView> _customizedViewMap;
+ protected Map<String, CustomizedView> _customizedViewCache;
+ protected String _clusterName;
+ private PropertyType _propertyType;
+ private String _aggregationType;
+
+ public CustomizedViewCache(String clusterName, String aggregationType) {
+ this(clusterName, PropertyType.CUSTOMIZEDVIEW, aggregationType);
+ }
+
+ protected CustomizedViewCache(String clusterName, PropertyType propertyType, String aggregationType) {
+ super(createDefaultControlContextProvider(clusterName));
+ _clusterName = clusterName;
+ _customizedViewMap = Collections.emptyMap();
+ _customizedViewCache = Collections.emptyMap();
+ _propertyType = propertyType;
+ _aggregationType = aggregationType;
+ }
+
+
+ /**
+ * This refreshes the CustomizedView data by re-fetching the data from zookeeper in an efficient
+ * way
+ * @param accessor
+ * @return
+ */
+ public void refresh(HelixDataAccessor accessor) {
+ long startTime = System.currentTimeMillis();
+ PropertyKey.Builder keyBuilder = accessor.keyBuilder();
+ Set<PropertyKey> currentPropertyKeys = new HashSet<>();
+
+ List<String> resources = accessor.getChildNames(customizedViewsKey(keyBuilder));
+
+ for (String resource : resources) {
+ currentPropertyKeys.add(customizedViewKey(keyBuilder, resource));
+ }
+
+ Set<PropertyKey> cachedKeys = new HashSet<>();
+ Map<PropertyKey, CustomizedView> cachedCustomizedViewMap = Maps.newHashMap();
+ for (String resource : _customizedViewCache.keySet()) {
+ PropertyKey key = customizedViewKey(keyBuilder, resource);
+ cachedKeys.add(key);
+ cachedCustomizedViewMap.put(key, _customizedViewCache.get(resource));
+ }
+ cachedKeys.retainAll(currentPropertyKeys);
+
+ Set<PropertyKey> reloadKeys = new HashSet<>(currentPropertyKeys);
+ reloadKeys.removeAll(cachedKeys);
+
+ Map<PropertyKey, CustomizedView> updatedMap =
+ refreshProperties(accessor, reloadKeys, new ArrayList<>(cachedKeys),
+ cachedCustomizedViewMap, new HashSet<>());
+
+ Map<String, CustomizedView> newCustomizedViewMap = Maps.newHashMap();
+ for (CustomizedView customizedView : updatedMap.values()) {
+ newCustomizedViewMap.put(customizedView.getResourceName(), customizedView);
+ }
+
+ _customizedViewCache = new HashMap<>(newCustomizedViewMap);
+ _customizedViewMap = new HashMap<>(newCustomizedViewMap);
+
+ long endTime = System.currentTimeMillis();
+ LOG.info("Refresh " + _customizedViewMap.size() + " CustomizedViews of type " + _aggregationType
+ + " for cluster " + _clusterName + ", took " + (endTime - startTime) + " ms");
+ }
+
+ private PropertyKey customizedViewsKey(PropertyKey.Builder keyBuilder) {
+ PropertyKey customizedViewPropertyKey;
+ if (_propertyType.equals(PropertyType.CUSTOMIZEDVIEW)){
+ customizedViewPropertyKey = keyBuilder.customizedView(_aggregationType);
+ } else {
+ throw new HelixException(
+ "Failed to refresh CustomizedViewCache, Wrong property type " + _propertyType + "!");
+ }
+ return customizedViewPropertyKey;
+ }
+
+ private PropertyKey customizedViewKey(PropertyKey.Builder keyBuilder, String resource) {
+ PropertyKey customizedViewPropertyKey;
+ if (_propertyType.equals(PropertyType.CUSTOMIZEDVIEW)) {
+ customizedViewPropertyKey = keyBuilder.customizedView(_aggregationType, resource);
+ } else {
+ throw new HelixException(
+ "Failed to refresh CustomizedViewCache, Wrong property type " + _propertyType + "!");
+ }
+ return customizedViewPropertyKey;
+ }
+
+ /**
+ * Return CustomizedView map for all resources.
+ * @return
+ */
+ public Map<String, CustomizedView> getCustomizedViewMap() {
+ return Collections.unmodifiableMap(_customizedViewMap);
+ }
+
+ public void clear() {
+ _customizedViewCache.clear();
+ _customizedViewMap.clear();
+ }
+}
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterEventType.java b/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterEventType.java
index 7e9ab15..2fd015e 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterEventType.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterEventType.java
@@ -29,6 +29,7 @@ public enum ClusterEventType {
LiveInstanceChange,
MessageChange,
ExternalViewChange,
+ CustomizedViewChange,
TargetExternalViewChange,
Resume,
PeriodicalRebalance,
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/CallbackHandler.java b/helix-core/src/main/java/org/apache/helix/manager/zk/CallbackHandler.java
index bbb8788..dc91028 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/CallbackHandler.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/CallbackHandler.java
@@ -45,6 +45,7 @@ import org.apache.helix.api.listeners.ClusterConfigChangeListener;
import org.apache.helix.api.listeners.ConfigChangeListener;
import org.apache.helix.api.listeners.ControllerChangeListener;
import org.apache.helix.api.listeners.CurrentStateChangeListener;
+import org.apache.helix.api.listeners.CustomizedViewChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
import org.apache.helix.api.listeners.InstanceConfigChangeListener;
@@ -56,6 +57,7 @@ import org.apache.helix.api.listeners.ScopedConfigChangeListener;
import org.apache.helix.common.DedupEventProcessor;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.CurrentState;
+import org.apache.helix.model.CustomizedView;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.InstanceConfig;
@@ -78,6 +80,7 @@ import static org.apache.helix.HelixConstants.ChangeType.CONFIG;
import static org.apache.helix.HelixConstants.ChangeType.CONTROLLER;
import static org.apache.helix.HelixConstants.ChangeType.CURRENT_STATE;
import static org.apache.helix.HelixConstants.ChangeType.EXTERNAL_VIEW;
+import static org.apache.helix.HelixConstants.ChangeType.CUSTOMIZED_VIEW;
import static org.apache.helix.HelixConstants.ChangeType.IDEAL_STATE;
import static org.apache.helix.HelixConstants.ChangeType.INSTANCE_CONFIG;
import static org.apache.helix.HelixConstants.ChangeType.LIVE_INSTANCE;
@@ -86,6 +89,7 @@ import static org.apache.helix.HelixConstants.ChangeType.MESSAGES_CONTROLLER;
import static org.apache.helix.HelixConstants.ChangeType.RESOURCE_CONFIG;
import static org.apache.helix.HelixConstants.ChangeType.TARGET_EXTERNAL_VIEW;
+
@PreFetch(enabled = false)
public class CallbackHandler implements IZkChildListener, IZkDataListener {
private static Logger logger = LoggerFactory.getLogger(CallbackHandler.class);
@@ -290,6 +294,9 @@ public class CallbackHandler implements IZkChildListener, IZkDataListener {
case TARGET_EXTERNAL_VIEW:
listenerClass = ExternalViewChangeListener.class;
break;
+ case CUSTOMIZED_VIEW:
+ listenerClass = CustomizedViewChangeListener.class;
+ break;
case CONTROLLER:
listenerClass = ControllerChangeListener.class;
}
@@ -434,6 +441,11 @@ public class CallbackHandler implements IZkChildListener, IZkDataListener {
List<ExternalView> externalViewList = preFetch(_propertyKey);
externalViewListener.onExternalViewChange(externalViewList, changeContext);
+ } else if (_changeType == CUSTOMIZED_VIEW) {
+ CustomizedViewChangeListener customizedViewListener = (CustomizedViewChangeListener) _listener;
+ List<CustomizedView> customizedViewListList = preFetch(_propertyKey);
+ customizedViewListener.onCustomizedViewChange(customizedViewListList, changeContext);
+
} else if (_changeType == CONTROLLER) {
ControllerChangeListener controllerChangelistener = (ControllerChangeListener) _listener;
controllerChangelistener.onControllerChange(changeContext);
@@ -524,6 +536,7 @@ public class CallbackHandler implements IZkChildListener, IZkDataListener {
case CURRENT_STATE:
case IDEAL_STATE:
case EXTERNAL_VIEW:
+ case CUSTOMIZED_VIEW:
case TARGET_EXTERNAL_VIEW: {
// check if bucketized
BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<>(_zkClient);
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
index 72ce07f..c06ddf1 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
@@ -57,6 +57,7 @@ import org.apache.helix.api.listeners.ClusterConfigChangeListener;
import org.apache.helix.api.listeners.ConfigChangeListener;
import org.apache.helix.api.listeners.ControllerChangeListener;
import org.apache.helix.api.listeners.CurrentStateChangeListener;
+import org.apache.helix.api.listeners.CustomizedViewChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
import org.apache.helix.api.listeners.InstanceConfigChangeListener;
@@ -581,6 +582,15 @@ public class ZKHelixManager implements HelixManager, IZkStateListener {
}
@Override
+ public void addCustomizedViewChangeListener(CustomizedViewChangeListener listener, String aggregationType)
+ throws Exception {
+ addListener(listener, new Builder(_clusterName).customizedView(aggregationType),
+ ChangeType.CUSTOMIZED_VIEW, new EventType[] {
+ EventType.NodeChildrenChanged
+ });
+ }
+
+ @Override
public void addTargetExternalViewChangeListener(ExternalViewChangeListener listener) throws Exception {
addListener(listener, new Builder(_clusterName).externalViews(), ChangeType.TARGET_EXTERNAL_VIEW,
new EventType[] { EventType.NodeChildrenChanged });
diff --git a/helix-core/src/test/java/org/apache/helix/controller/stages/DummyClusterManager.java b/helix-core/src/test/java/org/apache/helix/controller/stages/DummyClusterManager.java
index 05631c8..076f115 100644
--- a/helix-core/src/test/java/org/apache/helix/controller/stages/DummyClusterManager.java
+++ b/helix-core/src/test/java/org/apache/helix/controller/stages/DummyClusterManager.java
@@ -19,6 +19,7 @@ package org.apache.helix.controller.stages;
* under the License.
*/
+import java.util.List;
import java.util.Set;
import org.apache.helix.ClusterMessagingService;
@@ -37,6 +38,7 @@ import org.apache.helix.api.listeners.ConfigChangeListener;
import org.apache.helix.api.listeners.ControllerChangeListener;
import org.apache.helix.api.listeners.CurrentStateChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
+import org.apache.helix.api.listeners.CustomizedViewChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
import org.apache.helix.api.listeners.InstanceConfigChangeListener;
import org.apache.helix.api.listeners.LiveInstanceChangeListener;
@@ -136,6 +138,12 @@ public class DummyClusterManager implements HelixManager {
}
@Override
+ public void addCustomizedViewChangeListener(CustomizedViewChangeListener listener, String aggregationType) throws Exception {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
public void addTargetExternalViewChangeListener(ExternalViewChangeListener listener) throws Exception {
// TODO Auto-generated method stub
}
diff --git a/helix-core/src/test/java/org/apache/helix/integration/manager/TestParticipantManager.java b/helix-core/src/test/java/org/apache/helix/integration/manager/TestParticipantManager.java
index aef7ff6..6e972c5 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/manager/TestParticipantManager.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/manager/TestParticipantManager.java
@@ -151,7 +151,7 @@ public class TestParticipantManager extends ZkTestBase {
// check HelixCallback Monitor
Set<ObjectInstance> objs =
_server.queryMBeans(buildCallbackMonitorObjectName(type, clusterName, instanceName), null);
- Assert.assertEquals(objs.size(), 13);
+ Assert.assertEquals(objs.size(), 14);
// check HelixZkClient Monitors
objs =
diff --git a/helix-core/src/test/java/org/apache/helix/mock/MockManager.java b/helix-core/src/test/java/org/apache/helix/mock/MockManager.java
index dfb90ec..01e6b9e 100644
--- a/helix-core/src/test/java/org/apache/helix/mock/MockManager.java
+++ b/helix-core/src/test/java/org/apache/helix/mock/MockManager.java
@@ -19,6 +19,7 @@ package org.apache.helix.mock;
* under the License.
*/
+import java.util.List;
import java.util.Set;
import java.util.UUID;
@@ -38,6 +39,7 @@ import org.apache.helix.api.listeners.ClusterConfigChangeListener;
import org.apache.helix.api.listeners.ConfigChangeListener;
import org.apache.helix.api.listeners.ControllerChangeListener;
import org.apache.helix.api.listeners.CurrentStateChangeListener;
+import org.apache.helix.api.listeners.CustomizedViewChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
import org.apache.helix.api.listeners.InstanceConfigChangeListener;
@@ -140,6 +142,12 @@ public class MockManager implements HelixManager {
}
@Override
+ public void addCustomizedViewChangeListener(CustomizedViewChangeListener listener, String aggregationType) {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
public void addTargetExternalViewChangeListener(ExternalViewChangeListener listener) throws Exception {
// TODO Auto-generated method stub
}
diff --git a/helix-core/src/test/java/org/apache/helix/participant/MockZKHelixManager.java b/helix-core/src/test/java/org/apache/helix/participant/MockZKHelixManager.java
index 85b376f..67efbd5 100644
--- a/helix-core/src/test/java/org/apache/helix/participant/MockZKHelixManager.java
+++ b/helix-core/src/test/java/org/apache/helix/participant/MockZKHelixManager.java
@@ -19,6 +19,7 @@ package org.apache.helix.participant;
* under the License.
*/
+import java.util.List;
import java.util.Set;
import java.util.UUID;
@@ -37,6 +38,7 @@ import org.apache.helix.api.listeners.ClusterConfigChangeListener;
import org.apache.helix.api.listeners.ConfigChangeListener;
import org.apache.helix.api.listeners.ControllerChangeListener;
import org.apache.helix.api.listeners.CurrentStateChangeListener;
+import org.apache.helix.api.listeners.CustomizedViewChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
import org.apache.helix.api.listeners.InstanceConfigChangeListener;
@@ -165,6 +167,12 @@ public class MockZKHelixManager implements HelixManager {
}
@Override
+ public void addCustomizedViewChangeListener(CustomizedViewChangeListener listener, String aggregationType) throws Exception {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
public void addTargetExternalViewChangeListener(ExternalViewChangeListener listener) throws Exception {
// TODO Auto-generated method stub
}
[helix] 12/23: update cache functions for customized view
aggregation (#887)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit eeeab152355e48738d9e69fe6329053ba081d054
Author: zhangmeng916 <56...@users.noreply.github.com>
AuthorDate: Thu Mar 12 10:07:23 2020 -0700
update cache functions for customized view aggregation (#887)
Update some cache functions for customized view aggregation
---
.../main/java/org/apache/helix/ConfigAccessor.java | 2 +-
.../main/java/org/apache/helix/PropertyKey.java | 6 +++---
.../java/org/apache/helix/PropertyPathBuilder.java | 24 ++++++++++++++++++++--
.../CustomizedStateConfigChangeListener.java | 4 ++--
.../helix/common/caches/CustomizedStateCache.java | 17 +++++++--------
.../helix/common/caches/CustomizedViewCache.java | 11 ++++++++++
.../helix/common/caches/ParticipantStateCache.java | 6 +++---
.../org/apache/helix/model/HelixConfigScope.java | 5 ++++-
.../model/builder/HelixConfigScopeBuilder.java | 1 +
9 files changed, 54 insertions(+), 22 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
index 30a9a8f..34b72c3 100644
--- a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
+++ b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
@@ -598,7 +598,7 @@ public class ConfigAccessor {
throw new HelixException(String.format("Failed to get config. cluster: %s is not setup.", clusterName));
}
HelixConfigScope scope =
- new HelixConfigScopeBuilder(ConfigScopeProperty.CUSTOMIZED_STATE_AGGREGATION).forCluster(clusterName).build();
+ new HelixConfigScopeBuilder(ConfigScopeProperty.CUSTOMIZED_STATE).forCluster(clusterName).build();
ZNRecord record = getConfigZnRecord(scope);
if (record == null) {
diff --git a/helix-core/src/main/java/org/apache/helix/PropertyKey.java b/helix-core/src/main/java/org/apache/helix/PropertyKey.java
index 29e8d65..73e125e 100644
--- a/helix-core/src/main/java/org/apache/helix/PropertyKey.java
+++ b/helix-core/src/main/java/org/apache/helix/PropertyKey.java
@@ -255,9 +255,9 @@ public class PropertyKey {
* @return {@link PropertyKey}
*/
public PropertyKey customizedStateConfig() {
- return new PropertyKey(CONFIGS, ConfigScopeProperty.CUSTOMIZED_STATE_AGGREGATION,
+ return new PropertyKey(CONFIGS, ConfigScopeProperty.CUSTOMIZED_STATE,
CustomizedStateConfig.class, _clusterName,
- ConfigScopeProperty.CUSTOMIZED_STATE_AGGREGATION.name(), _clusterName);
+ ConfigScopeProperty.CUSTOMIZED_STATE.name(), _clusterName);
}
/**
@@ -645,7 +645,7 @@ public class PropertyKey {
* Get a property key associated with all {@link CustomizedView}
* @return {@link PropertyKey}
*/
- public PropertyKey customizedView() {
+ public PropertyKey customizedViews() {
return new PropertyKey(CUSTOMIZEDVIEW, CustomizedView.class, _clusterName);
}
diff --git a/helix-core/src/main/java/org/apache/helix/PropertyPathBuilder.java b/helix-core/src/main/java/org/apache/helix/PropertyPathBuilder.java
index 48db47a..d9bb2b4 100644
--- a/helix-core/src/main/java/org/apache/helix/PropertyPathBuilder.java
+++ b/helix-core/src/main/java/org/apache/helix/PropertyPathBuilder.java
@@ -27,7 +27,6 @@ import java.util.regex.Pattern;
import org.apache.helix.model.ControllerHistory;
import org.apache.helix.model.CurrentState;
-import org.apache.helix.model.CustomizedState;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.InstanceConfig;
@@ -43,7 +42,6 @@ import org.slf4j.LoggerFactory;
import static org.apache.helix.PropertyType.CONFIGS;
import static org.apache.helix.PropertyType.CURRENTSTATES;
-import static org.apache.helix.PropertyType.CUSTOMIZEDSTATES;
import static org.apache.helix.PropertyType.EXTERNALVIEW;
import static org.apache.helix.PropertyType.HISTORY;
import static org.apache.helix.PropertyType.IDEALSTATES;
@@ -99,6 +97,10 @@ public class PropertyPathBuilder {
addEntry(PropertyType.TARGETEXTERNALVIEW, 1, "/{clusterName}/TARGETEXTERNALVIEW");
addEntry(PropertyType.TARGETEXTERNALVIEW, 2,
"/{clusterName}/TARGETEXTERNALVIEW/{resourceName}");
+ addEntry(PropertyType.CUSTOMIZEDVIEW, 1, "/{clusterName}/CUSTOMIZEDVIEW");
+ addEntry(PropertyType.CUSTOMIZEDVIEW, 2, "/{clusterName}/CUSTOMIZEDVIEW/{resourceName}");
+ addEntry(PropertyType.CUSTOMIZEDVIEW, 3,
+ "/{clusterName}/CUSTOMIZEDVIEW/{resourceName}/{customizedStateName}");
addEntry(PropertyType.STATEMODELDEFS, 1, "/{clusterName}/STATEMODELDEFS");
addEntry(PropertyType.STATEMODELDEFS, 2, "/{clusterName}/STATEMODELDEFS/{stateModelName}");
addEntry(PropertyType.CONTROLLER, 1, "/{clusterName}/CONTROLLER");
@@ -276,6 +278,20 @@ public class PropertyPathBuilder {
return String.format("/%s/TARGETEXTERNALVIEW/%s", clusterName, resourceName);
}
+ public static String customizedView(String clusterName) {
+ return String.format("/%s/CUSTOMIZEDVIEW", clusterName);
+ }
+
+ public static String customizedView(String clusterName, String customizedStateName) {
+ return String.format("/%s/CUSTOMIZEDVIEW/%s", clusterName, customizedStateName);
+ }
+
+ public static String customizedView(String clusterName, String customizedStateName,
+ String resourceName) {
+ return String
+ .format("/%s/CUSTOMIZEDVIEW/%s/%s", clusterName, customizedStateName, resourceName);
+ }
+
public static String liveInstance(String clusterName) {
return String.format("/%s/LIVEINSTANCES", clusterName);
}
@@ -373,6 +389,10 @@ public class PropertyPathBuilder {
return String.format("/%s/CONFIGS/RESOURCE", clusterName);
}
+ public static String customizedStateConfig(String clusterName) {
+ return String.format("/%s/CONFIGS/CUSTOMIZED_STATE", clusterName);
+ }
+
public static String controller(String clusterName) {
return String.format("/%s/CONTROLLER", clusterName);
}
diff --git a/helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedStateConfigChangeListener.java b/helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedStateConfigChangeListener.java
index 4d31d87..2a0068b 100644
--- a/helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedStateConfigChangeListener.java
+++ b/helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedStateConfigChangeListener.java
@@ -23,11 +23,11 @@ import org.apache.helix.NotificationContext;
import org.apache.helix.model.CustomizedStateConfig;
/**
- * Interface to implement to listen for changes to customized state aggregation configurations.
+ * Interface to implement to listen for changes to customized state configurations.
*/
public interface CustomizedStateConfigChangeListener {
/**
- * Invoked when customized state aggregation config changes
+ * Invoked when customized state config changes
* @param customizedStateConfig
* @param context
*/
diff --git a/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedStateCache.java b/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedStateCache.java
index 2964b13..0875b00 100644
--- a/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedStateCache.java
+++ b/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedStateCache.java
@@ -27,7 +27,6 @@ import org.apache.helix.HelixDataAccessor;
import org.apache.helix.PropertyKey;
import org.apache.helix.common.controllers.ControlContextProvider;
import org.apache.helix.model.CustomizedState;
-import org.apache.helix.model.CustomizedStateConfig;
import org.apache.helix.model.LiveInstance;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -35,13 +34,16 @@ import org.slf4j.LoggerFactory;
public class CustomizedStateCache extends ParticipantStateCache<CustomizedState> {
private static final Logger LOG = LoggerFactory.getLogger(CurrentStateCache.class.getName());
+ private final Set<String> _aggregationEnabledTypes;
- public CustomizedStateCache(String clusterName) {
- this(createDefaultControlContextProvider(clusterName));
+ public CustomizedStateCache(String clusterName, Set<String> aggregationEnabledTypes) {
+ this(createDefaultControlContextProvider(clusterName), aggregationEnabledTypes);
}
- public CustomizedStateCache(ControlContextProvider contextProvider) {
+ public CustomizedStateCache(ControlContextProvider contextProvider,
+ Set<String> aggregationEnabledTypes) {
super(contextProvider);
+ _aggregationEnabledTypes = aggregationEnabledTypes;
}
@Override
@@ -49,13 +51,8 @@ public class CustomizedStateCache extends ParticipantStateCache<CustomizedState>
Map<String, LiveInstance> liveInstanceMap) {
Set<PropertyKey> participantStateKeys = new HashSet<>();
PropertyKey.Builder keyBuilder = accessor.keyBuilder();
- Set<String> restrictedKeys = new HashSet<>(
- accessor.getProperty(accessor.keyBuilder().customizedStateConfig()).getRecord()
- .getListFields().get(
- CustomizedStateConfig.CustomizedStateProperty.AGGREGATION_ENABLED_TYPES
- .name()));
for (String instanceName : liveInstanceMap.keySet()) {
- for (String customizedStateType : restrictedKeys) {
+ for (String customizedStateType : _aggregationEnabledTypes) {
accessor.getChildNames(keyBuilder.customizedStates(instanceName, customizedStateType))
.stream().forEach(resourceName -> participantStateKeys
.add(keyBuilder.customizedState(instanceName, customizedStateType, resourceName)));
diff --git a/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java b/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java
index 493ca69..a718872 100644
--- a/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java
+++ b/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java
@@ -139,6 +139,17 @@ public class CustomizedViewCache extends AbstractDataCache<CustomizedView> {
return Collections.unmodifiableMap(_customizedViewMap);
}
+ /**
+ * Remove dead customized views from map
+ * @param resourceNames
+ */
+
+ public synchronized void removeCustomizedView(List<String> resourceNames) {
+ for (String resourceName : resourceNames) {
+ _customizedViewCache.remove(resourceName);
+ }
+ }
+
public void clear() {
_customizedViewCache.clear();
_customizedViewMap.clear();
diff --git a/helix-core/src/main/java/org/apache/helix/common/caches/ParticipantStateCache.java b/helix-core/src/main/java/org/apache/helix/common/caches/ParticipantStateCache.java
index 61556b1..b7e69b3 100644
--- a/helix-core/src/main/java/org/apache/helix/common/caches/ParticipantStateCache.java
+++ b/helix-core/src/main/java/org/apache/helix/common/caches/ParticipantStateCache.java
@@ -19,8 +19,6 @@ package org.apache.helix.common.caches;
* under the License.
*/
-import com.google.common.collect.Maps;
-
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -28,6 +26,7 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import com.google.common.collect.Maps;
import org.apache.helix.HelixDataAccessor;
import org.apache.helix.PropertyKey;
import org.apache.helix.common.controllers.ControlContextProvider;
@@ -108,7 +107,8 @@ public abstract class ParticipantStateCache<T> extends AbstractDataCache {
Map<String, LiveInstance> liveInstanceMap) {
long start = System.currentTimeMillis();
- Set<PropertyKey> participantStateKeys = PopulateParticipantKeys(accessor, liveInstanceMap);
+ Set<PropertyKey> participantStateKeys =
+ PopulateParticipantKeys(accessor, liveInstanceMap);
// All new entries from zk not cached locally yet should be read from ZK.
Set<PropertyKey> reloadKeys = new HashSet<>(participantStateKeys);
diff --git a/helix-core/src/main/java/org/apache/helix/model/HelixConfigScope.java b/helix-core/src/main/java/org/apache/helix/model/HelixConfigScope.java
index cf49ccd..0b335ca 100644
--- a/helix-core/src/main/java/org/apache/helix/model/HelixConfigScope.java
+++ b/helix-core/src/main/java/org/apache/helix/model/HelixConfigScope.java
@@ -38,7 +38,8 @@ public class HelixConfigScope {
CONSTRAINT(2, 0),
REST(2, 0),
CLOUD(2, 0),
- CUSTOMIZED_STATE_AGGREGATION(2, 0);
+ CUSTOMIZED_STATE_AGGREGATION(2, 0),
+ CUSTOMIZED_STATE(2, 0);
final int _zkPathArgNum;
final int _mapKeyArgNum;
@@ -91,6 +92,8 @@ public class HelixConfigScope {
template.addEntry(ConfigScopeProperty.CLOUD, 1, "/{clusterName}/CONFIGS/CLOUD");
template.addEntry(ConfigScopeProperty.CUSTOMIZED_STATE_AGGREGATION, 2,
"/{clusterName}/CONFIGS/CUSTOMIZED_STATE_AGGREGATION/{clusterName}");
+ template.addEntry(ConfigScopeProperty.CUSTOMIZED_STATE, 2,
+ "/{clusterName}/CONFIGS/CUSTOMIZED_STATE/{clusterName}");
}
final ConfigScopeProperty _type;
diff --git a/helix-core/src/main/java/org/apache/helix/model/builder/HelixConfigScopeBuilder.java b/helix-core/src/main/java/org/apache/helix/model/builder/HelixConfigScopeBuilder.java
index 98de1e7..8166bf4 100644
--- a/helix-core/src/main/java/org/apache/helix/model/builder/HelixConfigScopeBuilder.java
+++ b/helix-core/src/main/java/org/apache/helix/model/builder/HelixConfigScopeBuilder.java
@@ -130,6 +130,7 @@ public class HelixConfigScopeBuilder {
scope = new HelixConfigScope(_type, Arrays.asList(_clusterName, _clusterName), null);
break;
case CUSTOMIZED_STATE_AGGREGATION:
+ case CUSTOMIZED_STATE:
scope = new HelixConfigScope(_type, Arrays.asList(_clusterName, _clusterName), null);
default:
break;
[helix] 21/23: Add registration logic for CustomizedView listeners
(#944)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 36ba93f5ad87a14a87a307c62722a726d0535e53
Author: Ali Reza Zamani Zadeh Najari <an...@linkedin.com>
AuthorDate: Mon Apr 13 10:53:44 2020 -0700
Add registration logic for CustomizedView listeners (#944)
In this commit, a new logic is added which target the scenario where user
disables and enables specific types. In this case, since the CustomizedView
path for that type is removed by the controller, the router looses its
listener. In this commit, we added root change lister and re-registers the
listens again.
---
.../main/java/org/apache/helix/HelixConstants.java | 1 +
.../main/java/org/apache/helix/HelixManager.java | 7 ++++
.../CustomizedViewRootChangeListener.java | 39 ++++++++++++++++++++++
.../apache/helix/manager/zk/CallbackHandler.java | 16 +++++++++
.../apache/helix/manager/zk/ZKHelixManager.java | 9 +++++
.../helix/spectator/RoutingTableProvider.java | 34 ++++++++++++++++++-
.../controller/stages/DummyClusterManager.java | 6 ++++
.../manager/TestParticipantManager.java | 2 +-
.../java/org/apache/helix/mock/MockManager.java | 6 ++++
.../helix/participant/MockZKHelixManager.java | 5 +++
10 files changed, 123 insertions(+), 2 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/HelixConstants.java b/helix-core/src/main/java/org/apache/helix/HelixConstants.java
index ca32c44..8691ffa 100644
--- a/helix-core/src/main/java/org/apache/helix/HelixConstants.java
+++ b/helix-core/src/main/java/org/apache/helix/HelixConstants.java
@@ -39,6 +39,7 @@ public interface HelixConstants {
MESSAGE (PropertyType.MESSAGES),
EXTERNAL_VIEW (PropertyType.EXTERNALVIEW),
CUSTOMIZED_VIEW (PropertyType.CUSTOMIZEDVIEW),
+ CUSTOMIZED_VIEW_ROOT (PropertyType.CUSTOMIZEDVIEW),
TARGET_EXTERNAL_VIEW (PropertyType.TARGETEXTERNALVIEW),
CONTROLLER (PropertyType.CONTROLLER),
MESSAGES_CONTROLLER (PropertyType.MESSAGES_CONTROLLER),
diff --git a/helix-core/src/main/java/org/apache/helix/HelixManager.java b/helix-core/src/main/java/org/apache/helix/HelixManager.java
index 9702015..c7e31a5 100644
--- a/helix-core/src/main/java/org/apache/helix/HelixManager.java
+++ b/helix-core/src/main/java/org/apache/helix/HelixManager.java
@@ -30,6 +30,7 @@ import org.apache.helix.api.listeners.CustomizedStateChangeListener;
import org.apache.helix.api.listeners.CustomizedStateConfigChangeListener;
import org.apache.helix.api.listeners.CustomizedStateRootChangeListener;
import org.apache.helix.api.listeners.CustomizedViewChangeListener;
+import org.apache.helix.api.listeners.CustomizedViewRootChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
import org.apache.helix.api.listeners.InstanceConfigChangeListener;
@@ -257,6 +258,12 @@ public interface HelixManager {
void addCustomizedViewChangeListener(CustomizedViewChangeListener listener, String customizedStateType) throws Exception;
/**
+ * @see CustomizedViewRootChangeListener#onCustomizedViewRootChange(List, NotificationContext)
+ * @param listener
+ */
+ void addCustomizedViewRootChangeListener(CustomizedViewRootChangeListener listener) throws Exception;
+
+ /**
* @see ExternalViewChangeListener#onExternalViewChange(List, NotificationContext)
* @param listener
*/
diff --git a/helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedViewRootChangeListener.java b/helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedViewRootChangeListener.java
new file mode 100644
index 0000000..a1ee62a
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedViewRootChangeListener.java
@@ -0,0 +1,39 @@
+package org.apache.helix.api.listeners;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.List;
+
+import org.apache.helix.NotificationContext;
+
+
+/**
+ * Interface to implement to respond to changes in the root path of customized View
+ */
+public interface CustomizedViewRootChangeListener {
+
+ /**
+ * Invoked when root path of CustomizedView changes
+ * @param customizedViewTypes CustomizedView types
+ * @param changeContext
+ */
+ void onCustomizedViewRootChange(List<String> customizedViewTypes,
+ NotificationContext changeContext);
+}
\ No newline at end of file
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/CallbackHandler.java b/helix-core/src/main/java/org/apache/helix/manager/zk/CallbackHandler.java
index 436873a..58460a3 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/CallbackHandler.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/CallbackHandler.java
@@ -50,6 +50,7 @@ import org.apache.helix.api.listeners.CustomizedStateChangeListener;
import org.apache.helix.api.listeners.CustomizedStateConfigChangeListener;
import org.apache.helix.api.listeners.CustomizedStateRootChangeListener;
import org.apache.helix.api.listeners.CustomizedViewChangeListener;
+import org.apache.helix.api.listeners.CustomizedViewRootChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
import org.apache.helix.api.listeners.InstanceConfigChangeListener;
@@ -89,6 +90,7 @@ import static org.apache.helix.HelixConstants.ChangeType.CUSTOMIZED_STATE;
import static org.apache.helix.HelixConstants.ChangeType.CUSTOMIZED_STATE_CONFIG;
import static org.apache.helix.HelixConstants.ChangeType.CUSTOMIZED_STATE_ROOT;
import static org.apache.helix.HelixConstants.ChangeType.CUSTOMIZED_VIEW;
+import static org.apache.helix.HelixConstants.ChangeType.CUSTOMIZED_VIEW_ROOT;
import static org.apache.helix.HelixConstants.ChangeType.EXTERNAL_VIEW;
import static org.apache.helix.HelixConstants.ChangeType.IDEAL_STATE;
import static org.apache.helix.HelixConstants.ChangeType.INSTANCE_CONFIG;
@@ -98,6 +100,7 @@ import static org.apache.helix.HelixConstants.ChangeType.MESSAGES_CONTROLLER;
import static org.apache.helix.HelixConstants.ChangeType.RESOURCE_CONFIG;
import static org.apache.helix.HelixConstants.ChangeType.TARGET_EXTERNAL_VIEW;
+
@PreFetch(enabled = false)
public class CallbackHandler implements IZkChildListener, IZkDataListener {
private static Logger logger = LoggerFactory.getLogger(CallbackHandler.class);
@@ -313,6 +316,9 @@ public class CallbackHandler implements IZkChildListener, IZkDataListener {
case CUSTOMIZED_VIEW:
listenerClass = CustomizedViewChangeListener.class;
break;
+ case CUSTOMIZED_VIEW_ROOT:
+ listenerClass = CustomizedViewRootChangeListener.class;
+ break;
case CONTROLLER:
listenerClass = ControllerChangeListener.class;
}
@@ -484,6 +490,16 @@ public class CallbackHandler implements IZkChildListener, IZkDataListener {
List<ExternalView> externalViewList = preFetch(_propertyKey);
externalViewListener.onExternalViewChange(externalViewList, changeContext);
+ } else if (_changeType == CUSTOMIZED_VIEW_ROOT) {
+ CustomizedViewRootChangeListener customizedViewRootChangeListener =
+ (CustomizedViewRootChangeListener) _listener;
+ List<String> customizedViewTypes = new ArrayList<>();
+ if (_preFetchEnabled) {
+ customizedViewTypes = _accessor.getChildNames(_accessor.keyBuilder().customizedViews());
+ }
+ customizedViewRootChangeListener.onCustomizedViewRootChange(customizedViewTypes,
+ changeContext);
+
} else if (_changeType == CUSTOMIZED_VIEW) {
CustomizedViewChangeListener customizedViewListener = (CustomizedViewChangeListener) _listener;
List<CustomizedView> customizedViewListList = preFetch(_propertyKey);
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
index ad56f95..e2429d7 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
@@ -61,6 +61,7 @@ import org.apache.helix.api.listeners.CustomizedStateChangeListener;
import org.apache.helix.api.listeners.CustomizedStateConfigChangeListener;
import org.apache.helix.api.listeners.CustomizedStateRootChangeListener;
import org.apache.helix.api.listeners.CustomizedViewChangeListener;
+import org.apache.helix.api.listeners.CustomizedViewRootChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
import org.apache.helix.api.listeners.InstanceConfigChangeListener;
@@ -618,6 +619,14 @@ public class ZKHelixManager implements HelixManager, IZkStateListener {
}
@Override
+ public void addCustomizedViewRootChangeListener(CustomizedViewRootChangeListener listener) throws Exception {
+ addListener(listener, new Builder(_clusterName).customizedViews(),
+ ChangeType.CUSTOMIZED_VIEW_ROOT, new EventType[] {
+ EventType.NodeChildrenChanged
+ });
+ }
+
+ @Override
public void addTargetExternalViewChangeListener(ExternalViewChangeListener listener) throws Exception {
addListener(listener, new Builder(_clusterName).externalViews(), ChangeType.TARGET_EXTERNAL_VIEW,
new EventType[] { EventType.NodeChildrenChanged });
diff --git a/helix-core/src/main/java/org/apache/helix/spectator/RoutingTableProvider.java b/helix-core/src/main/java/org/apache/helix/spectator/RoutingTableProvider.java
index 1e11965..cfd4350 100644
--- a/helix-core/src/main/java/org/apache/helix/spectator/RoutingTableProvider.java
+++ b/helix-core/src/main/java/org/apache/helix/spectator/RoutingTableProvider.java
@@ -46,6 +46,7 @@ import org.apache.helix.PropertyType;
import org.apache.helix.api.listeners.ConfigChangeListener;
import org.apache.helix.api.listeners.CurrentStateChangeListener;
import org.apache.helix.api.listeners.CustomizedViewChangeListener;
+import org.apache.helix.api.listeners.CustomizedViewRootChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.InstanceConfigChangeListener;
import org.apache.helix.api.listeners.LiveInstanceChangeListener;
@@ -67,7 +68,8 @@ import org.slf4j.LoggerFactory;
public class RoutingTableProvider
implements ExternalViewChangeListener, InstanceConfigChangeListener, ConfigChangeListener,
- LiveInstanceChangeListener, CurrentStateChangeListener, CustomizedViewChangeListener {
+ LiveInstanceChangeListener, CurrentStateChangeListener, CustomizedViewChangeListener,
+ CustomizedViewRootChangeListener {
private static final Logger logger = LoggerFactory.getLogger(RoutingTableProvider.class);
private static final long DEFAULT_PERIODIC_REFRESH_INTERVAL = 300000L; // 5 minutes
private final Map<String, AtomicReference<RoutingTable>> _routingTableRefMap;
@@ -230,6 +232,15 @@ public class RoutingTableProvider
}
break;
case CUSTOMIZEDVIEW:
+ // Add CustomizedView root change listener
+ try {
+ _helixManager.addCustomizedViewRootChangeListener(this);
+ } catch (Exception e) {
+ shutdown();
+ throw new HelixException(
+ "Failed to attach CustomizedView Root Listener to HelixManager!", e);
+ }
+ // Add individual listeners for each customizedStateType
List<String> customizedStateTypes = _sourceDataTypeMap.get(propertyType);
for (String customizedStateType : customizedStateTypes) {
try {
@@ -683,6 +694,27 @@ public class RoutingTableProvider
}
}
+ @Override
+ @PreFetch(enabled = false)
+ public void onCustomizedViewRootChange(List<String> customizedViewTypes,
+ NotificationContext changeContext) {
+ logger.info(
+ "Registering the CustomizedView listeners again due to the CustomizedView root change.");
+ List<String> userRequestedTypes =
+ _sourceDataTypeMap.getOrDefault(PropertyType.CUSTOMIZEDVIEW, Collections.emptyList());
+ for (String customizedStateType : userRequestedTypes) {
+ try {
+ _helixManager.addCustomizedViewChangeListener(this, customizedStateType);
+ } catch (Exception e) {
+ shutdown();
+ throw new HelixException(
+ String.format("Failed to attach CustomizedView Listener to HelixManager for type %s!",
+ customizedStateType),
+ e);
+ }
+ }
+ }
+
final AtomicReference<Map<String, LiveInstance>> _lastSeenSessions = new AtomicReference<>();
/**
diff --git a/helix-core/src/test/java/org/apache/helix/controller/stages/DummyClusterManager.java b/helix-core/src/test/java/org/apache/helix/controller/stages/DummyClusterManager.java
index 83dd49e..ba214c7 100644
--- a/helix-core/src/test/java/org/apache/helix/controller/stages/DummyClusterManager.java
+++ b/helix-core/src/test/java/org/apache/helix/controller/stages/DummyClusterManager.java
@@ -40,6 +40,7 @@ import org.apache.helix.api.listeners.CustomizedStateChangeListener;
import org.apache.helix.api.listeners.CustomizedStateConfigChangeListener;
import org.apache.helix.api.listeners.CustomizedStateRootChangeListener;
import org.apache.helix.api.listeners.CustomizedViewChangeListener;
+import org.apache.helix.api.listeners.CustomizedViewRootChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
import org.apache.helix.api.listeners.InstanceConfigChangeListener;
@@ -160,6 +161,11 @@ public class DummyClusterManager implements HelixManager {
}
@Override
+ public void addCustomizedViewRootChangeListener(CustomizedViewRootChangeListener listener) throws Exception {
+ // TODO Auto-generated method stub
+ }
+
+ @Override
public void addTargetExternalViewChangeListener(ExternalViewChangeListener listener) throws Exception {
// TODO Auto-generated method stub
}
diff --git a/helix-core/src/test/java/org/apache/helix/integration/manager/TestParticipantManager.java b/helix-core/src/test/java/org/apache/helix/integration/manager/TestParticipantManager.java
index 7922153..75d14b3 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/manager/TestParticipantManager.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/manager/TestParticipantManager.java
@@ -151,7 +151,7 @@ public class TestParticipantManager extends ZkTestBase {
// check HelixCallback Monitor
Set<ObjectInstance> objs =
_server.queryMBeans(buildCallbackMonitorObjectName(type, clusterName, instanceName), null);
- Assert.assertEquals(objs.size(), 17);
+ Assert.assertEquals(objs.size(), 18);
// check HelixZkClient Monitors
objs =
diff --git a/helix-core/src/test/java/org/apache/helix/mock/MockManager.java b/helix-core/src/test/java/org/apache/helix/mock/MockManager.java
index 51b932d..4ac1a9f 100644
--- a/helix-core/src/test/java/org/apache/helix/mock/MockManager.java
+++ b/helix-core/src/test/java/org/apache/helix/mock/MockManager.java
@@ -42,6 +42,7 @@ import org.apache.helix.api.listeners.CustomizedStateChangeListener;
import org.apache.helix.api.listeners.CustomizedStateConfigChangeListener;
import org.apache.helix.api.listeners.CustomizedStateRootChangeListener;
import org.apache.helix.api.listeners.CustomizedViewChangeListener;
+import org.apache.helix.api.listeners.CustomizedViewRootChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
import org.apache.helix.api.listeners.InstanceConfigChangeListener;
@@ -163,6 +164,11 @@ public class MockManager implements HelixManager {
}
+ public void addCustomizedViewRootChangeListener(CustomizedViewRootChangeListener listener) throws Exception {
+ // TODO Auto-generated method stub
+
+ }
+
@Override
public void addTargetExternalViewChangeListener(ExternalViewChangeListener listener) throws Exception {
// TODO Auto-generated method stub
diff --git a/helix-core/src/test/java/org/apache/helix/participant/MockZKHelixManager.java b/helix-core/src/test/java/org/apache/helix/participant/MockZKHelixManager.java
index e671482..7d28304 100644
--- a/helix-core/src/test/java/org/apache/helix/participant/MockZKHelixManager.java
+++ b/helix-core/src/test/java/org/apache/helix/participant/MockZKHelixManager.java
@@ -41,6 +41,7 @@ import org.apache.helix.api.listeners.CustomizedStateChangeListener;
import org.apache.helix.api.listeners.CustomizedStateConfigChangeListener;
import org.apache.helix.api.listeners.CustomizedStateRootChangeListener;
import org.apache.helix.api.listeners.CustomizedViewChangeListener;
+import org.apache.helix.api.listeners.CustomizedViewRootChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
import org.apache.helix.api.listeners.InstanceConfigChangeListener;
@@ -195,6 +196,10 @@ public class MockZKHelixManager implements HelixManager {
}
+ public void addCustomizedViewRootChangeListener(CustomizedViewRootChangeListener listener) throws Exception {
+ // TODO Auto-generated method stub
+ }
+
@Override
public void addTargetExternalViewChangeListener(ExternalViewChangeListener listener) throws Exception {
// TODO Auto-generated method stub
[helix] 15/23: Add new stages in Helix generic controller for
customized view aggregation. (#851)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit fc3dfdc9f7fd0ec2fbdd41846f847e948e651cdd
Author: zhangmeng916 <56...@users.noreply.github.com>
AuthorDate: Wed Mar 25 14:24:25 2020 -0700
Add new stages in Helix generic controller for customized view aggregation. (#851)
Add extra stages and pipelines in controller for customized state computation and customized view aggregation.
Add refresh logic in resource data provider for customized view related data refresh.
Add customized state event handling in CallbackHandler.
Add integration test for customized view aggregation.
Modify existing tests to verify new logic.
Co-authored-by: Meng Zhang <mn...@mnzhang-mn1.linkedin.biz>
---
.../main/java/org/apache/helix/HelixConstants.java | 1 +
.../main/java/org/apache/helix/HelixManager.java | 14 +-
.../main/java/org/apache/helix/PropertyKey.java | 9 +
.../listeners/CustomizedStateChangeListener.java | 2 +-
...java => CustomizedStateRootChangeListener.java} | 14 +-
.../helix/controller/GenericHelixController.java | 151 ++++++++++--
.../helix/controller/stages/ClusterEventType.java | 2 +
.../apache/helix/manager/zk/CallbackHandler.java | 200 +++++++++------
.../helix/manager/zk/ControllerManagerHelper.java | 2 +
.../org/apache/helix/manager/zk/ZKHelixAdmin.java | 2 +
.../apache/helix/manager/zk/ZKHelixManager.java | 20 +-
.../test/java/org/apache/helix/TestZKCallback.java | 29 ++-
.../controller/stages/DummyClusterManager.java | 15 +-
.../TestComputeAndCleanupCustomizedView.java | 271 +++++++++++++++++++++
.../integration/TestZkCallbackHandlerLeak.java | 16 +-
.../manager/TestParticipantManager.java | 2 +-
.../apache/helix/manager/zk/TestZkHelixAdmin.java | 2 +
.../java/org/apache/helix/mock/MockHelixAdmin.java | 4 +
.../java/org/apache/helix/mock/MockManager.java | 15 +-
.../helix/participant/MockZKHelixManager.java | 15 +-
20 files changed, 656 insertions(+), 130 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/HelixConstants.java b/helix-core/src/main/java/org/apache/helix/HelixConstants.java
index c71f340..ca32c44 100644
--- a/helix-core/src/main/java/org/apache/helix/HelixConstants.java
+++ b/helix-core/src/main/java/org/apache/helix/HelixConstants.java
@@ -34,6 +34,7 @@ public interface HelixConstants {
CLUSTER_CONFIG (PropertyType.CONFIGS),
LIVE_INSTANCE (PropertyType.LIVEINSTANCES),
CURRENT_STATE (PropertyType.CURRENTSTATES),
+ CUSTOMIZED_STATE_ROOT (PropertyType.CUSTOMIZEDSTATES),
CUSTOMIZED_STATE (PropertyType.CUSTOMIZEDSTATES),
MESSAGE (PropertyType.MESSAGES),
EXTERNAL_VIEW (PropertyType.EXTERNALVIEW),
diff --git a/helix-core/src/main/java/org/apache/helix/HelixManager.java b/helix-core/src/main/java/org/apache/helix/HelixManager.java
index 90aaa0c..9702015 100644
--- a/helix-core/src/main/java/org/apache/helix/HelixManager.java
+++ b/helix-core/src/main/java/org/apache/helix/HelixManager.java
@@ -26,10 +26,11 @@ import org.apache.helix.api.listeners.ClusterConfigChangeListener;
import org.apache.helix.api.listeners.ConfigChangeListener;
import org.apache.helix.api.listeners.ControllerChangeListener;
import org.apache.helix.api.listeners.CurrentStateChangeListener;
-import org.apache.helix.api.listeners.CustomizedStateConfigChangeListener;
import org.apache.helix.api.listeners.CustomizedStateChangeListener;
-import org.apache.helix.api.listeners.ExternalViewChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateConfigChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateRootChangeListener;
import org.apache.helix.api.listeners.CustomizedViewChangeListener;
+import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
import org.apache.helix.api.listeners.InstanceConfigChangeListener;
import org.apache.helix.api.listeners.LiveInstanceChangeListener;
@@ -227,6 +228,15 @@ public interface HelixManager {
String sessionId) throws Exception;
/**
+
+ * @see CustomizedStateRootChangeListener#onCustomizedStateRootChange(String, NotificationContext)
+ * @param listener
+ * @param instanceName
+ */
+ void addCustomizedStateRootChangeListener(CustomizedStateRootChangeListener listener,
+ String instanceName) throws Exception;
+
+ /**
* @see CustomizedStateChangeListener#onCustomizedStateChange(String, List, NotificationContext)
* @param listener
* @param instanceName
diff --git a/helix-core/src/main/java/org/apache/helix/PropertyKey.java b/helix-core/src/main/java/org/apache/helix/PropertyKey.java
index 41bc9cd..64d57f8 100644
--- a/helix-core/src/main/java/org/apache/helix/PropertyKey.java
+++ b/helix-core/src/main/java/org/apache/helix/PropertyKey.java
@@ -486,6 +486,15 @@ public class PropertyKey {
}
/**
+ * Get a property key associated with the root of {@link CustomizedState} of an instance
+ * @param instanceName
+ * @return {@link PropertyKey}
+ */
+ public PropertyKey customizedStatesRoot(String instanceName) {
+ return new PropertyKey(CUSTOMIZEDSTATES, CustomizedState.class, _clusterName, instanceName);
+ }
+
+ /**
* Get a property key associated with {@link CustomizedState} of an instance and customized state
* @param instanceName
* @param customizedStateName
diff --git a/helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedStateChangeListener.java b/helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedStateChangeListener.java
index 0753f67..2a59065 100644
--- a/helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedStateChangeListener.java
+++ b/helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedStateChangeListener.java
@@ -24,7 +24,7 @@ import org.apache.helix.NotificationContext;
import org.apache.helix.model.CustomizedState;
/**
- * Interface to implement to respond to changes in the current state
+ * Interface to implement to respond to changes in the customized state
*/
public interface CustomizedStateChangeListener {
diff --git a/helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedStateChangeListener.java b/helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedStateRootChangeListener.java
similarity index 72%
copy from helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedStateChangeListener.java
copy to helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedStateRootChangeListener.java
index 0753f67..593a975 100644
--- a/helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedStateChangeListener.java
+++ b/helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedStateRootChangeListener.java
@@ -20,20 +20,20 @@ package org.apache.helix.api.listeners;
*/
import java.util.List;
+
import org.apache.helix.NotificationContext;
-import org.apache.helix.model.CustomizedState;
/**
- * Interface to implement to respond to changes in the current state
+ * Interface to implement to respond to changes in the root path of customized state
*/
-public interface CustomizedStateChangeListener {
+public interface CustomizedStateRootChangeListener {
/**
- * Invoked when customized state changes
+ * Invoked when root path customized state changes
* @param instanceName name of the instance whose state changed
- * @param customizedStatesInfo a list of the customized states
+ * @param customizedStateTypes the state types under the root
* @param changeContext the change event and state
*/
- void onCustomizedStateChange(String instanceName, List<CustomizedState> customizedStatesInfo,
- NotificationContext changeContext);
+ void onCustomizedStateRootChange(String instanceName,
+ List<String> customizedStateTypes, NotificationContext changeContext);
}
\ No newline at end of file
diff --git a/helix-core/src/main/java/org/apache/helix/controller/GenericHelixController.java b/helix-core/src/main/java/org/apache/helix/controller/GenericHelixController.java
index 8f7da2d..8644441 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/GenericHelixController.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/GenericHelixController.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -47,6 +48,9 @@ import org.apache.helix.api.exceptions.HelixMetaDataAccessException;
import org.apache.helix.api.listeners.ClusterConfigChangeListener;
import org.apache.helix.api.listeners.ControllerChangeListener;
import org.apache.helix.api.listeners.CurrentStateChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateConfigChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateRootChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
import org.apache.helix.api.listeners.InstanceConfigChangeListener;
import org.apache.helix.api.listeners.LiveInstanceChangeListener;
@@ -69,6 +73,8 @@ import org.apache.helix.controller.stages.ClusterEvent;
import org.apache.helix.controller.stages.ClusterEventType;
import org.apache.helix.controller.stages.CompatibilityCheckStage;
import org.apache.helix.controller.stages.CurrentStateComputationStage;
+import org.apache.helix.controller.stages.CustomizedStateComputationStage;
+import org.apache.helix.controller.stages.CustomizedViewAggregationStage;
import org.apache.helix.controller.stages.ExternalViewComputeStage;
import org.apache.helix.controller.stages.IntermediateStateCalcStage;
import org.apache.helix.controller.stages.MaintenanceRecoveryStage;
@@ -89,6 +95,8 @@ import org.apache.helix.controller.stages.task.TaskPersistDataStage;
import org.apache.helix.controller.stages.task.TaskSchedulingStage;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.CurrentState;
+import org.apache.helix.model.CustomizedState;
+import org.apache.helix.model.CustomizedStateConfig;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.InstanceConfig;
import org.apache.helix.model.LiveInstance;
@@ -117,10 +125,12 @@ import static org.apache.helix.HelixConstants.ChangeType;
* 4. select the messages that can be sent, needs messages and state model constraints <br>
* 5. send messages
*/
-public class GenericHelixController implements IdealStateChangeListener,
- LiveInstanceChangeListener, MessageListener, CurrentStateChangeListener,
- ControllerChangeListener, InstanceConfigChangeListener, ResourceConfigChangeListener,
- ClusterConfigChangeListener {
+public class GenericHelixController implements IdealStateChangeListener, LiveInstanceChangeListener,
+ MessageListener, CurrentStateChangeListener,
+ CustomizedStateRootChangeListener,
+ CustomizedStateChangeListener,
+ CustomizedStateConfigChangeListener, ControllerChangeListener,
+ InstanceConfigChangeListener, ResourceConfigChangeListener, ClusterConfigChangeListener {
private static final Logger logger =
LoggerFactory.getLogger(GenericHelixController.class.getName());
@@ -132,6 +142,8 @@ public class GenericHelixController implements IdealStateChangeListener,
final AtomicReference<Map<String, LiveInstance>> _lastSeenInstances;
final AtomicReference<Map<String, LiveInstance>> _lastSeenSessions;
+ final AtomicReference<Set<String>> _lastSeenCustomizedStateTypes;
+
// By default not reporting status until controller status is changed to activate
// TODO This flag should be inside ClusterStatusMonitor. When false, no MBean registering.
private boolean _isMonitoring = false;
@@ -440,6 +452,7 @@ public class GenericHelixController implements IdealStateChangeListener,
dataPreprocess.addStage(new ResourceComputationStage());
dataPreprocess.addStage(new ResourceValidationStage());
dataPreprocess.addStage(new CurrentStateComputationStage());
+ dataPreprocess.addStage(new CustomizedStateComputationStage());
dataPreprocess.addStage(new TopStateHandoffReportStage());
// rebalance pipeline
@@ -460,6 +473,10 @@ public class GenericHelixController implements IdealStateChangeListener,
Pipeline externalViewPipeline = new Pipeline(pipelineName);
externalViewPipeline.addStage(new ExternalViewComputeStage());
+ // customized state view generation
+ Pipeline customizedViewPipeline = new Pipeline(pipelineName);
+ customizedViewPipeline.addStage(new CustomizedViewAggregationStage());
+
// backward compatibility check
Pipeline liveInstancePipeline = new Pipeline(pipelineName);
liveInstancePipeline.addStage(new CompatibilityCheckStage());
@@ -468,16 +485,38 @@ public class GenericHelixController implements IdealStateChangeListener,
Pipeline autoExitMaintenancePipeline = new Pipeline(pipelineName);
autoExitMaintenancePipeline.addStage(new MaintenanceRecoveryStage());
- registry.register(ClusterEventType.IdealStateChange, dataRefresh, dataPreprocess, rebalancePipeline);
- registry.register(ClusterEventType.CurrentStateChange, dataRefresh, dataPreprocess, externalViewPipeline, rebalancePipeline);
- registry.register(ClusterEventType.InstanceConfigChange, dataRefresh, dataPreprocess, rebalancePipeline);
- registry.register(ClusterEventType.ResourceConfigChange, dataRefresh, dataPreprocess, rebalancePipeline);
- registry.register(ClusterEventType.ClusterConfigChange, dataRefresh, autoExitMaintenancePipeline, dataPreprocess, rebalancePipeline);
- registry.register(ClusterEventType.LiveInstanceChange, dataRefresh, autoExitMaintenancePipeline, liveInstancePipeline, dataPreprocess, externalViewPipeline, rebalancePipeline);
- registry.register(ClusterEventType.MessageChange, dataRefresh, dataPreprocess, rebalancePipeline);
- registry.register(ClusterEventType.Resume, dataRefresh, dataPreprocess, externalViewPipeline, rebalancePipeline);
- registry.register(ClusterEventType.PeriodicalRebalance, dataRefresh, autoExitMaintenancePipeline, dataPreprocess, externalViewPipeline, rebalancePipeline);
- registry.register(ClusterEventType.OnDemandRebalance, dataRefresh, autoExitMaintenancePipeline, dataPreprocess, externalViewPipeline, rebalancePipeline);
+ registry.register(ClusterEventType.IdealStateChange, dataRefresh, dataPreprocess,
+ rebalancePipeline);
+ registry.register(ClusterEventType.CurrentStateChange, dataRefresh, dataPreprocess,
+ externalViewPipeline, rebalancePipeline);
+ registry.register(ClusterEventType.InstanceConfigChange, dataRefresh, dataPreprocess,
+ rebalancePipeline);
+ registry.register(ClusterEventType.ResourceConfigChange, dataRefresh, dataPreprocess,
+ rebalancePipeline);
+ registry
+ .register(ClusterEventType.ClusterConfigChange, dataRefresh, autoExitMaintenancePipeline,
+ dataPreprocess, rebalancePipeline);
+ registry
+ .register(ClusterEventType.LiveInstanceChange, dataRefresh, autoExitMaintenancePipeline,
+ liveInstancePipeline, dataPreprocess, externalViewPipeline, customizedViewPipeline,
+ rebalancePipeline);
+ registry
+ .register(ClusterEventType.MessageChange, dataRefresh, dataPreprocess, rebalancePipeline);
+ registry.register(ClusterEventType.Resume, dataRefresh, dataPreprocess, externalViewPipeline,
+ rebalancePipeline);
+ registry
+ .register(ClusterEventType.PeriodicalRebalance, dataRefresh, autoExitMaintenancePipeline,
+ dataPreprocess, externalViewPipeline, rebalancePipeline);
+ registry
+ .register(ClusterEventType.OnDemandRebalance, dataRefresh, autoExitMaintenancePipeline,
+ dataPreprocess, externalViewPipeline, rebalancePipeline);
+ // TODO: We now include rebalance pipeline in customized state change for correctness.
+ // However, it is not efficient, and we should improve this by splitting the pipeline or
+ // controller roles to multiple hosts.
+ registry.register(ClusterEventType.CustomizedStateChange, dataRefresh, dataPreprocess,
+ customizedViewPipeline, rebalancePipeline);
+ registry.register(ClusterEventType.CustomizeStateConfigChange, dataRefresh, dataPreprocess,
+ customizedViewPipeline, rebalancePipeline);
return registry;
}
}
@@ -546,6 +585,7 @@ public class GenericHelixController implements IdealStateChangeListener,
_taskRegistry = taskRegistry;
_lastSeenInstances = new AtomicReference<>();
_lastSeenSessions = new AtomicReference<>();
+ _lastSeenCustomizedStateTypes = new AtomicReference<>();
_clusterName = clusterName;
_lastPipelineEndTimestamp = TopStateHandoffReportStage.TIMESTAMP_NOT_RECORDED;
_clusterStatusMonitor = new ClusterStatusMonitor(_clusterName);
@@ -830,6 +870,57 @@ public class GenericHelixController implements IdealStateChangeListener,
@Override
@PreFetch(enabled = false)
+ public void onCustomizedStateRootChange(String instanceName, List<String> customizedStateTypes,
+ NotificationContext changeContext) {
+ logger.info("START: GenericClusterController.onCustomizedStateRootChange()");
+ HelixManager manager = changeContext.getManager();
+ Builder keyBuilder = new Builder(manager.getClusterName());
+ if (customizedStateTypes.isEmpty()) {
+ customizedStateTypes = manager.getHelixDataAccessor()
+ .getChildNames(keyBuilder.customizedStatesRoot(instanceName));
+ }
+
+ // TODO: remove the synchronization here once we move this update into dataCache.
+ synchronized (_lastSeenCustomizedStateTypes) {
+ Set<String> lastSeenCustomizedStateTypes = _lastSeenCustomizedStateTypes.get();
+ for (String customizedState : customizedStateTypes) {
+ try {
+ if (lastSeenCustomizedStateTypes == null || !lastSeenCustomizedStateTypes
+ .contains(customizedState)) {
+ manager.addCustomizedStateChangeListener(this, instanceName, customizedState);
+ logger.info(
+ manager.getInstanceName() + " added customized state listener for " + instanceName
+ + ", listener: " + this);
+ }
+ } catch (Exception e) {
+ logger.error("Fail to add customized state listener for instance: " + instanceName, e);
+ }
+ }
+
+ for (String previousCustomizedState : lastSeenCustomizedStateTypes) {
+ if (!customizedStateTypes.contains(previousCustomizedState)) {
+ manager.removeListener(keyBuilder.customizedStates(instanceName, previousCustomizedState),
+ this);
+ }
+ }
+
+ _lastSeenCustomizedStateTypes.set(new HashSet<>(customizedStateTypes));
+ }
+ }
+
+ @Override
+ @PreFetch(enabled = false)
+ public void onCustomizedStateChange(String instanceName, List<CustomizedState> statesInfo,
+ NotificationContext changeContext) {
+ logger.info("START: GenericClusterController.onCustomizedStateChange()");
+ notifyCaches(changeContext, ChangeType.CUSTOMIZED_STATE);
+ pushToEventQueues(ClusterEventType.CustomizedStateChange, changeContext, Collections
+ .<String, Object>singletonMap(AttributeName.instanceName.name(), instanceName));
+ logger.info("END: GenericClusterController.onCustomizedStateChange()");
+ }
+
+ @Override
+ @PreFetch(enabled = false)
public void onMessage(String instanceName, List<Message> messages,
NotificationContext changeContext) {
logger.info("START: GenericClusterController.onMessage() for cluster " + _clusterName);
@@ -954,6 +1045,22 @@ public class GenericHelixController implements IdealStateChangeListener,
@Override
@PreFetch(enabled = false)
+ public void onCustomizedStateConfigChange(
+ CustomizedStateConfig customizedStateConfig,
+ NotificationContext context) {
+ logger.info(
+ "START: GenericClusterController.onCustomizedStateConfigChange() for cluster "
+ + _clusterName);
+ notifyCaches(context, ChangeType.CUSTOMIZED_STATE_CONFIG);
+ pushToEventQueues(ClusterEventType.CustomizeStateConfigChange, context,
+ Collections.<String, Object> emptyMap());
+ logger.info(
+ "END: GenericClusterController.onCustomizedStateConfigChange() for cluster "
+ + _clusterName);
+ }
+
+ @Override
+ @PreFetch(enabled = false)
public void onClusterConfigChange(ClusterConfig clusterConfig,
NotificationContext context) {
logger.info(
@@ -1099,6 +1206,8 @@ public class GenericHelixController implements IdealStateChangeListener,
if (!curInstances.containsKey(instance)) {
// remove message listener for disconnected instances
manager.removeListener(keyBuilder.messages(instance), this);
+ // remove customized state root listener for disconnected instances
+ manager.removeListener(keyBuilder.customizedStatesRoot(instance), this);
}
}
}
@@ -1131,6 +1240,20 @@ public class GenericHelixController implements IdealStateChangeListener,
}
}
+ for (String instance : curInstances.keySet()) {
+ if (lastInstances == null || !lastInstances.containsKey(instance)) {
+ try {
+ manager.addCustomizedStateRootChangeListener(this, instance);
+ logger.info(manager.getInstanceName() + " added customized root change listener for"
+ + " " + instance
+ + ", listener: " + this);
+ } catch (Exception e) {
+ logger.error("Fail to add customized root change listener for instance: " + instance,
+ e);
+ }
+ }
+ }
+
// update last-seen
_lastSeenInstances.set(curInstances);
_lastSeenSessions.set(curSessions);
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterEventType.java b/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterEventType.java
index 2fd015e..b54f03b 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterEventType.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterEventType.java
@@ -22,10 +22,12 @@ package org.apache.helix.controller.stages;
public enum ClusterEventType {
IdealStateChange,
CurrentStateChange,
+ CustomizedStateChange,
ConfigChange,
ClusterConfigChange,
ResourceConfigChange,
InstanceConfigChange,
+ CustomizeStateConfigChange,
LiveInstanceChange,
MessageChange,
ExternalViewChange,
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/CallbackHandler.java b/helix-core/src/main/java/org/apache/helix/manager/zk/CallbackHandler.java
index dc91028..436873a 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/CallbackHandler.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/CallbackHandler.java
@@ -20,6 +20,7 @@ package org.apache.helix.manager.zk;
*/
import java.lang.reflect.Method;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
@@ -45,6 +46,9 @@ import org.apache.helix.api.listeners.ClusterConfigChangeListener;
import org.apache.helix.api.listeners.ConfigChangeListener;
import org.apache.helix.api.listeners.ControllerChangeListener;
import org.apache.helix.api.listeners.CurrentStateChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateConfigChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateRootChangeListener;
import org.apache.helix.api.listeners.CustomizedViewChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
@@ -57,6 +61,8 @@ import org.apache.helix.api.listeners.ScopedConfigChangeListener;
import org.apache.helix.common.DedupEventProcessor;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.CurrentState;
+import org.apache.helix.model.CustomizedState;
+import org.apache.helix.model.CustomizedStateConfig;
import org.apache.helix.model.CustomizedView;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.IdealState;
@@ -79,8 +85,11 @@ import static org.apache.helix.HelixConstants.ChangeType.CLUSTER_CONFIG;
import static org.apache.helix.HelixConstants.ChangeType.CONFIG;
import static org.apache.helix.HelixConstants.ChangeType.CONTROLLER;
import static org.apache.helix.HelixConstants.ChangeType.CURRENT_STATE;
-import static org.apache.helix.HelixConstants.ChangeType.EXTERNAL_VIEW;
+import static org.apache.helix.HelixConstants.ChangeType.CUSTOMIZED_STATE;
+import static org.apache.helix.HelixConstants.ChangeType.CUSTOMIZED_STATE_CONFIG;
+import static org.apache.helix.HelixConstants.ChangeType.CUSTOMIZED_STATE_ROOT;
import static org.apache.helix.HelixConstants.ChangeType.CUSTOMIZED_VIEW;
+import static org.apache.helix.HelixConstants.ChangeType.EXTERNAL_VIEW;
import static org.apache.helix.HelixConstants.ChangeType.IDEAL_STATE;
import static org.apache.helix.HelixConstants.ChangeType.INSTANCE_CONFIG;
import static org.apache.helix.HelixConstants.ChangeType.LIVE_INSTANCE;
@@ -89,7 +98,6 @@ import static org.apache.helix.HelixConstants.ChangeType.MESSAGES_CONTROLLER;
import static org.apache.helix.HelixConstants.ChangeType.RESOURCE_CONFIG;
import static org.apache.helix.HelixConstants.ChangeType.TARGET_EXTERNAL_VIEW;
-
@PreFetch(enabled = false)
public class CallbackHandler implements IZkChildListener, IZkDataListener {
private static Logger logger = LoggerFactory.getLogger(CallbackHandler.class);
@@ -123,7 +131,7 @@ public class CallbackHandler implements IZkChildListener, IZkDataListener {
// TODO: make this be per _manager or per _listener instaed of per callbackHandler -- Lei
private CallbackProcessor _batchCallbackProcessor;
private boolean _watchChild = true; // Whether we should subscribe to the child znode's data
- // change.
+ // change.
// indicated whether this CallbackHandler is ready to serve event callback from ZkClient.
private boolean _ready = false;
@@ -260,45 +268,53 @@ public class CallbackHandler implements IZkChildListener, IZkDataListener {
Class listenerClass = null;
switch (_changeType) {
- case IDEAL_STATE:
- listenerClass = IdealStateChangeListener.class;
- break;
- case INSTANCE_CONFIG:
- if (_listener instanceof ConfigChangeListener) {
+ case IDEAL_STATE:
+ listenerClass = IdealStateChangeListener.class;
+ break;
+ case INSTANCE_CONFIG:
+ if (_listener instanceof ConfigChangeListener) {
+ listenerClass = ConfigChangeListener.class;
+ } else if (_listener instanceof InstanceConfigChangeListener) {
+ listenerClass = InstanceConfigChangeListener.class;
+ }
+ break;
+ case CLUSTER_CONFIG:
+ listenerClass = ClusterConfigChangeListener.class;
+ break;
+ case RESOURCE_CONFIG:
+ listenerClass = ResourceConfigChangeListener.class;
+ break;
+ case CUSTOMIZED_STATE_CONFIG:
+ listenerClass = CustomizedStateConfigChangeListener.class;
+ break;
+ case CONFIG:
listenerClass = ConfigChangeListener.class;
- } else if (_listener instanceof InstanceConfigChangeListener) {
- listenerClass = InstanceConfigChangeListener.class;
- }
- break;
- case CLUSTER_CONFIG:
- listenerClass = ClusterConfigChangeListener.class;
- break;
- case RESOURCE_CONFIG:
- listenerClass = ResourceConfigChangeListener.class;
- break;
- case CONFIG:
- listenerClass = ConfigChangeListener.class;
- break;
- case LIVE_INSTANCE:
- listenerClass = LiveInstanceChangeListener.class;
- break;
- case CURRENT_STATE:
- listenerClass = CurrentStateChangeListener.class;
- ;
- break;
- case MESSAGE:
- case MESSAGES_CONTROLLER:
- listenerClass = MessageListener.class;
- break;
- case EXTERNAL_VIEW:
- case TARGET_EXTERNAL_VIEW:
- listenerClass = ExternalViewChangeListener.class;
- break;
- case CUSTOMIZED_VIEW:
- listenerClass = CustomizedViewChangeListener.class;
- break;
- case CONTROLLER:
- listenerClass = ControllerChangeListener.class;
+ break;
+ case LIVE_INSTANCE:
+ listenerClass = LiveInstanceChangeListener.class;
+ break;
+ case CURRENT_STATE:
+ listenerClass = CurrentStateChangeListener.class;
+ break;
+ case CUSTOMIZED_STATE_ROOT:
+ listenerClass = CustomizedStateRootChangeListener.class;
+ break;
+ case CUSTOMIZED_STATE:
+ listenerClass = CustomizedStateChangeListener.class;
+ break;
+ case MESSAGE:
+ case MESSAGES_CONTROLLER:
+ listenerClass = MessageListener.class;
+ break;
+ case EXTERNAL_VIEW:
+ case TARGET_EXTERNAL_VIEW:
+ listenerClass = ExternalViewChangeListener.class;
+ break;
+ case CUSTOMIZED_VIEW:
+ listenerClass = CustomizedViewChangeListener.class;
+ break;
+ case CONTROLLER:
+ listenerClass = ControllerChangeListener.class;
}
Method callbackMethod = listenerClass.getMethods()[0];
@@ -399,6 +415,14 @@ public class CallbackHandler implements IZkChildListener, IZkDataListener {
List<ResourceConfig> configs = preFetch(_propertyKey);
listener.onResourceConfigChange(configs, changeContext);
+ } else if (_changeType == CUSTOMIZED_STATE_CONFIG) {
+ CustomizedStateConfigChangeListener listener = (CustomizedStateConfigChangeListener) _listener;
+ CustomizedStateConfig config = null;
+ if (_preFetchEnabled) {
+ config = _accessor.getProperty(_propertyKey);
+ }
+ listener.onCustomizedStateConfigChange(config, changeContext);
+
} else if (_changeType == CLUSTER_CONFIG) {
ClusterConfigChangeListener listener = (ClusterConfigChangeListener) _listener;
ClusterConfig config = null;
@@ -425,6 +449,25 @@ public class CallbackHandler implements IZkChildListener, IZkDataListener {
List<CurrentState> currentStates = preFetch(_propertyKey);
currentStateChangeListener.onStateChange(instanceName, currentStates, changeContext);
+ } else if (_changeType == CUSTOMIZED_STATE_ROOT) {
+ CustomizedStateRootChangeListener customizedStateRootChangeListener =
+ (CustomizedStateRootChangeListener) _listener;
+ String instanceName = PropertyPathConfig.getInstanceNameFromPath(_path);
+ List<String> customizedStateTypes = new ArrayList<>();
+ if (_preFetchEnabled) {
+ customizedStateTypes =
+ _accessor.getChildNames(_accessor.keyBuilder().customizedStatesRoot(instanceName));
+ }
+ customizedStateRootChangeListener
+ .onCustomizedStateRootChange(instanceName, customizedStateTypes, changeContext);
+
+ } else if (_changeType == CUSTOMIZED_STATE) {
+ CustomizedStateChangeListener customizedStateChangeListener =
+ (CustomizedStateChangeListener) _listener;
+ String instanceName = PropertyPathConfig.getInstanceNameFromPath(_path);
+ List<CustomizedState> customizedStates = preFetch(_propertyKey);
+ customizedStateChangeListener.onCustomizedStateChange(instanceName, customizedStates, changeContext);
+
} else if (_changeType == MESSAGE) {
MessageListener messageListener = (MessageListener) _listener;
String instanceName = PropertyPathConfig.getInstanceNameFromPath(_path);
@@ -533,49 +576,50 @@ public class CallbackHandler implements IZkChildListener, IZkDataListener {
try {
switch (_changeType) {
- case CURRENT_STATE:
- case IDEAL_STATE:
- case EXTERNAL_VIEW:
- case CUSTOMIZED_VIEW:
- case TARGET_EXTERNAL_VIEW: {
- // check if bucketized
- BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<>(_zkClient);
- List<ZNRecord> records = baseAccessor.getChildren(path, null, 0);
- for (ZNRecord record : records) {
- HelixProperty property = new HelixProperty(record);
- String childPath = path + "/" + record.getId();
-
- int bucketSize = property.getBucketSize();
- if (bucketSize > 0) {
- // subscribe both data-change and child-change on bucketized parent node
- // data-change gives a delete-callback which is used to remove watch
- subscribeChildChange(childPath, callbackType);
- subscribeDataChange(childPath, callbackType);
-
- // subscribe data-change on bucketized child
- List<String> bucketizedChildNames = _zkClient.getChildren(childPath);
- if (bucketizedChildNames != null) {
- for (String bucketizedChildName : bucketizedChildNames) {
- String bucketizedChildPath = childPath + "/" + bucketizedChildName;
- subscribeDataChange(bucketizedChildPath, callbackType);
+ case CURRENT_STATE:
+ case CUSTOMIZED_STATE:
+ case IDEAL_STATE:
+ case EXTERNAL_VIEW:
+ case CUSTOMIZED_VIEW:
+ case TARGET_EXTERNAL_VIEW: {
+ // check if bucketized
+ BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<>(_zkClient);
+ List<ZNRecord> records = baseAccessor.getChildren(path, null, 0);
+ for (ZNRecord record : records) {
+ HelixProperty property = new HelixProperty(record);
+ String childPath = path + "/" + record.getId();
+
+ int bucketSize = property.getBucketSize();
+ if (bucketSize > 0) {
+ // subscribe both data-change and child-change on bucketized parent node
+ // data-change gives a delete-callback which is used to remove watch
+ subscribeChildChange(childPath, callbackType);
+ subscribeDataChange(childPath, callbackType);
+
+ // subscribe data-change on bucketized child
+ List<String> bucketizedChildNames = _zkClient.getChildren(childPath);
+ if (bucketizedChildNames != null) {
+ for (String bucketizedChildName : bucketizedChildNames) {
+ String bucketizedChildPath = childPath + "/" + bucketizedChildName;
+ subscribeDataChange(bucketizedChildPath, callbackType);
+ }
}
+ } else {
+ subscribeDataChange(childPath, callbackType);
}
- } else {
- subscribeDataChange(childPath, callbackType);
}
+ break;
}
- break;
- }
- default: {
- List<String> childNames = _zkClient.getChildren(path);
- if (childNames != null) {
- for (String childName : childNames) {
- String childPath = path + "/" + childName;
- subscribeDataChange(childPath, callbackType);
+ default: {
+ List<String> childNames = _zkClient.getChildren(path);
+ if (childNames != null) {
+ for (String childName : childNames) {
+ String childPath = path + "/" + childName;
+ subscribeDataChange(childPath, callbackType);
+ }
}
+ break;
}
- break;
- }
}
} catch (ZkNoNodeException e) {
logger.warn(
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ControllerManagerHelper.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ControllerManagerHelper.java
index 3b25f0f..ca7210c 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ControllerManagerHelper.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ControllerManagerHelper.java
@@ -82,6 +82,7 @@ public class ControllerManagerHelper {
_manager.addInstanceConfigChangeListener(controller);
_manager.addResourceConfigChangeListener(controller);
_manager.addClusterfigChangeListener(controller);
+ _manager.addCustomizedStateConfigChangeListener(controller);
_manager.addLiveInstanceChangeListener(controller);
_manager.addIdealStateChangeListener(controller);
} catch (ZkInterruptedException e) {
@@ -99,6 +100,7 @@ public class ControllerManagerHelper {
*/
_manager.removeListener(keyBuilder.idealStates(), controller);
_manager.removeListener(keyBuilder.liveInstances(), controller);
+ _manager.removeListener(keyBuilder.customizedStateConfig(), controller);
_manager.removeListener(keyBuilder.clusterConfig(), controller);
_manager.removeListener(keyBuilder.resourceConfigs(), controller);
_manager.removeListener(keyBuilder.instanceConfigs(), controller);
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
index af037f0..f100fec 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
@@ -798,6 +798,8 @@ public class ZKHelixAdmin implements HelixAdmin {
_zkClient.createPersistent(path);
path = PropertyPathBuilder.resourceConfig(clusterName);
_zkClient.createPersistent(path);
+ path = PropertyPathBuilder.customizedStateConfig(clusterName);
+ _zkClient.createPersistent(path);
// PROPERTY STORE
path = PropertyPathBuilder.propertyStore(clusterName);
_zkClient.createPersistent(path);
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
index 72928ef..ad56f95 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
@@ -57,9 +57,10 @@ import org.apache.helix.api.listeners.ClusterConfigChangeListener;
import org.apache.helix.api.listeners.ConfigChangeListener;
import org.apache.helix.api.listeners.ControllerChangeListener;
import org.apache.helix.api.listeners.CurrentStateChangeListener;
-import org.apache.helix.api.listeners.CustomizedViewChangeListener;
-import org.apache.helix.api.listeners.CustomizedStateConfigChangeListener;
import org.apache.helix.api.listeners.CustomizedStateChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateConfigChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateRootChangeListener;
+import org.apache.helix.api.listeners.CustomizedViewChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
import org.apache.helix.api.listeners.InstanceConfigChangeListener;
@@ -587,11 +588,18 @@ public class ZKHelixManager implements HelixManager, IZkStateListener {
}
@Override
+ public void addCustomizedStateRootChangeListener(CustomizedStateRootChangeListener listener,
+ String instanceName) throws Exception {
+ addListener(listener, new Builder(_clusterName).customizedStatesRoot(instanceName),
+ ChangeType.CUSTOMIZED_STATE_ROOT, new EventType[]{EventType.NodeChildrenChanged});
+ }
+
+ @Override
public void addCustomizedStateChangeListener(CustomizedStateChangeListener listener,
- String instanceName, String customizedStateName) throws Exception {
- addListener(listener, new Builder(_clusterName).customizedStates(instanceName, customizedStateName),
- ChangeType.CUSTOMIZED_STATE, new EventType[] { EventType.NodeChildrenChanged
- });
+ String instanceName, String customizedStateType) throws Exception {
+ addListener(listener,
+ new Builder(_clusterName).customizedStates(instanceName, customizedStateType),
+ ChangeType.CUSTOMIZED_STATE, new EventType[]{EventType.NodeChildrenChanged});
}
@Override
diff --git a/helix-core/src/test/java/org/apache/helix/TestZKCallback.java b/helix-core/src/test/java/org/apache/helix/TestZKCallback.java
index d9b6d8d..57c2579 100644
--- a/helix-core/src/test/java/org/apache/helix/TestZKCallback.java
+++ b/helix-core/src/test/java/org/apache/helix/TestZKCallback.java
@@ -21,16 +21,20 @@ package org.apache.helix;
import java.util.Arrays;
import java.util.List;
+import java.util.Set;
import java.util.UUID;
import org.apache.helix.PropertyKey.Builder;
import org.apache.helix.api.listeners.ConfigChangeListener;
import org.apache.helix.api.listeners.CurrentStateChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateConfigChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateRootChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
import org.apache.helix.api.listeners.LiveInstanceChangeListener;
import org.apache.helix.api.listeners.MessageListener;
import org.apache.helix.model.CurrentState;
+import org.apache.helix.model.CustomizedStateConfig;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.InstanceConfig;
@@ -53,12 +57,17 @@ public class TestZKCallback extends ZkUnitTestBase {
}
public class TestCallbackListener implements MessageListener, LiveInstanceChangeListener,
- ConfigChangeListener, CurrentStateChangeListener, ExternalViewChangeListener,
+ ConfigChangeListener, CurrentStateChangeListener,
+ CustomizedStateConfigChangeListener,
+ CustomizedStateRootChangeListener,
+ ExternalViewChangeListener,
IdealStateChangeListener {
boolean externalViewChangeReceived = false;
boolean liveInstanceChangeReceived = false;
boolean configChangeReceived = false;
boolean currentStateChangeReceived = false;
+ boolean customizedStateConfigChangeReceived = false;
+ boolean customizedStateRootChangeReceived = false;
boolean messageChangeReceived = false;
boolean idealStateChangeReceived = false;
@@ -96,6 +105,8 @@ public class TestZKCallback extends ZkUnitTestBase {
liveInstanceChangeReceived = false;
configChangeReceived = false;
currentStateChangeReceived = false;
+ customizedStateConfigChangeReceived = false;
+ customizedStateRootChangeReceived = false;
messageChangeReceived = false;
idealStateChangeReceived = false;
}
@@ -105,6 +116,21 @@ public class TestZKCallback extends ZkUnitTestBase {
// TODO Auto-generated method stub
idealStateChangeReceived = true;
}
+
+
+ @Override
+ public void onCustomizedStateRootChange(String instanceName, List<String> customizedStateTypes,
+ NotificationContext changeContext) {
+ // TODO Auto-generated method stub
+ customizedStateRootChangeReceived = true;
+ }
+
+ @Override
+ public void onCustomizedStateConfigChange(CustomizedStateConfig customizedStateConfig,
+ NotificationContext context) {
+ // TODO Auto-generated method stub
+ customizedStateConfigChangeReceived = true;
+ }
}
@Test()
@@ -122,6 +148,7 @@ public class TestZKCallback extends ZkUnitTestBase {
testHelixManager.addMessageListener(testListener, "localhost_8900");
testHelixManager.addCurrentStateChangeListener(testListener, "localhost_8900",
testHelixManager.getSessionId());
+ testHelixManager.addCustomizedStateRootChangeListener(testListener, "localhost_8900");
testHelixManager.addConfigChangeListener(testListener);
testHelixManager.addIdealStateChangeListener(testListener);
testHelixManager.addExternalViewChangeListener(testListener);
diff --git a/helix-core/src/test/java/org/apache/helix/controller/stages/DummyClusterManager.java b/helix-core/src/test/java/org/apache/helix/controller/stages/DummyClusterManager.java
index 655276b..83dd49e 100644
--- a/helix-core/src/test/java/org/apache/helix/controller/stages/DummyClusterManager.java
+++ b/helix-core/src/test/java/org/apache/helix/controller/stages/DummyClusterManager.java
@@ -19,7 +19,6 @@ package org.apache.helix.controller.stages;
* under the License.
*/
-import java.util.List;
import java.util.Set;
import org.apache.helix.ClusterMessagingService;
@@ -37,10 +36,11 @@ import org.apache.helix.api.listeners.ClusterConfigChangeListener;
import org.apache.helix.api.listeners.ConfigChangeListener;
import org.apache.helix.api.listeners.ControllerChangeListener;
import org.apache.helix.api.listeners.CurrentStateChangeListener;
-import org.apache.helix.api.listeners.CustomizedStateConfigChangeListener;
import org.apache.helix.api.listeners.CustomizedStateChangeListener;
-import org.apache.helix.api.listeners.ExternalViewChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateConfigChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateRootChangeListener;
import org.apache.helix.api.listeners.CustomizedViewChangeListener;
+import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
import org.apache.helix.api.listeners.InstanceConfigChangeListener;
import org.apache.helix.api.listeners.LiveInstanceChangeListener;
@@ -134,8 +134,15 @@ public class DummyClusterManager implements HelixManager {
}
@Override
+ public void addCustomizedStateRootChangeListener(CustomizedStateRootChangeListener listener,
+ String instanceName) throws Exception {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
public void addCustomizedStateChangeListener(CustomizedStateChangeListener listener,
- String instanceName, String sessionId) throws Exception {
+ String instanceName, String customizedStateType) throws Exception {
// TODO Auto-generated method stub
}
diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestComputeAndCleanupCustomizedView.java b/helix-core/src/test/java/org/apache/helix/integration/TestComputeAndCleanupCustomizedView.java
new file mode 100644
index 0000000..6cdf72d
--- /dev/null
+++ b/helix-core/src/test/java/org/apache/helix/integration/TestComputeAndCleanupCustomizedView.java
@@ -0,0 +1,271 @@
+package org.apache.helix.integration;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.helix.PropertyKey;
+import org.apache.helix.TestHelper;
+import org.apache.helix.ZNRecord;
+import org.apache.helix.ZkTestHelper;
+import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.integration.manager.ClusterControllerManager;
+import org.apache.helix.integration.manager.MockParticipantManager;
+import org.apache.helix.manager.zk.ZKHelixAdmin;
+import org.apache.helix.manager.zk.ZKHelixDataAccessor;
+import org.apache.helix.manager.zk.ZkBaseDataAccessor;
+import org.apache.helix.model.CustomizedState;
+import org.apache.helix.model.CustomizedStateConfig;
+import org.apache.helix.model.CustomizedView;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+
+/**
+ * Test compute and clean customized view - if customized state is remove externally, controller
+ * should remove the
+ * orphan customized view
+ */
+public class TestComputeAndCleanupCustomizedView extends ZkUnitTestBase {
+
+ private final String RESOURCE_NAME = "TestDB0";
+ private final String PARTITION_NAME1 = "TestDB0_0";
+ private final String PARTITION_NAME2 = "TestDB0_1";
+ private final String CUSTOMIZED_STATE_NAME1 = "customizedState1";
+ private final String CUSTOMIZED_STATE_NAME2 = "customizedState2";
+ private final String INSTANCE_NAME1 = "localhost_12918";
+ private final String INSTANCE_NAME2 = "localhost_12919";
+
+ @Test
+ public void test() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+ int n = 2;
+
+ System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
+
+ TestHelper.setupCluster(clusterName, ZK_ADDR, 12918, // participant port
+ "localhost", // participant name prefix
+ "TestDB", // resource name prefix
+ 1, // resources
+ 2, // partitions per resource
+ n, // number of nodes
+ 2, // replicas
+ "MasterSlave", true); // do rebalance
+
+ ClusterControllerManager controller =
+ new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
+ controller.syncStart();
+
+ // start participants
+ MockParticipantManager[] participants = new MockParticipantManager[n];
+ for (int i = 0; i < n; i++) {
+ String instanceName = "localhost_" + (12918 + i);
+
+ participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
+ participants[i].syncStart();
+ }
+
+ ZKHelixDataAccessor accessor =
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
+ PropertyKey.Builder keyBuilder = accessor.keyBuilder();
+
+ // add CUSTOMIZED_STATE_NAME2 to aggregation enabled types
+ CustomizedStateConfig config = new CustomizedStateConfig();
+ List<String> aggregationEnabledTypes = new ArrayList<>();
+ aggregationEnabledTypes.add(CUSTOMIZED_STATE_NAME2);
+ config.setAggregationEnabledTypes(aggregationEnabledTypes);
+ accessor.setProperty(keyBuilder.customizedStateConfig(), config);
+
+ // set INSTANCE1 to "STARTED" for CUSTOMIZED_STATE_NAME1
+ CustomizedState customizedState = new CustomizedState(RESOURCE_NAME);
+ customizedState.setState(PARTITION_NAME1, "STARTED");
+ accessor.setProperty(
+ keyBuilder.customizedState(INSTANCE_NAME1, CUSTOMIZED_STATE_NAME1, RESOURCE_NAME),
+ customizedState);
+
+ // verify the customized view is empty for CUSTOMIZED_STATE_NAME1
+ Boolean result = TestHelper.verify(new TestHelper.Verifier() {
+ @Override
+ public boolean verify() {
+ CustomizedView customizedView =
+ accessor.getProperty(keyBuilder.customizedView(CUSTOMIZED_STATE_NAME1, RESOURCE_NAME));
+ if (customizedView == null) {
+ return true;
+ }
+ return false;
+ }
+ }, 12000);
+
+ Thread.sleep(50);
+ Assert.assertTrue(result, String
+ .format("Customized view should not have state for" + " resource: %s, partition: %s",
+ RESOURCE_NAME, PARTITION_NAME1));
+
+ // add CUSTOMIZED_STATE_NAME1 to aggregation enabled types
+ aggregationEnabledTypes.add(CUSTOMIZED_STATE_NAME1);
+ config.setAggregationEnabledTypes(aggregationEnabledTypes);
+ config.setAggregationEnabledTypes(aggregationEnabledTypes);
+ accessor.setProperty(keyBuilder.customizedStateConfig(), config);
+
+ // verify the customized view should have "STARTED" for CUSTOMIZED_STATE_NAME1 for INSTANCE1,
+ // but no state for INSTANCE2
+ result = TestHelper.verify(new TestHelper.Verifier() {
+ @Override
+ public boolean verify() {
+ CustomizedView customizedView =
+ accessor.getProperty(keyBuilder.customizedView(CUSTOMIZED_STATE_NAME1, RESOURCE_NAME));
+ if (customizedView != null) {
+ Map<String, String> stateMap = customizedView.getRecord().getMapField(PARTITION_NAME1);
+ return (stateMap.get(INSTANCE_NAME1).equals("STARTED"));
+ }
+ return false;
+ }
+ }, 12000);
+
+ Thread.sleep(50);
+ Assert.assertTrue(result, String.format(
+ "Customized view should have the state as STARTED for" + " instance: %s,"
+ + " resource: %s, partition: %s and state: %s", INSTANCE_NAME1, RESOURCE_NAME,
+ PARTITION_NAME1, CUSTOMIZED_STATE_NAME1));
+
+ result = TestHelper.verify(new TestHelper.Verifier() {
+ @Override
+ public boolean verify() {
+ CustomizedView customizedView =
+ accessor.getProperty(keyBuilder.customizedView(CUSTOMIZED_STATE_NAME1, RESOURCE_NAME));
+ if (customizedView != null) {
+ Map<String, String> stateMap = customizedView.getRecord().getMapField(PARTITION_NAME1);
+ return !stateMap.containsKey(INSTANCE_NAME2);
+ }
+ return false;
+ }
+ }, 12000);
+
+ Thread.sleep(50);
+ Assert.assertTrue(result, String.format("Customized view should not have state for instance: "
+ + "%s", INSTANCE_NAME2));
+
+ // set INSTANCE2 to "STARTED" for CUSTOMIZED_STATE_NAME1
+ customizedState = new CustomizedState(RESOURCE_NAME);
+ customizedState.setState(PARTITION_NAME1, "STARTED");
+ accessor.setProperty(
+ keyBuilder.customizedState(INSTANCE_NAME2, CUSTOMIZED_STATE_NAME1, RESOURCE_NAME),
+ customizedState);
+
+ result = TestHelper.verify(new TestHelper.Verifier() {
+ @Override
+ public boolean verify() {
+ CustomizedView customizedView =
+ accessor.getProperty(keyBuilder.customizedView(CUSTOMIZED_STATE_NAME1, RESOURCE_NAME));
+ if (customizedView != null) {
+ Map<String, String> stateMap = customizedView.getRecord().getMapField(PARTITION_NAME1);
+ if (stateMap.containsKey(INSTANCE_NAME2)) {
+ return (stateMap.get(INSTANCE_NAME1).equals("STARTED") && stateMap.get(INSTANCE_NAME2)
+ .equals("STARTED"));
+ }
+ }
+ return false;
+ }
+ }, 12000);
+
+ Thread.sleep(50);
+ Assert.assertTrue(result, String.format(
+ "Customized view should have both instances state " + "as STARTED for"
+ + " resource: %s, partition: %s and state: %s", RESOURCE_NAME, PARTITION_NAME1,
+ CUSTOMIZED_STATE_NAME1));
+
+ // set INSTANCE2 to "STARTED" for CUSTOMIZED_STATE_NAME2
+ customizedState = new CustomizedState(RESOURCE_NAME);
+ customizedState.setState(PARTITION_NAME2, "STARTED");
+ accessor.setProperty(
+ keyBuilder.customizedState(INSTANCE_NAME2, CUSTOMIZED_STATE_NAME2, RESOURCE_NAME),
+ customizedState);
+
+ result = TestHelper.verify(new TestHelper.Verifier() {
+ @Override
+ public boolean verify() {
+ CustomizedView customizedView =
+ accessor.getProperty(keyBuilder.customizedView(CUSTOMIZED_STATE_NAME2, RESOURCE_NAME));
+ if (customizedView != null) {
+ Map<String, String> stateMap = customizedView.getRecord().getMapField(PARTITION_NAME2);
+ return (stateMap.get(INSTANCE_NAME2).equals("STARTED"));
+ }
+ return false;
+ }
+ }, 12000);
+
+ Thread.sleep(50);
+ Assert.assertTrue(result, String.format(
+ "Customized view should have state " + "as STARTED " + "for instance: %s, "
+ + " resource: %s, partition: %s and state: %s", INSTANCE_NAME2, RESOURCE_NAME,
+ PARTITION_NAME2, CUSTOMIZED_STATE_NAME2));
+
+ // disable controller
+ ZKHelixAdmin admin = new ZKHelixAdmin(_gZkClient);
+ admin.enableCluster(clusterName, false);
+ ZkTestHelper.tryWaitZkEventsCleaned(controller.getZkClient());
+
+ // drop resource
+ admin.dropResource(clusterName, RESOURCE_NAME);
+
+ // delete customized state manually, controller shall remove customized view when cluster is
+ //enabled again
+
+ accessor.removeProperty(
+ keyBuilder.customizedState(INSTANCE_NAME1, CUSTOMIZED_STATE_NAME1, RESOURCE_NAME));
+ accessor.removeProperty(
+ keyBuilder.currentState(INSTANCE_NAME2, CUSTOMIZED_STATE_NAME1, RESOURCE_NAME));
+
+ // re-enable controller shall remove orphan external view
+ // System.out.println("re-enabling controller");
+ admin.enableCluster(clusterName, true);
+
+ result = TestHelper.verify(new TestHelper.Verifier() {
+ @Override
+ public boolean verify() {
+ CustomizedView customizedView =
+ accessor.getProperty(keyBuilder.customizedView(CUSTOMIZED_STATE_NAME1));
+ if (customizedView == null) {
+ return true;
+ }
+ return false;
+ }
+ }, 12000);
+
+ Thread.sleep(50);
+ Assert.assertTrue(result, String
+ .format("customized view for should be null for resource: %s, partition: %s and state: %s",
+ RESOURCE_NAME, PARTITION_NAME1, CUSTOMIZED_STATE_NAME1));
+
+ // clean up
+ controller.syncStop();
+ for (int i = 0; i < n; i++) {
+ participants[i].syncStop();
+ }
+ TestHelper.dropCluster(clusterName, _gZkClient);
+
+ System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
+ }
+}
\ No newline at end of file
diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestZkCallbackHandlerLeak.java b/helix-core/src/test/java/org/apache/helix/integration/TestZkCallbackHandlerLeak.java
index 4682c52..7eb74a7 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/TestZkCallbackHandlerLeak.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/TestZkCallbackHandlerLeak.java
@@ -98,7 +98,7 @@ public class TestZkCallbackHandlerLeak extends ZkUnitTestBase {
// System.out.println("controller watch paths: " + watchPaths);
// where n is number of nodes and r is number of resources
- return watchPaths.size() == (7 + r + ( 4 + r) * n);
+ return watchPaths.size() == (8 + r + ( 5 + r) * n);
}
}, 2000);
Assert.assertTrue(result, "Controller has incorrect number of zk-watchers.");
@@ -123,8 +123,8 @@ public class TestZkCallbackHandlerLeak extends ZkUnitTestBase {
// printHandlers(participantManagerToExpire);
int controllerHandlerNb = controller.getHandlers().size();
int particHandlerNb = participantManagerToExpire.getHandlers().size();
- Assert.assertEquals(controllerHandlerNb, 11,
- "HelixController should have 10 (5+2n) callback handlers for 2 (n) participant");
+ Assert.assertEquals(controllerHandlerNb, 14,
+ "HelixController should have 14 (8+3n) callback handlers for 2 (n) participant");
Assert.assertEquals(particHandlerNb, 1,
"HelixParticipant should have 1 (msg->HelixTaskExecutor) callback handlers");
@@ -135,7 +135,7 @@ public class TestZkCallbackHandlerLeak extends ZkUnitTestBase {
ZkTestHelper.expireSession(participantManagerToExpire.getZkClient());
String newSessionId = participantManagerToExpire.getSessionId();
System.out.println(
- "Expried participant session. oldSessionId: " + oldSessionId + ", newSessionId: "
+ "Expired participant session. oldSessionId: " + oldSessionId + ", newSessionId: "
+ newSessionId);
result =
@@ -153,7 +153,7 @@ public class TestZkCallbackHandlerLeak extends ZkUnitTestBase {
// System.out.println("controller watch paths after session expiry: " + watchPaths);
// where n is number of nodes and r is number of resources
- return watchPaths.size() == (7 + r + ( 4 + r) * n);
+ return watchPaths.size() == (8 + r + ( 5 + r) * n);
}
}, 2000);
Assert.assertTrue(result, "Controller has incorrect number of zk-watchers after session expiry.");
@@ -247,8 +247,8 @@ public class TestZkCallbackHandlerLeak extends ZkUnitTestBase {
int controllerHandlerNb = controller.getHandlers().size();
int particHandlerNb = participantManager.getHandlers().size();
- Assert.assertEquals(controllerHandlerNb, 7 + 2 * n,
- "HelixController should have 10 (6+2n) callback handlers for 2 participant, but was "
+ Assert.assertEquals(controllerHandlerNb, 8 + 3 * n,
+ "HelixController should have 14 (8+3n) callback handlers for 2 participant, but was "
+ controllerHandlerNb + ", " + printHandlers(controller));
Assert.assertEquals(particHandlerNb, 1,
"HelixParticipant should have 1 (msg->HelixTaskExecutor) callback handler, but was "
@@ -276,7 +276,7 @@ public class TestZkCallbackHandlerLeak extends ZkUnitTestBase {
System.err.println("controller watch paths after session expiry: " + watchPaths.size());
// where r is number of resources and n is number of nodes
- int expected = (7 + r + (4 + r) * n);
+ int expected = (8 + r + (5 + r) * n);
return watchPaths.size() == expected;
}
}, 2000);
diff --git a/helix-core/src/test/java/org/apache/helix/integration/manager/TestParticipantManager.java b/helix-core/src/test/java/org/apache/helix/integration/manager/TestParticipantManager.java
index c7a3752..7922153 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/manager/TestParticipantManager.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/manager/TestParticipantManager.java
@@ -151,7 +151,7 @@ public class TestParticipantManager extends ZkTestBase {
// check HelixCallback Monitor
Set<ObjectInstance> objs =
_server.queryMBeans(buildCallbackMonitorObjectName(type, clusterName, instanceName), null);
- Assert.assertEquals(objs.size(), 16);
+ Assert.assertEquals(objs.size(), 17);
// check HelixZkClient Monitors
objs =
diff --git a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
index 5bc6d4f..ee64524 100644
--- a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
+++ b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
@@ -99,8 +99,10 @@ public class TestZkHelixAdmin extends ZkUnitTestBase {
HelixAdmin tool = new ZKHelixAdmin(_gZkClient);
tool.addCluster(clusterName, true);
Assert.assertTrue(ZKUtil.isClusterSetup(clusterName, _gZkClient));
+ Assert.assertTrue(_gZkClient.exists(PropertyPathBuilder.customizedStateConfig(clusterName)));
tool.addCluster(clusterName, true);
Assert.assertTrue(ZKUtil.isClusterSetup(clusterName, _gZkClient));
+ Assert.assertTrue(_gZkClient.exists(PropertyPathBuilder.customizedStateConfig(clusterName)));
List<String> list = tool.getClusters();
AssertJUnit.assertTrue(list.size() > 0);
diff --git a/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java b/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java
index 29e64f6..18046c4 100644
--- a/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java
+++ b/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java
@@ -105,6 +105,10 @@ public class MockHelixAdmin implements HelixAdmin {
path = PropertyPathBuilder.resourceConfig(clusterName);
_baseDataAccessor.create(path, new ZNRecord(clusterName), 0);
+
+ path = PropertyPathBuilder.customizedStateConfig(clusterName);
+ _baseDataAccessor.create(path, new ZNRecord(clusterName), 0);
+
// PROPERTY STORE
path = PropertyPathBuilder.propertyStore(clusterName);
_baseDataAccessor.create(path, new ZNRecord(clusterName), 0);
diff --git a/helix-core/src/test/java/org/apache/helix/mock/MockManager.java b/helix-core/src/test/java/org/apache/helix/mock/MockManager.java
index 90c9bc1..51b932d 100644
--- a/helix-core/src/test/java/org/apache/helix/mock/MockManager.java
+++ b/helix-core/src/test/java/org/apache/helix/mock/MockManager.java
@@ -19,7 +19,6 @@ package org.apache.helix.mock;
* under the License.
*/
-import java.util.List;
import java.util.Set;
import java.util.UUID;
@@ -39,9 +38,10 @@ import org.apache.helix.api.listeners.ClusterConfigChangeListener;
import org.apache.helix.api.listeners.ConfigChangeListener;
import org.apache.helix.api.listeners.ControllerChangeListener;
import org.apache.helix.api.listeners.CurrentStateChangeListener;
-import org.apache.helix.api.listeners.CustomizedViewChangeListener;
-import org.apache.helix.api.listeners.CustomizedStateConfigChangeListener;
import org.apache.helix.api.listeners.CustomizedStateChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateConfigChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateRootChangeListener;
+import org.apache.helix.api.listeners.CustomizedViewChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
import org.apache.helix.api.listeners.InstanceConfigChangeListener;
@@ -138,8 +138,15 @@ public class MockManager implements HelixManager {
}
@Override
+ public void addCustomizedStateRootChangeListener(CustomizedStateRootChangeListener listener,
+ String instanceName) {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
public void addCustomizedStateChangeListener(CustomizedStateChangeListener listener,
- String instanceName, String sessionId) {
+ String instanceName, String customizedStateType) {
// TODO Auto-generated method stub
}
diff --git a/helix-core/src/test/java/org/apache/helix/participant/MockZKHelixManager.java b/helix-core/src/test/java/org/apache/helix/participant/MockZKHelixManager.java
index b31ba82..e671482 100644
--- a/helix-core/src/test/java/org/apache/helix/participant/MockZKHelixManager.java
+++ b/helix-core/src/test/java/org/apache/helix/participant/MockZKHelixManager.java
@@ -19,7 +19,6 @@ package org.apache.helix.participant;
* under the License.
*/
-import java.util.List;
import java.util.Set;
import java.util.UUID;
@@ -38,9 +37,10 @@ import org.apache.helix.api.listeners.ClusterConfigChangeListener;
import org.apache.helix.api.listeners.ConfigChangeListener;
import org.apache.helix.api.listeners.ControllerChangeListener;
import org.apache.helix.api.listeners.CurrentStateChangeListener;
-import org.apache.helix.api.listeners.CustomizedViewChangeListener;
-import org.apache.helix.api.listeners.CustomizedStateConfigChangeListener;
import org.apache.helix.api.listeners.CustomizedStateChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateConfigChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateRootChangeListener;
+import org.apache.helix.api.listeners.CustomizedViewChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
import org.apache.helix.api.listeners.InstanceConfigChangeListener;
@@ -170,8 +170,15 @@ public class MockZKHelixManager implements HelixManager {
}
@Override
+ public void addCustomizedStateRootChangeListener(CustomizedStateRootChangeListener listener,
+ String instanceName) throws Exception {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
public void addCustomizedStateChangeListener(CustomizedStateChangeListener listener,
- String instanceName, String sessionId) throws Exception {
+ String instanceName, String customizedStateType) throws Exception {
// TODO Auto-generated method stub
}
[helix] 17/23: Replace customized view cache with property cache
(#869)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 9c40583cdc411a1731bb19c8d941a45b584039e8
Author: Ali Reza Zamani Zadeh Najari <an...@linkedin.com>
AuthorDate: Mon Mar 30 16:47:13 2020 -0700
Replace customized view cache with property cache (#869)
In this commit, the custom implementation of customized view cache
has been replaced with property cache implementation.
---
.../helix/common/caches/CustomizedViewCache.java | 116 +++++----------------
.../spectator/TestRoutingTableProvider.java | 69 ++++++++++--
2 files changed, 86 insertions(+), 99 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java b/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java
index d46be64..cdbc0f8 100644
--- a/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java
+++ b/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java
@@ -19,23 +19,16 @@ package org.apache.helix.common.caches;
* under the License.
*/
-import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
import java.util.Map;
-import java.util.Set;
import org.apache.helix.HelixDataAccessor;
-import org.apache.helix.HelixException;
import org.apache.helix.PropertyKey;
import org.apache.helix.PropertyType;
import org.apache.helix.model.CustomizedView;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Maps;
/**
* Cache to hold all CustomizedView of a specific type.
@@ -43,8 +36,7 @@ import com.google.common.collect.Maps;
public class CustomizedViewCache extends AbstractDataCache<CustomizedView> {
private static final Logger LOG = LoggerFactory.getLogger(CustomizedViewCache.class.getName());
- protected Map<String, CustomizedView> _customizedViewMap;
- protected Map<String, CustomizedView> _customizedViewCache;
+ private final PropertyCache<CustomizedView> _customizedViewCache;
protected String _clusterName;
private PropertyType _propertyType;
private String _customizedStateType;
@@ -54,13 +46,27 @@ public class CustomizedViewCache extends AbstractDataCache<CustomizedView> {
}
protected CustomizedViewCache(String clusterName, PropertyType propertyType, String customizedStateType) {
- super(createDefaultControlContextProvider(clusterName));
- _clusterName = clusterName;
- _customizedViewMap = Collections.emptyMap();
- _customizedViewCache = Collections.emptyMap();
- _propertyType = propertyType;
- _customizedStateType = customizedStateType;
- }
+ super(createDefaultControlContextProvider(clusterName));
+ _clusterName = clusterName;
+ _propertyType = propertyType;
+ _customizedStateType = customizedStateType;
+ _customizedViewCache = new PropertyCache<>(AbstractDataCache.createDefaultControlContextProvider(clusterName), "CustomizedView", new PropertyCache.PropertyCacheKeyFuncs<CustomizedView>() {
+ @Override
+ public PropertyKey getRootKey(HelixDataAccessor accessor) {
+ return accessor.keyBuilder().customizedView(_customizedStateType);
+ }
+
+ @Override
+ public PropertyKey getObjPropertyKey(HelixDataAccessor accessor, String objName) {
+ return accessor.keyBuilder().customizedView(_customizedStateType, objName);
+ }
+
+ @Override
+ public String getObjName(CustomizedView obj) {
+ return obj.getResourceName();
+ }
+ }, true);
+ }
/**
@@ -70,65 +76,7 @@ public class CustomizedViewCache extends AbstractDataCache<CustomizedView> {
* @return
*/
public void refresh(HelixDataAccessor accessor) {
- long startTime = System.currentTimeMillis();
- PropertyKey.Builder keyBuilder = accessor.keyBuilder();
- Set<PropertyKey> currentPropertyKeys = new HashSet<>();
-
- List<String> resources = accessor.getChildNames(customizedViewsKey(keyBuilder));
-
- for (String resource : resources) {
- currentPropertyKeys.add(customizedViewKey(keyBuilder, resource));
- }
-
- Set<PropertyKey> cachedKeys = new HashSet<>();
- Map<PropertyKey, CustomizedView> cachedCustomizedViewMap = Maps.newHashMap();
- for (String resource : _customizedViewCache.keySet()) {
- PropertyKey key = customizedViewKey(keyBuilder, resource);
- cachedKeys.add(key);
- cachedCustomizedViewMap.put(key, _customizedViewCache.get(resource));
- }
- cachedKeys.retainAll(currentPropertyKeys);
-
- Set<PropertyKey> reloadKeys = new HashSet<>(currentPropertyKeys);
- reloadKeys.removeAll(cachedKeys);
-
- Map<PropertyKey, CustomizedView> updatedMap =
- refreshProperties(accessor, reloadKeys, new ArrayList<>(cachedKeys),
- cachedCustomizedViewMap, new HashSet<>());
-
- Map<String, CustomizedView> newCustomizedViewMap = Maps.newHashMap();
- for (CustomizedView customizedView : updatedMap.values()) {
- newCustomizedViewMap.put(customizedView.getResourceName(), customizedView);
- }
-
- _customizedViewCache = new HashMap<>(newCustomizedViewMap);
- _customizedViewMap = new HashMap<>(newCustomizedViewMap);
-
- long endTime = System.currentTimeMillis();
- LOG.info("Refresh " + _customizedViewMap.size() + " CustomizedViews of type " + _customizedStateType
- + " for cluster " + _clusterName + ", took " + (endTime - startTime) + " ms");
- }
-
- private PropertyKey customizedViewsKey(PropertyKey.Builder keyBuilder) {
- PropertyKey customizedViewPropertyKey;
- if (_propertyType.equals(PropertyType.CUSTOMIZEDVIEW)){
- customizedViewPropertyKey = keyBuilder.customizedView(_customizedStateType);
- } else {
- throw new HelixException(
- "Failed to refresh CustomizedViewCache, Wrong property type " + _propertyType + "!");
- }
- return customizedViewPropertyKey;
- }
-
- private PropertyKey customizedViewKey(PropertyKey.Builder keyBuilder, String resource) {
- PropertyKey customizedViewPropertyKey;
- if (_propertyType.equals(PropertyType.CUSTOMIZEDVIEW)) {
- customizedViewPropertyKey = keyBuilder.customizedView(_customizedStateType, resource);
- } else {
- throw new HelixException(
- "Failed to refresh CustomizedViewCache, Wrong property type " + _propertyType + "!");
- }
- return customizedViewPropertyKey;
+ _customizedViewCache.refresh(accessor);
}
/**
@@ -136,22 +84,6 @@ public class CustomizedViewCache extends AbstractDataCache<CustomizedView> {
* @return
*/
public Map<String, CustomizedView> getCustomizedViewMap() {
- return Collections.unmodifiableMap(_customizedViewMap);
- }
-
- /**
- * Remove dead customized views from map
- * @param resourceNames
- */
-
- public synchronized void removeCustomizedView(List<String> resourceNames) {
- for (String resourceName : resourceNames) {
- _customizedViewCache.remove(resourceName);
- }
- }
-
- public void clear() {
- _customizedViewCache.clear();
- _customizedViewMap.clear();
+ return Collections.unmodifiableMap(_customizedViewCache.getPropertyMap());
}
}
diff --git a/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProvider.java b/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProvider.java
index d1a2b4c..43ccdc9 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProvider.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProvider.java
@@ -1,5 +1,24 @@
package org.apache.helix.integration.spectator;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -47,6 +66,7 @@ public class TestRoutingTableProvider extends ZkTestBase {
static final String CLUSTER_NAME = CLUSTER_PREFIX + "_" + CLASS_NAME;
static final int PARTICIPANT_NUMBER = 3;
static final int PARTICIPANT_START_PORT = 12918;
+ static final long WAIT_DURATION = 5 * 1000L; // 5 seconds
static final int PARTITION_NUMBER = 20;
static final int REPLICA_NUMBER = 3;
@@ -269,7 +289,7 @@ public class TestRoutingTableProvider extends ZkTestBase {
customizedView.getRecord(), AccessOption.PERSISTENT);
boolean onCustomizedViewChangeCalled =
- TestHelper.verify(() -> customizedViewChangeCalled.get(), TestHelper.WAIT_DURATION);
+ TestHelper.verify(() -> customizedViewChangeCalled.get(), WAIT_DURATION);
Assert.assertTrue(onCustomizedViewChangeCalled);
_spectator.getHelixDataAccessor().getBaseDataAccessor().remove(customizedViewPath,
@@ -278,7 +298,7 @@ public class TestRoutingTableProvider extends ZkTestBase {
}
@Test(dependsOnMethods = "testCustomizedViewCorrectConstructor")
- public void testGetRoutingTableSnapshot() {
+ public void testGetRoutingTableSnapshot() throws Exception {
Map<PropertyType, List<String>> sourceDataTypes = new HashMap<>();
sourceDataTypes.put(PropertyType.CUSTOMIZEDVIEW, Arrays.asList("typeA", "typeB"));
sourceDataTypes.put(PropertyType.EXTERNALVIEW, Collections.emptyList());
@@ -308,19 +328,54 @@ public class TestRoutingTableProvider extends ZkTestBase {
routingTableProvider.getRoutingTableSnapshot(PropertyType.CUSTOMIZEDVIEW, "typeA");
Assert.assertEquals(routingTableSnapshot.getPropertyType(), PropertyType.CUSTOMIZEDVIEW);
Assert.assertEquals(routingTableSnapshot.getCustomizedStateType(), "typeA");
+
routingTableSnapshot =
routingTableProvider.getRoutingTableSnapshot(PropertyType.CUSTOMIZEDVIEW, "typeB");
Assert.assertEquals(routingTableSnapshot.getPropertyType(), PropertyType.CUSTOMIZEDVIEW);
Assert.assertEquals(routingTableSnapshot.getCustomizedStateType(), "typeB");
- Map<String, Map<String, RoutingTableSnapshot>> routingTableSnapshots =
- routingTableProvider.getRoutingTableSnapshots();
- Assert.assertEquals(routingTableSnapshots.size(), 2);
- Assert.assertEquals(routingTableSnapshots.get(PropertyType.CUSTOMIZEDVIEW.name()).size(), 2);
+ // Make sure snapshot information is correct
+ // Check resources are in a correct state
+ boolean isRoutingTableUpdatedProperly = TestHelper.verify(() -> {
+ Map<String, Map<String, RoutingTableSnapshot>> routingTableSnapshots =
+ routingTableProvider.getRoutingTableSnapshots();
+ RoutingTableSnapshot routingTableSnapshotTypeA =
+ routingTableSnapshots.get(PropertyType.CUSTOMIZEDVIEW.name()).get("typeA");
+ RoutingTableSnapshot routingTableSnapshotTypeB =
+ routingTableSnapshots.get(PropertyType.CUSTOMIZEDVIEW.name()).get("typeB");
+ String typeAp1h1 = "noState";
+ String typeAp1h2 = "noState";
+ String typeAp2h1 = "noState";
+ String typeAp3h2 = "noState";
+ String typeBp1h2 = "noState";
+ String typeBp1h4 = "noState";
+ try {
+ typeAp1h1 = routingTableSnapshotTypeA.getCustomizeViews().iterator().next()
+ .getStateMap("p1").get("h1");
+ typeAp1h2 = routingTableSnapshotTypeA.getCustomizeViews().iterator().next()
+ .getStateMap("p1").get("h2");
+ typeAp2h1 = routingTableSnapshotTypeA.getCustomizeViews().iterator().next()
+ .getStateMap("p2").get("h1");
+ typeAp3h2 = routingTableSnapshotTypeA.getCustomizeViews().iterator().next()
+ .getStateMap("p3").get("h2");
+ typeBp1h2 = routingTableSnapshotTypeB.getCustomizeViews().iterator().next()
+ .getStateMap("p1").get("h3");
+ typeBp1h4 = routingTableSnapshotTypeB.getCustomizeViews().iterator().next()
+ .getStateMap("p1").get("h4");
+ } catch (Exception e) {
+ // ok because RoutingTable has not been updated yet
+ return false;
+ }
+ return (routingTableSnapshots.size() == 2
+ && routingTableSnapshots.get(PropertyType.CUSTOMIZEDVIEW.name()).size() == 2
+ && typeAp1h1.equals("testState1") && typeAp1h2.equals("testState1")
+ && typeAp2h1.equals("testState2") && typeAp3h2.equals("testState3")
+ && typeBp1h2.equals("testState3") && typeBp1h4.equals("testState2"));
+ }, WAIT_DURATION);
+ Assert.assertTrue(isRoutingTableUpdatedProperly, "RoutingTable has been updated properly");
routingTableProvider.shutdown();
}
-
private void validateRoutingTable(RoutingTableProvider routingTableProvider,
Set<String> masterNodes, Set<String> slaveNodes) {
IdealState is =
[helix] 16/23: minor fix for customized view aggregation (#917)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit fc78947a4c42b3e2f905861ac697a3b301e52b3b
Author: zhangmeng916 <56...@users.noreply.github.com>
AuthorDate: Mon Mar 30 14:26:57 2020 -0700
minor fix for customized view aggregation (#917)
Fix minor issues in customized view aggregation logic and add some more tests.
Co-authored-by: Meng Zhang <mn...@mnzhang-mn1.linkedin.biz>
---
.../helix/controller/GenericHelixController.java | 30 ++++++++++++++++------
.../ResourceControllerDataProvider.java | 12 ++++-----
.../stages/CustomizedViewAggregationStage.java | 6 ++---
.../controller/stages/TestCustomizedViewStage.java | 19 ++++++++++----
.../TestComputeAndCleanupCustomizedView.java | 26 ++++++++++++++++---
5 files changed, 66 insertions(+), 27 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/controller/GenericHelixController.java b/helix-core/src/main/java/org/apache/helix/controller/GenericHelixController.java
index 8644441..5678a5c 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/GenericHelixController.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/GenericHelixController.java
@@ -23,7 +23,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -142,7 +141,9 @@ public class GenericHelixController implements IdealStateChangeListener, LiveIns
final AtomicReference<Map<String, LiveInstance>> _lastSeenInstances;
final AtomicReference<Map<String, LiveInstance>> _lastSeenSessions;
- final AtomicReference<Set<String>> _lastSeenCustomizedStateTypes;
+ // map that stores the mapping between instance and the customized state types available on that
+ //instance
+ final AtomicReference<Map<String, Set<String>>> _lastSeenCustomizedStateTypesMapRef;
// By default not reporting status until controller status is changed to activate
// TODO This flag should be inside ClusterStatusMonitor. When false, no MBean registering.
@@ -585,7 +586,7 @@ public class GenericHelixController implements IdealStateChangeListener, LiveIns
_taskRegistry = taskRegistry;
_lastSeenInstances = new AtomicReference<>();
_lastSeenSessions = new AtomicReference<>();
- _lastSeenCustomizedStateTypes = new AtomicReference<>();
+ _lastSeenCustomizedStateTypesMapRef = new AtomicReference<>();
_clusterName = clusterName;
_lastPipelineEndTimestamp = TopStateHandoffReportStage.TIMESTAMP_NOT_RECORDED;
_clusterStatusMonitor = new ClusterStatusMonitor(_clusterName);
@@ -881,12 +882,24 @@ public class GenericHelixController implements IdealStateChangeListener, LiveIns
}
// TODO: remove the synchronization here once we move this update into dataCache.
- synchronized (_lastSeenCustomizedStateTypes) {
- Set<String> lastSeenCustomizedStateTypes = _lastSeenCustomizedStateTypes.get();
+ synchronized (_lastSeenCustomizedStateTypesMapRef) {
+ Map<String, Set<String>> lastSeenCustomizedStateTypesMap =
+ _lastSeenCustomizedStateTypesMapRef.get();
+ if (null == lastSeenCustomizedStateTypesMap) {
+ lastSeenCustomizedStateTypesMap = new HashMap();
+ // lazy init the AtomicReference
+ _lastSeenCustomizedStateTypesMapRef.set(lastSeenCustomizedStateTypesMap);
+ }
+
+ if (!lastSeenCustomizedStateTypesMap.containsKey(instanceName)) {
+ lastSeenCustomizedStateTypesMap.put(instanceName, Collections.emptySet());
+ }
+
+ Set<String> lastSeenCustomizedStateTypes = lastSeenCustomizedStateTypesMap.get(instanceName);
+
for (String customizedState : customizedStateTypes) {
try {
- if (lastSeenCustomizedStateTypes == null || !lastSeenCustomizedStateTypes
- .contains(customizedState)) {
+ if (!lastSeenCustomizedStateTypes.contains(customizedState)) {
manager.addCustomizedStateChangeListener(this, instanceName, customizedState);
logger.info(
manager.getInstanceName() + " added customized state listener for " + instanceName
@@ -904,7 +917,8 @@ public class GenericHelixController implements IdealStateChangeListener, LiveIns
}
}
- _lastSeenCustomizedStateTypes.set(new HashSet<>(customizedStateTypes));
+ lastSeenCustomizedStateTypes.clear();
+ lastSeenCustomizedStateTypes.addAll(customizedStateTypes);
}
}
diff --git a/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java b/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java
index 50a3ac9..3625630 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java
@@ -19,7 +19,6 @@ package org.apache.helix.controller.dataproviders;
* under the License.
*/
-import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -220,9 +219,8 @@ public class ResourceControllerDataProvider extends BaseControllerDataProvider {
public void refreshCustomizedViewMap(final HelixDataAccessor accessor) {
// As we are not listening on customized view change, customized view will be
// refreshed once during the cache's first refresh() call, or when full refresh is required
- List<String> newStateTypes = accessor.getChildNames(accessor.keyBuilder().customizedViews());
if (_propertyDataChangedMap.get(HelixConstants.ChangeType.CUSTOMIZED_VIEW).getAndSet(false)) {
- for (String stateType : newStateTypes) {
+ for (String stateType : _aggregationEnabledTypes) {
if (!_customizedViewCacheMap.containsKey(stateType)) {
CustomizedViewCache newCustomizedViewCache =
new CustomizedViewCache(getClusterName(), stateType);
@@ -230,10 +228,10 @@ public class ResourceControllerDataProvider extends BaseControllerDataProvider {
}
_customizedViewCacheMap.get(stateType).refresh(accessor);
}
- Set<String> previousCachedStateTypes = _customizedViewCacheMap.keySet();
- previousCachedStateTypes.removeAll(newStateTypes);
+ Set<String> previousCachedStateTypes = new HashSet<>(_customizedViewCacheMap.keySet());
+ previousCachedStateTypes.removeAll(_aggregationEnabledTypes);
logger.info("Remove customizedView for state: " + previousCachedStateTypes);
- removeCustomizedViewTypes(new ArrayList<>(previousCachedStateTypes));
+ removeCustomizedViewTypes(previousCachedStateTypes);
}
}
@@ -306,7 +304,7 @@ public class ResourceControllerDataProvider extends BaseControllerDataProvider {
* @param stateTypeNames
*/
- private void removeCustomizedViewTypes(List<String> stateTypeNames) {
+ private void removeCustomizedViewTypes(Set<String> stateTypeNames) {
for (String stateType : stateTypeNames) {
_customizedViewCacheMap.remove(stateType);
}
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/CustomizedViewAggregationStage.java b/helix-core/src/main/java/org/apache/helix/controller/stages/CustomizedViewAggregationStage.java
index de0bc1c..cdcdfd2 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/stages/CustomizedViewAggregationStage.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/stages/CustomizedViewAggregationStage.java
@@ -72,19 +72,17 @@ public class CustomizedViewAggregationStage extends AbstractAsyncBaseStage {
Map<String, CustomizedViewCache> customizedViewCacheMap = cache.getCustomizedViewCacheMap();
- // remove stale customized view type from ZK and cache
- List<String> customizedViewTypesToRemove = new ArrayList<>();
+ // remove stale customized view type from ZK
for (String stateType : customizedViewCacheMap.keySet()) {
if (!customizedStateOutput.getAllStateTypes().contains(stateType)) {
LogUtil.logInfo(LOG, _eventId, "Remove customizedView for stateType: " + stateType);
dataAccessor.removeProperty(keyBuilder.customizedView(stateType));
- customizedViewTypesToRemove.add(stateType);
}
}
- List<CustomizedView> updatedCustomizedViews = new ArrayList<>();
// update customized view
for (String stateType : customizedStateOutput.getAllStateTypes()) {
+ List<CustomizedView> updatedCustomizedViews = new ArrayList<>();
Map<String, CustomizedView> curCustomizedViews = new HashMap<>();
CustomizedViewCache customizedViewCache = customizedViewCacheMap.get(stateType);
if (customizedViewCache != null) {
diff --git a/helix-core/src/test/java/org/apache/helix/controller/stages/TestCustomizedViewStage.java b/helix-core/src/test/java/org/apache/helix/controller/stages/TestCustomizedViewStage.java
index a8a167b..5c04daf 100644
--- a/helix-core/src/test/java/org/apache/helix/controller/stages/TestCustomizedViewStage.java
+++ b/helix-core/src/test/java/org/apache/helix/controller/stages/TestCustomizedViewStage.java
@@ -39,8 +39,8 @@ import org.testng.annotations.Test;
public class TestCustomizedViewStage extends ZkUnitTestBase {
- private final String RESOURCE_NAME = "testResourceName";
- private final String PARTITION_NAME = "testResourceName_0";
+ private final String RESOURCE_NAME = "TestDB";
+ private final String PARTITION_NAME = "TestDB_0";
private final String CUSTOMIZED_STATE_NAME = "customizedState1";
private final String INSTANCE_NAME = "localhost_1";
@@ -52,7 +52,16 @@ public class TestCustomizedViewStage extends ZkUnitTestBase {
new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<>(_gZkClient));
HelixManager manager = new DummyClusterManager(clusterName, accessor);
- setupLiveInstances(clusterName, new int[]{0, 1});
+ // ideal state: node0 is MASTER, node1 is SLAVE
+ // replica=2 means 1 master and 1 slave
+ setupIdealState(clusterName, new int[] {
+ 0, 1
+ }, new String[] {
+ "TestDB"
+ }, 1, 2);
+ setupLiveInstances(clusterName, new int[] {
+ 0, 1
+ });
setupStateModel(clusterName);
ClusterEvent event = new ClusterEvent(ClusterEventType.Unknown);
@@ -82,8 +91,8 @@ public class TestCustomizedViewStage extends ZkUnitTestBase {
runStage(event, new ResourceComputationStage());
runStage(event, new CustomizedStateComputationStage());
runStage(event, customizedViewComputeStage);
- Assert.assertEquals(cache.getCustomizedViewCacheMap().values(),
- accessor.getChildValues(accessor.keyBuilder().customizedViews()));
+ Assert.assertEquals(cache.getCustomizedViewCacheMap().size(),
+ accessor.getChildNames(accessor.keyBuilder().customizedViews()).size());
// Assure there is no customized view got updated when running the stage again
List<CustomizedView> oldCustomizedViews =
diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestComputeAndCleanupCustomizedView.java b/helix-core/src/test/java/org/apache/helix/integration/TestComputeAndCleanupCustomizedView.java
index 6cdf72d..6de2521 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/TestComputeAndCleanupCustomizedView.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/TestComputeAndCleanupCustomizedView.java
@@ -126,7 +126,6 @@ public class TestComputeAndCleanupCustomizedView extends ZkUnitTestBase {
// add CUSTOMIZED_STATE_NAME1 to aggregation enabled types
aggregationEnabledTypes.add(CUSTOMIZED_STATE_NAME1);
config.setAggregationEnabledTypes(aggregationEnabledTypes);
- config.setAggregationEnabledTypes(aggregationEnabledTypes);
accessor.setProperty(keyBuilder.customizedStateConfig(), config);
// verify the customized view should have "STARTED" for CUSTOMIZED_STATE_NAME1 for INSTANCE1,
@@ -164,8 +163,8 @@ public class TestComputeAndCleanupCustomizedView extends ZkUnitTestBase {
}, 12000);
Thread.sleep(50);
- Assert.assertTrue(result, String.format("Customized view should not have state for instance: "
- + "%s", INSTANCE_NAME2));
+ Assert.assertTrue(result, String
+ .format("Customized view should not have state for instance: " + "%s", INSTANCE_NAME2));
// set INSTANCE2 to "STARTED" for CUSTOMIZED_STATE_NAME1
customizedState = new CustomizedState(RESOURCE_NAME);
@@ -222,6 +221,27 @@ public class TestComputeAndCleanupCustomizedView extends ZkUnitTestBase {
+ " resource: %s, partition: %s and state: %s", INSTANCE_NAME2, RESOURCE_NAME,
PARTITION_NAME2, CUSTOMIZED_STATE_NAME2));
+ // remove CUSTOMIZED_STATE_NAME1 from aggregation enabled types
+ aggregationEnabledTypes.remove(CUSTOMIZED_STATE_NAME1);
+ config.setAggregationEnabledTypes(aggregationEnabledTypes);
+ accessor.setProperty(keyBuilder.customizedStateConfig(), config);
+
+ result = TestHelper.verify(new TestHelper.Verifier() {
+ @Override
+ public boolean verify() {
+ CustomizedView customizedView =
+ accessor.getProperty(keyBuilder.customizedView(CUSTOMIZED_STATE_NAME1, RESOURCE_NAME));
+ if (customizedView == null) {
+ return true;
+ }
+ return false;
+ }
+ }, 12000);
+
+ Thread.sleep(50);
+ Assert.assertTrue(result,
+ String.format("Customized view should not have state %s", CUSTOMIZED_STATE_NAME1));
+
// disable controller
ZKHelixAdmin admin = new ZKHelixAdmin(_gZkClient);
admin.enableCluster(clusterName, false);
[helix] 01/23: Add the CustomizedView Helix property (#723)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 4ece90aac1bedb58e087b760be42d27739d60def
Author: Ali Reza Zamani Zadeh Najari <an...@linkedin.com>
AuthorDate: Wed Feb 5 13:19:22 2020 -0800
Add the CustomizedView Helix property (#723)
This commit contains the bare minimum properties for CustomizedView Helix property.
---
.../org/apache/helix/model/CustomizedView.java | 98 ++++++++++++++++++++++
1 file changed, 98 insertions(+)
diff --git a/helix-core/src/main/java/org/apache/helix/model/CustomizedView.java b/helix-core/src/main/java/org/apache/helix/model/CustomizedView.java
new file mode 100644
index 0000000..791829a
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/model/CustomizedView.java
@@ -0,0 +1,98 @@
+package org.apache.helix.model;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.helix.HelixProperty;
+import org.apache.helix.ZNRecord;
+
+/**
+ * Customized view is an aggregation (across all instances)
+ * of customized states for the resource
+ */
+public class CustomizedView extends HelixProperty {
+
+ /**
+ * Instantiate a customized view with the resource it corresponds to
+ * @param resource the name of the resource
+ */
+ public CustomizedView(String resource) {
+ super(new ZNRecord(resource));
+ }
+
+ /**
+ * Instantiate a customized view with a pre-populated record
+ * @param record ZNRecord corresponding to a customized view
+ */
+ public CustomizedView(ZNRecord record) {
+ super(record);
+ }
+
+ /**
+ * For a given replica, specify which partition it corresponds to, where it is served, and its
+ * current state
+ * @param partition the partition of the replica being served
+ * @param instance the instance serving the replica
+ * @param customState the customized state the replica is in
+ */
+ public void setState(String partition, String instance, String customState) {
+ if (_record.getMapField(partition) == null) {
+ _record.setMapField(partition, new TreeMap<String, String>());
+ }
+ _record.getMapField(partition).put(instance, customState);
+ }
+
+ /**
+ * For a given partition, indicate where and in what customized state each of its replicas is in
+ * @param partitionName the partition to set
+ * @param customizedStateMap (instance, state) pairs
+ */
+ public void setStateMap(String partitionName, Map<String, String> customizedStateMap) {
+ _record.setMapField(partitionName, customizedStateMap);
+ }
+
+ /**
+ * Get all the partitions of the resource
+ * @return a set of partition names
+ */
+ public Set<String> getPartitionSet() {
+ return _record.getMapFields().keySet();
+ }
+
+ /**
+ * Get the instance and the state for each partition replica
+ * @param partitionName the partition to look up
+ * @return (instance, state) pairs
+ */
+ public Map<String, String> getStateMap(String partitionName) {
+ return _record.getMapField(partitionName);
+ }
+
+ /**
+ * Get the resource represented by this view
+ * @return the name of the resource
+ */
+ public String getResourceName() {
+ return _record.getId();
+ }
+}
[helix] 14/23: Complete the Routing Table Provider for
CustomizedView (#834)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit cae7afedea9aedce18a67666861ffd9e4268f295
Author: Ali Reza Zamani Zadeh Najari <an...@linkedin.com>
AuthorDate: Fri Mar 20 11:32:47 2020 -0700
Complete the Routing Table Provider for CustomizedView (#834)
In this commit, the routing table provider has been changed in a way to include customized view feature.
---
.../main/java/org/apache/helix/HelixManager.java | 2 +-
.../main/java/org/apache/helix/PropertyKey.java | 12 +-
.../java/org/apache/helix/PropertyPathBuilder.java | 9 +-
.../helix/common/caches/CustomizedViewCache.java | 16 +-
.../apache/helix/manager/zk/ZKHelixManager.java | 4 +-
.../spectator/CustomizedViewRoutingTable.java | 112 ++++
.../apache/helix/spectator/RoutingDataCache.java | 116 ++--
.../org/apache/helix/spectator/RoutingTable.java | 52 +-
.../helix/spectator/RoutingTableProvider.java | 582 +++++++++++++++------
.../helix/spectator/RoutingTableSnapshot.java | 37 ++
.../controller/stages/DummyClusterManager.java | 2 +-
.../spectator/TestRoutingTableProvider.java | 109 ++++
.../TestRoutingTableProviderPeriodicRefresh.java | 26 +-
.../java/org/apache/helix/mock/MockManager.java | 2 +-
.../helix/participant/MockZKHelixManager.java | 2 +-
15 files changed, 848 insertions(+), 235 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/HelixManager.java b/helix-core/src/main/java/org/apache/helix/HelixManager.java
index 30f6126..90aaa0c 100644
--- a/helix-core/src/main/java/org/apache/helix/HelixManager.java
+++ b/helix-core/src/main/java/org/apache/helix/HelixManager.java
@@ -244,7 +244,7 @@ public interface HelixManager {
* @see CustomizedViewChangeListener#onCustomizedViewChange(List, NotificationContext)
* @param listener
*/
- void addCustomizedViewChangeListener(CustomizedViewChangeListener listener, String aggregationType) throws Exception;
+ void addCustomizedViewChangeListener(CustomizedViewChangeListener listener, String customizedStateType) throws Exception;
/**
* @see ExternalViewChangeListener#onExternalViewChange(List, NotificationContext)
diff --git a/helix-core/src/main/java/org/apache/helix/PropertyKey.java b/helix-core/src/main/java/org/apache/helix/PropertyKey.java
index 73e125e..41bc9cd 100644
--- a/helix-core/src/main/java/org/apache/helix/PropertyKey.java
+++ b/helix-core/src/main/java/org/apache/helix/PropertyKey.java
@@ -651,20 +651,20 @@ public class PropertyKey {
/**
* Get a property key associated with an {@link CustomizedView} of a type
- * @param aggregationType
+ * @param customizedStateType
* @return {@link PropertyKey}
*/
- public PropertyKey customizedView(String aggregationType) {
- return new PropertyKey(CUSTOMIZEDVIEW, CustomizedView.class, _clusterName, aggregationType);
+ public PropertyKey customizedView(String customizedStateType) {
+ return new PropertyKey(CUSTOMIZEDVIEW, CustomizedView.class, _clusterName, customizedStateType);
}
/**
* Get a property key associated with an {@link CustomizedView} of a type and resource
- * @param aggregationType
+ * @param customizedStateType
* @return {@link PropertyKey}
*/
- public PropertyKey customizedView(String aggregationType, String resourceName) {
- return new PropertyKey(CUSTOMIZEDVIEW, CustomizedView.class, _clusterName, aggregationType,
+ public PropertyKey customizedView(String customizedStateType, String resourceName) {
+ return new PropertyKey(CUSTOMIZEDVIEW, CustomizedView.class, _clusterName, customizedStateType,
resourceName);
}
diff --git a/helix-core/src/main/java/org/apache/helix/PropertyPathBuilder.java b/helix-core/src/main/java/org/apache/helix/PropertyPathBuilder.java
index d9bb2b4..f39530e 100644
--- a/helix-core/src/main/java/org/apache/helix/PropertyPathBuilder.java
+++ b/helix-core/src/main/java/org/apache/helix/PropertyPathBuilder.java
@@ -27,6 +27,7 @@ import java.util.regex.Pattern;
import org.apache.helix.model.ControllerHistory;
import org.apache.helix.model.CurrentState;
+import org.apache.helix.model.CustomizedView;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.InstanceConfig;
@@ -42,6 +43,7 @@ import org.slf4j.LoggerFactory;
import static org.apache.helix.PropertyType.CONFIGS;
import static org.apache.helix.PropertyType.CURRENTSTATES;
+import static org.apache.helix.PropertyType.CUSTOMIZEDVIEW;
import static org.apache.helix.PropertyType.EXTERNALVIEW;
import static org.apache.helix.PropertyType.HISTORY;
import static org.apache.helix.PropertyType.IDEALSTATES;
@@ -70,6 +72,7 @@ public class PropertyPathBuilder {
typeToClassMapping.put(IDEALSTATES, IdealState.class);
typeToClassMapping.put(CONFIGS, InstanceConfig.class);
typeToClassMapping.put(EXTERNALVIEW, ExternalView.class);
+ typeToClassMapping.put(CUSTOMIZEDVIEW, CustomizedView.class);
typeToClassMapping.put(STATEMODELDEFS, StateModelDefinition.class);
typeToClassMapping.put(MESSAGES, Message.class);
typeToClassMapping.put(CURRENTSTATES, CurrentState.class);
@@ -94,6 +97,10 @@ public class PropertyPathBuilder {
addEntry(PropertyType.IDEALSTATES, 2, "/{clusterName}/IDEALSTATES/{resourceName}");
addEntry(PropertyType.EXTERNALVIEW, 1, "/{clusterName}/EXTERNALVIEW");
addEntry(PropertyType.EXTERNALVIEW, 2, "/{clusterName}/EXTERNALVIEW/{resourceName}");
+ addEntry(PropertyType.CUSTOMIZEDVIEW, 1, "/{clusterName}/CUSTOMIZEDVIEW");
+ addEntry(PropertyType.CUSTOMIZEDVIEW, 2, "/{clusterName}/CUSTOMIZEDVIEW/{customizedStateType}");
+ addEntry(PropertyType.CUSTOMIZEDVIEW, 3, "/{clusterName}/CUSTOMIZEDVIEW/{customizedStateType}/{resourceName}");
+
addEntry(PropertyType.TARGETEXTERNALVIEW, 1, "/{clusterName}/TARGETEXTERNALVIEW");
addEntry(PropertyType.TARGETEXTERNALVIEW, 2,
"/{clusterName}/TARGETEXTERNALVIEW/{resourceName}");
@@ -269,7 +276,7 @@ public class PropertyPathBuilder {
public static String externalView(String clusterName, String resourceName) {
return String.format("/%s/EXTERNALVIEW/%s", clusterName, resourceName);
}
-
+
public static String targetExternalView(String clusterName) {
return String.format("/%s/TARGETEXTERNALVIEW", clusterName);
}
diff --git a/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java b/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java
index a718872..d46be64 100644
--- a/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java
+++ b/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java
@@ -47,19 +47,19 @@ public class CustomizedViewCache extends AbstractDataCache<CustomizedView> {
protected Map<String, CustomizedView> _customizedViewCache;
protected String _clusterName;
private PropertyType _propertyType;
- private String _aggregationType;
+ private String _customizedStateType;
- public CustomizedViewCache(String clusterName, String aggregationType) {
- this(clusterName, PropertyType.CUSTOMIZEDVIEW, aggregationType);
+ public CustomizedViewCache(String clusterName, String customizedStateType) {
+ this(clusterName, PropertyType.CUSTOMIZEDVIEW, customizedStateType);
}
- protected CustomizedViewCache(String clusterName, PropertyType propertyType, String aggregationType) {
+ protected CustomizedViewCache(String clusterName, PropertyType propertyType, String customizedStateType) {
super(createDefaultControlContextProvider(clusterName));
_clusterName = clusterName;
_customizedViewMap = Collections.emptyMap();
_customizedViewCache = Collections.emptyMap();
_propertyType = propertyType;
- _aggregationType = aggregationType;
+ _customizedStateType = customizedStateType;
}
@@ -105,14 +105,14 @@ public class CustomizedViewCache extends AbstractDataCache<CustomizedView> {
_customizedViewMap = new HashMap<>(newCustomizedViewMap);
long endTime = System.currentTimeMillis();
- LOG.info("Refresh " + _customizedViewMap.size() + " CustomizedViews of type " + _aggregationType
+ LOG.info("Refresh " + _customizedViewMap.size() + " CustomizedViews of type " + _customizedStateType
+ " for cluster " + _clusterName + ", took " + (endTime - startTime) + " ms");
}
private PropertyKey customizedViewsKey(PropertyKey.Builder keyBuilder) {
PropertyKey customizedViewPropertyKey;
if (_propertyType.equals(PropertyType.CUSTOMIZEDVIEW)){
- customizedViewPropertyKey = keyBuilder.customizedView(_aggregationType);
+ customizedViewPropertyKey = keyBuilder.customizedView(_customizedStateType);
} else {
throw new HelixException(
"Failed to refresh CustomizedViewCache, Wrong property type " + _propertyType + "!");
@@ -123,7 +123,7 @@ public class CustomizedViewCache extends AbstractDataCache<CustomizedView> {
private PropertyKey customizedViewKey(PropertyKey.Builder keyBuilder, String resource) {
PropertyKey customizedViewPropertyKey;
if (_propertyType.equals(PropertyType.CUSTOMIZEDVIEW)) {
- customizedViewPropertyKey = keyBuilder.customizedView(_aggregationType, resource);
+ customizedViewPropertyKey = keyBuilder.customizedView(_customizedStateType, resource);
} else {
throw new HelixException(
"Failed to refresh CustomizedViewCache, Wrong property type " + _propertyType + "!");
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
index b731635..72928ef 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
@@ -601,9 +601,9 @@ public class ZKHelixManager implements HelixManager, IZkStateListener {
}
@Override
- public void addCustomizedViewChangeListener(CustomizedViewChangeListener listener, String aggregationType)
+ public void addCustomizedViewChangeListener(CustomizedViewChangeListener listener, String customizedStateType)
throws Exception {
- addListener(listener, new Builder(_clusterName).customizedView(aggregationType),
+ addListener(listener, new Builder(_clusterName).customizedView(customizedStateType),
ChangeType.CUSTOMIZED_VIEW, new EventType[] {
EventType.NodeChildrenChanged
});
diff --git a/helix-core/src/main/java/org/apache/helix/spectator/CustomizedViewRoutingTable.java b/helix-core/src/main/java/org/apache/helix/spectator/CustomizedViewRoutingTable.java
new file mode 100644
index 0000000..5f986ce
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/spectator/CustomizedViewRoutingTable.java
@@ -0,0 +1,112 @@
+package org.apache.helix.spectator;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import org.apache.helix.PropertyType;
+import org.apache.helix.model.CustomizedView;
+import org.apache.helix.model.ExternalView;
+import org.apache.helix.model.InstanceConfig;
+import org.apache.helix.model.LiveInstance;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CustomizedViewRoutingTable extends RoutingTable {
+ private static final Logger logger = LoggerFactory.getLogger(CustomizedViewRoutingTable.class);
+
+ private final Collection<CustomizedView> _customizedViews;
+ /*
+ * The customizedStateType field is the type the controller is aggregating.
+ * For example if RoutingTableProvider initialized using the code below:
+ * Map<PropertyType, List<String>> sourceDataTypes = new HashMap<>();
+ * sourceDataTypes.put(PropertyType.CUSTOMIZEDVIEW, Arrays.asList("typeA", "typeB"));
+ * RoutingTableProvider routingTableProvider =
+ * new RoutingTableProvider(_spectator, sourceDataTypes);
+ * Each one of the TypeA and TypeB is a customizedStateType.
+ */
+ private final String _customizedStateType;
+
+ public CustomizedViewRoutingTable(PropertyType propertyType, String customizedStateType) {
+ this(Collections.<CustomizedView> emptyList(), propertyType, customizedStateType);
+ }
+
+ protected CustomizedViewRoutingTable(Collection<CustomizedView> customizedViews,
+ PropertyType propertytype, String customizedStateType) {
+ this(customizedViews, Collections.<InstanceConfig> emptyList(),
+ Collections.<LiveInstance> emptyList(), propertytype, customizedStateType);
+ }
+
+ protected CustomizedViewRoutingTable(Collection<CustomizedView> customizedViews,
+ Collection<InstanceConfig> instanceConfigs, Collection<LiveInstance> liveInstances,
+ PropertyType propertytype, String customizedStateType) {
+ super(Collections.<ExternalView> emptyList(), instanceConfigs, liveInstances,
+ PropertyType.CUSTOMIZEDVIEW);
+ _customizedStateType = customizedStateType;
+ _customizedViews = new HashSet<>(customizedViews);
+ refresh(_customizedViews);
+ }
+
+ private void refresh(Collection<CustomizedView> customizedViewList) {
+ Map<String, InstanceConfig> instanceConfigMap = new HashMap<>();
+ if (customizedViewList != null && !customizedViewList.isEmpty()) {
+ for (InstanceConfig config : _instanceConfigs) {
+ instanceConfigMap.put(config.getId(), config);
+ }
+ for (CustomizedView customizeView : customizedViewList) {
+ String resourceName = customizeView.getId();
+ for (String partitionName : customizeView.getPartitionSet()) {
+ Map<String, String> stateMap = customizeView.getStateMap(partitionName);
+ for (String instanceName : stateMap.keySet()) {
+ String customizedState = stateMap.get(instanceName);
+ if (instanceConfigMap.containsKey(instanceName)) {
+ InstanceConfig instanceConfig = instanceConfigMap.get(instanceName);
+ addEntry(resourceName, partitionName, customizedState, instanceConfig);
+ } else {
+ logger.warn(
+ "Participant {} is not found with proper configuration information. It might already be removed from the cluster. "
+ + "Skip recording partition assignment entry: Partition {}, Participant {}, State {}.",
+ instanceName, partitionName, instanceName, stateMap.get(instanceName));
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Returns CustomizedView.
+ * @return a collection of CustomizedView
+ */
+ protected Collection<CustomizedView> geCustomizedViews() {
+ return Collections.unmodifiableCollection(_customizedViews);
+ }
+
+ /**
+ * Returns CustomizedStateType
+ * @return the CustomizedStateType of this RoutingTable (Used for CustomizedView)
+ */
+ protected String getStateType() {
+ return _customizedStateType;
+ }
+}
diff --git a/helix-core/src/main/java/org/apache/helix/spectator/RoutingDataCache.java b/helix-core/src/main/java/org/apache/helix/spectator/RoutingDataCache.java
index 4896455..406080e 100644
--- a/helix-core/src/main/java/org/apache/helix/spectator/RoutingDataCache.java
+++ b/helix-core/src/main/java/org/apache/helix/spectator/RoutingDataCache.java
@@ -19,16 +19,23 @@ package org.apache.helix.spectator;
* under the License.
*/
+import com.google.common.collect.ImmutableMap;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.apache.helix.HelixConstants;
import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.HelixException;
import org.apache.helix.PropertyType;
import org.apache.helix.common.caches.BasicClusterDataCache;
import org.apache.helix.common.caches.CurrentStateCache;
import org.apache.helix.common.caches.CurrentStateSnapshot;
+import org.apache.helix.common.caches.CustomizedViewCache;
import org.apache.helix.common.caches.TargetExternalViewCache;
import org.apache.helix.model.CurrentState;
+import org.apache.helix.model.CustomizedView;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.LiveInstance;
import org.slf4j.Logger;
@@ -40,14 +47,31 @@ import org.slf4j.LoggerFactory;
class RoutingDataCache extends BasicClusterDataCache {
private static Logger LOG = LoggerFactory.getLogger(RoutingDataCache.class.getName());
- private final PropertyType _sourceDataType;
+ private final Map<PropertyType, List<String>> _sourceDataTypeMap;
+
private CurrentStateCache _currentStateCache;
+ // TODO: CustomizedCache needs to be migrated to propertyCache. Once we migrate all cache to
+ // propertyCache, this hardcoded list of fields won't be necessary.
+ private Map<String, CustomizedViewCache> _customizedViewCaches;
private TargetExternalViewCache _targetExternalViewCache;
public RoutingDataCache(String clusterName, PropertyType sourceDataType) {
+ this (clusterName, ImmutableMap.of(sourceDataType, Collections.emptyList()));
+ }
+
+ /**
+ * Initialize empty RoutingDataCache with clusterName, _propertyTypes.
+ * @param clusterName
+ * @param sourceDataTypeMap
+ */
+ public RoutingDataCache(String clusterName, Map<PropertyType, List<String>> sourceDataTypeMap) {
super(clusterName);
- _sourceDataType = sourceDataType;
+ _sourceDataTypeMap = sourceDataTypeMap;
_currentStateCache = new CurrentStateCache(clusterName);
+ _customizedViewCaches = new HashMap<>();
+ sourceDataTypeMap.getOrDefault(PropertyType.CUSTOMIZEDVIEW, Collections.emptyList())
+ .forEach(customizedStateType -> _customizedViewCaches.put(customizedStateType,
+ new CustomizedViewCache(clusterName, customizedStateType)));
_targetExternalViewCache = new TargetExternalViewCache(clusterName);
requireFullRefresh();
}
@@ -65,36 +89,52 @@ class RoutingDataCache extends BasicClusterDataCache {
long startTime = System.currentTimeMillis();
super.refresh(accessor);
-
- if (_sourceDataType.equals(PropertyType.TARGETEXTERNALVIEW) && _propertyDataChangedMap
- .get(HelixConstants.ChangeType.TARGET_EXTERNAL_VIEW)) {
- long start = System.currentTimeMillis();
- _propertyDataChangedMap.put(HelixConstants.ChangeType.TARGET_EXTERNAL_VIEW, false);
- _targetExternalViewCache.refresh(accessor);
- LOG.info("Reload " + _targetExternalViewCache.getExternalViewMap().keySet().size()
- + " TargetExternalViews. Takes " + (System.currentTimeMillis() - start) + " ms");
- }
-
- if (_sourceDataType.equals(PropertyType.CURRENTSTATES) && _propertyDataChangedMap
- .get(HelixConstants.ChangeType.CURRENT_STATE)) {
+ for (PropertyType propertyType : _sourceDataTypeMap.keySet()) {
long start = System.currentTimeMillis();
- _propertyDataChangedMap.put(HelixConstants.ChangeType.CURRENT_STATE, false);
- /**
- * Workaround of https://github.com/apache/helix/issues/919.
- * Why it is workaround?
- * 1. Before a larger scale refactoring, to minimize the impact on cache logic, this change
- * introduces extra read to update the liveInstance list before processing current states.
- * 2. This change does not handle the corresponding callback handlers, which should also be
- * registered when a new liveInstance node is found.
- * TODO: Refactor cache processing logic and also refine the callback handler registration
- * TODO: logic.
- **/
- _liveInstancePropertyCache.refresh(accessor);
- Map<String, LiveInstance> liveInstanceMap = getLiveInstances();
- _currentStateCache.refresh(accessor, liveInstanceMap);
- LOG.info("Reload CurrentStates. Takes " + (System.currentTimeMillis() - start) + " ms");
+ switch (propertyType) {
+ case TARGETEXTERNALVIEW:
+ if (_propertyDataChangedMap.get(HelixConstants.ChangeType.TARGET_EXTERNAL_VIEW)) {
+ _propertyDataChangedMap.put(HelixConstants.ChangeType.TARGET_EXTERNAL_VIEW, false);
+ _targetExternalViewCache.refresh(accessor);
+ LOG.info("Reload " + _targetExternalViewCache.getExternalViewMap().keySet().size()
+ + " TargetExternalViews. Takes " + (System.currentTimeMillis() - start) + " ms");
+ }
+ break;
+ case CURRENTSTATES:
+ if (_propertyDataChangedMap.get(HelixConstants.ChangeType.CURRENT_STATE)) {
+ _propertyDataChangedMap.put(HelixConstants.ChangeType.CURRENT_STATE, false);
+ /**
+ * Workaround of https://github.com/apache/helix/issues/919.
+ * Why it is workaround?
+ * 1. Before a larger scale refactoring, to minimize the impact on cache logic, this change
+ * introduces extra read to update the liveInstance list before processing current states.
+ * 2. This change does not handle the corresponding callback handlers, which should also be
+ * registered when a new liveInstance node is found.
+ * TODO: Refactor cache processing logic and also refine the callback handler registration
+ * TODO: logic.
+ **/
+ _liveInstancePropertyCache.refresh(accessor);
+ Map<String, LiveInstance> liveInstanceMap = getLiveInstances();
+ _currentStateCache.refresh(accessor, liveInstanceMap);
+ LOG.info("Reload CurrentStates. Takes " + (System.currentTimeMillis() - start) + " ms");
+ }
+ break;
+ case CUSTOMIZEDVIEW: {
+ if (_propertyDataChangedMap.get(HelixConstants.ChangeType.CUSTOMIZED_VIEW)) {
+ for (String customizedStateType : _sourceDataTypeMap.get(PropertyType.CUSTOMIZEDVIEW)) {
+ _customizedViewCaches.get(customizedStateType).refresh(accessor);
+ }
+ LOG.info("Reload CustomizedView for types "
+ + _sourceDataTypeMap.get(PropertyType.CUSTOMIZEDVIEW) + " Takes "
+ + (System.currentTimeMillis() - start) + " ms");
+ }
+ _propertyDataChangedMap.put(HelixConstants.ChangeType.CUSTOMIZED_VIEW, false);
+ }
+ break;
+ default:
+ break;
+ }
}
-
long endTime = System.currentTimeMillis();
LOG.info("END: RoutingDataCache.refresh() for cluster " + _clusterName + ", took " + (endTime
- startTime) + " ms");
@@ -102,6 +142,10 @@ class RoutingDataCache extends BasicClusterDataCache {
if (LOG.isDebugEnabled()) {
LOG.debug("CurrentStates: " + _currentStateCache);
LOG.debug("TargetExternalViews: " + _targetExternalViewCache.getExternalViewMap());
+ for (String customizedStateType : _sourceDataTypeMap.get(PropertyType.CUSTOMIZEDVIEW)) {
+ LOG.debug("CustomizedViews customizedStateType: " + customizedStateType + " "
+ + _customizedViewCaches.get(customizedStateType).getCustomizedViewMap());
+ }
}
}
@@ -115,6 +159,18 @@ class RoutingDataCache extends BasicClusterDataCache {
}
/**
+ * Retrieves the CustomizedView for all resources
+ * @return
+ */
+ public Map<String, CustomizedView> getCustomizedView(String customizedStateType) {
+ if (_customizedViewCaches.containsKey(customizedStateType)) {
+ return _customizedViewCaches.get(customizedStateType).getCustomizedViewMap();
+ }
+ throw new HelixException(String.format(
+ "customizedStateType %s does not exist in customizedViewCaches.", customizedStateType));
+ }
+
+ /**
* Get map of current states in cluster. {InstanceName -> {SessionId -> {ResourceName ->
* CurrentState}}}
*
diff --git a/helix-core/src/main/java/org/apache/helix/spectator/RoutingTable.java b/helix-core/src/main/java/org/apache/helix/spectator/RoutingTable.java
index dd1b623..89ad16f 100644
--- a/helix-core/src/main/java/org/apache/helix/spectator/RoutingTable.java
+++ b/helix-core/src/main/java/org/apache/helix/spectator/RoutingTable.java
@@ -30,6 +30,7 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
+import org.apache.helix.PropertyType;
import org.apache.helix.model.CurrentState;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.InstanceConfig;
@@ -38,42 +39,64 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * A class to consume ExternalViews of a cluster and provide {resource, partition, state} to
- * {instances} map function.
+ * A class to consume ExternalViews or CustomizedViews of a cluster and provide
+ * {resource, partition, state} to {instances} map function.
*/
class RoutingTable {
private static final Logger logger = LoggerFactory.getLogger(RoutingTable.class);
// mapping a resourceName to the ResourceInfo
private final Map<String, ResourceInfo> _resourceInfoMap;
+
// mapping a resource group name to a resourceGroupInfo
private final Map<String, ResourceGroupInfo> _resourceGroupInfoMap;
private final Collection<LiveInstance> _liveInstances;
- private final Collection<InstanceConfig> _instanceConfigs;
+ protected final Collection<InstanceConfig> _instanceConfigs;
private final Collection<ExternalView> _externalViews;
+ private final PropertyType _propertyType;
+
+ @Deprecated
public RoutingTable() {
this(Collections.<ExternalView> emptyList(), Collections.<InstanceConfig> emptyList(),
Collections.<LiveInstance> emptyList());
}
+ /**
+ * Initialize empty RoutingTable and set _propertyType fields.
+ * @param propertyType
+ */
+ protected RoutingTable(PropertyType propertyType) {
+ this(Collections.<ExternalView> emptyList(), Collections.<InstanceConfig> emptyList(),
+ Collections.<LiveInstance> emptyList(), propertyType);
+ }
+
public RoutingTable(Map<String, Map<String, Map<String, CurrentState>>> currentStateMap,
Collection<InstanceConfig> instanceConfigs, Collection<LiveInstance> liveInstances) {
// TODO Aggregate currentState to an ExternalView in the RoutingTable, so there is no need to
// refresh according to the currentStateMap. - jjwang
- this(Collections.<ExternalView> emptyList(), instanceConfigs, liveInstances);
+ this(Collections.<ExternalView> emptyList(),
+ instanceConfigs, liveInstances, PropertyType.CURRENTSTATES);
refresh(currentStateMap);
}
public RoutingTable(Collection<ExternalView> externalViews,
Collection<InstanceConfig> instanceConfigs, Collection<LiveInstance> liveInstances) {
+ this(externalViews, instanceConfigs, liveInstances,
+ PropertyType.EXTERNALVIEW);
+ }
+
+ protected RoutingTable(Collection<ExternalView> externalViews, Collection<InstanceConfig> instanceConfigs,
+ Collection<LiveInstance> liveInstances, PropertyType propertytype) {
+ // TODO Refactor these constructors so we don't have so many constructor.
+ _propertyType = propertytype;
_resourceInfoMap = new HashMap<>();
_resourceGroupInfoMap = new HashMap<>();
_liveInstances = new HashSet<>(liveInstances);
_instanceConfigs = new HashSet<>(instanceConfigs);
_externalViews = new HashSet<>(externalViews);
- refresh(externalViews);
+ refresh(_externalViews);
}
private void refresh(Collection<ExternalView> externalViewList) {
@@ -145,7 +168,7 @@ class RoutingTable {
}
}
- private void addEntry(String resourceName, String partitionName, String state,
+ protected void addEntry(String resourceName, String partitionName, String state,
InstanceConfig config) {
if (!_resourceInfoMap.containsKey(resourceName)) {
_resourceInfoMap.put(resourceName, new ResourceInfo());
@@ -352,6 +375,23 @@ class RoutingTable {
}
/**
+ * Returns PropertyTYpe
+ * @return the PropertyTYpe of this RoutingTable
+ */
+ protected PropertyType getPropertyType() {
+ return _propertyType;
+ }
+
+ /**
+ * Returns CustomizedStateType
+ * @return the CustomizedStateType of this RoutingTable (Used for CustomizedView)
+ */
+ protected String getStateType() {
+ return RoutingTableProvider.DEFAULT_STATE_TYPE;
+ }
+
+
+ /**
* Class to store instances, partitions and their states for each resource.
*/
class ResourceInfo {
diff --git a/helix-core/src/main/java/org/apache/helix/spectator/RoutingTableProvider.java b/helix-core/src/main/java/org/apache/helix/spectator/RoutingTableProvider.java
index 1a8f641..1e11965 100644
--- a/helix-core/src/main/java/org/apache/helix/spectator/RoutingTableProvider.java
+++ b/helix-core/src/main/java/org/apache/helix/spectator/RoutingTableProvider.java
@@ -33,6 +33,7 @@ import java.util.concurrent.Future;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
+import com.google.common.collect.ImmutableMap;
import javax.management.JMException;
import org.apache.helix.HelixConstants;
@@ -44,6 +45,7 @@ import org.apache.helix.PropertyKey;
import org.apache.helix.PropertyType;
import org.apache.helix.api.listeners.ConfigChangeListener;
import org.apache.helix.api.listeners.CurrentStateChangeListener;
+import org.apache.helix.api.listeners.CustomizedViewChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.InstanceConfigChangeListener;
import org.apache.helix.api.listeners.LiveInstanceChangeListener;
@@ -55,6 +57,7 @@ import org.apache.helix.controller.stages.AttributeName;
import org.apache.helix.controller.stages.ClusterEvent;
import org.apache.helix.controller.stages.ClusterEventType;
import org.apache.helix.model.CurrentState;
+import org.apache.helix.model.CustomizedView;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.InstanceConfig;
import org.apache.helix.model.LiveInstance;
@@ -64,15 +67,15 @@ import org.slf4j.LoggerFactory;
public class RoutingTableProvider
implements ExternalViewChangeListener, InstanceConfigChangeListener, ConfigChangeListener,
- LiveInstanceChangeListener, CurrentStateChangeListener {
+ LiveInstanceChangeListener, CurrentStateChangeListener, CustomizedViewChangeListener {
private static final Logger logger = LoggerFactory.getLogger(RoutingTableProvider.class);
private static final long DEFAULT_PERIODIC_REFRESH_INTERVAL = 300000L; // 5 minutes
- private final AtomicReference<RoutingTable> _routingTableRef;
+ private final Map<String, AtomicReference<RoutingTable>> _routingTableRefMap;
private final HelixManager _helixManager;
private final RouterUpdater _routerUpdater;
- private final PropertyType _sourceDataType;
+ private final Map<PropertyType, List<String>> _sourceDataTypeMap;
private final Map<RoutingTableChangeListener, ListenerContext> _routingTableChangeListenerMap;
- private final RoutingTableProviderMonitor _monitor;
+ private final Map<PropertyType, RoutingTableProviderMonitor> _monitorMap;
// For periodic refresh
private long _lastRefreshTimestamp;
@@ -83,17 +86,27 @@ public class RoutingTableProvider
private ExecutorService _reportExecutor;
private Future _reportingTask = null;
+ protected static final String DEFAULT_PROPERTY_TYPE = "HELIX_DEFAULT_PROPERTY";
+ protected static final String DEFAULT_STATE_TYPE = "HELIX_DEFAULT";
+
+
public RoutingTableProvider() {
this(null);
}
public RoutingTableProvider(HelixManager helixManager) throws HelixException {
- this(helixManager, PropertyType.EXTERNALVIEW, true, DEFAULT_PERIODIC_REFRESH_INTERVAL);
+ this(helixManager, ImmutableMap.of(PropertyType.EXTERNALVIEW, Collections.emptyList()), true,
+ DEFAULT_PERIODIC_REFRESH_INTERVAL);
}
public RoutingTableProvider(HelixManager helixManager, PropertyType sourceDataType)
throws HelixException {
- this(helixManager, sourceDataType, true, DEFAULT_PERIODIC_REFRESH_INTERVAL);
+ this(helixManager, ImmutableMap.of(sourceDataType, Collections.emptyList()), true,
+ DEFAULT_PERIODIC_REFRESH_INTERVAL);
+ }
+
+ public RoutingTableProvider(HelixManager helixManager, Map<PropertyType, List<String>> sourceDataTypeMap) {
+ this(helixManager, sourceDataTypeMap, true, DEFAULT_PERIODIC_REFRESH_INTERVAL);
}
/**
@@ -106,73 +119,75 @@ public class RoutingTableProvider
*/
public RoutingTableProvider(HelixManager helixManager, PropertyType sourceDataType,
boolean isPeriodicRefreshEnabled, long periodRefreshInterval) throws HelixException {
- _routingTableRef = new AtomicReference<>(new RoutingTable());
+ this(helixManager, ImmutableMap.of(sourceDataType, Collections.emptyList()),
+ isPeriodicRefreshEnabled, periodRefreshInterval);
+ }
+
+ /**
+ * Initialize an instance of RoutingTableProvider
+ * @param helixManager
+ * @param sourceDataTypeMap
+ * @param isPeriodicRefreshEnabled true if periodic refresh is enabled, false otherwise
+ * @param periodRefreshInterval only effective if isPeriodRefreshEnabled is true
+ * @throws HelixException
+ */
+ public RoutingTableProvider(HelixManager helixManager,
+ Map<PropertyType, List<String>> sourceDataTypeMap, boolean isPeriodicRefreshEnabled,
+ long periodRefreshInterval) throws HelixException {
+
+ validateSourceDataTypeMap(sourceDataTypeMap);
+
+ _routingTableRefMap = new HashMap<>();
_helixManager = helixManager;
- _sourceDataType = sourceDataType;
+ _sourceDataTypeMap = sourceDataTypeMap;
_routingTableChangeListenerMap = new ConcurrentHashMap<>();
String clusterName = _helixManager != null ? _helixManager.getClusterName() : null;
- _monitor = new RoutingTableProviderMonitor(_sourceDataType, clusterName);
- try {
- _monitor.register();
- } catch (JMException e) {
- logger.error("Failed to register RoutingTableProvider monitor MBean.", e);
- }
- _reportExecutor = Executors.newSingleThreadExecutor();
-
- _routerUpdater = new RouterUpdater(clusterName, _sourceDataType);
- _routerUpdater.start();
-
- if (_helixManager != null) {
- switch (_sourceDataType) {
- case EXTERNALVIEW:
- try {
- _helixManager.addExternalViewChangeListener(this);
- } catch (Exception e) {
- shutdown();
- logger.error("Failed to attach ExternalView Listener to HelixManager!");
- throw new HelixException("Failed to attach ExternalView Listener to HelixManager!", e);
+ // Initialize the tables
+ for (PropertyType propertyType : _sourceDataTypeMap.keySet()) {
+ if (_sourceDataTypeMap.get(propertyType).size() == 0) {
+ if (propertyType.equals(PropertyType.CUSTOMIZEDVIEW)) {
+ throw new HelixException("CustomizedView has been used without any aggregation type!");
}
- break;
-
- case TARGETEXTERNALVIEW:
- // Check whether target external has been enabled or not
- if (!_helixManager.getHelixDataAccessor().getBaseDataAccessor().exists(
- _helixManager.getHelixDataAccessor().keyBuilder().targetExternalViews().getPath(), 0)) {
- shutdown();
- throw new HelixException("Target External View is not enabled!");
+ String key = generateReferenceKey(propertyType.name(), DEFAULT_STATE_TYPE);
+ if (_routingTableRefMap.get(key) == null) {
+ _routingTableRefMap.put(key, new AtomicReference<>(new RoutingTable(propertyType)));
}
-
- try {
- _helixManager.addTargetExternalViewChangeListener(this);
- } catch (Exception e) {
- shutdown();
- logger.error("Failed to attach TargetExternalView Listener to HelixManager!");
- throw new HelixException("Failed to attach TargetExternalView Listener to HelixManager!",
- e);
+ } else {
+ if (!propertyType.equals(PropertyType.CUSTOMIZEDVIEW)) {
+ throw new HelixException(
+ String.format("Type %s has been used in addition to the propertyType %s !",
+ sourceDataTypeMap.get(propertyType), propertyType.name()));
+ }
+ for (String customizedStateType : _sourceDataTypeMap.get(propertyType)) {
+ String key = generateReferenceKey(propertyType.name(), customizedStateType);
+ if (_routingTableRefMap.get(key) == null) {
+ _routingTableRefMap.put(key, new AtomicReference<>(
+ new CustomizedViewRoutingTable(propertyType, customizedStateType)));
+ }
}
- break;
-
- case CURRENTSTATES:
- // CurrentState change listeners will be added later in LiveInstanceChange call.
- break;
-
- default:
- throw new HelixException(String.format("Unsupported source data type: %s", sourceDataType));
}
+ }
+
+ // Start Monitoring
+ _monitorMap = new HashMap<>();
+ for (PropertyType propertyType : _sourceDataTypeMap.keySet()) {
+ _monitorMap.put(propertyType, new RoutingTableProviderMonitor(propertyType, clusterName));
try {
- _helixManager.addInstanceConfigChangeListener(this);
- _helixManager.addLiveInstanceChangeListener(this);
- } catch (Exception e) {
- shutdown();
- logger.error(
- "Failed to attach InstanceConfig and LiveInstance Change listeners to HelixManager!");
- throw new HelixException(
- "Failed to attach InstanceConfig and LiveInstance Change listeners to HelixManager!",
- e);
+ _monitorMap.get(propertyType).register();
+ } catch (JMException e) {
+ logger.error("Failed to register RoutingTableProvider monitor MBean.", e);
}
}
+ _reportExecutor = Executors.newSingleThreadExecutor();
+
+ // Start Updaters
+ _routerUpdater = new RouterUpdater(clusterName, sourceDataTypeMap);
+ _routerUpdater.start();
+
+ // Add listeners
+ addListeners();
// For periodic refresh
if (isPeriodicRefreshEnabled && _helixManager != null) {
@@ -200,6 +215,92 @@ public class RoutingTableProvider
}
/**
+ * A method that adds the ChangeListeners to HelixManager
+ */
+ private void addListeners() {
+ if (_helixManager != null) {
+ for (PropertyType propertyType : _sourceDataTypeMap.keySet()) {
+ switch (propertyType) {
+ case EXTERNALVIEW:
+ try {
+ _helixManager.addExternalViewChangeListener(this);
+ } catch (Exception e) {
+ shutdown();
+ throw new HelixException("Failed to attach ExternalView Listener to HelixManager!", e);
+ }
+ break;
+ case CUSTOMIZEDVIEW:
+ List<String> customizedStateTypes = _sourceDataTypeMap.get(propertyType);
+ for (String customizedStateType : customizedStateTypes) {
+ try {
+ _helixManager.addCustomizedViewChangeListener(this, customizedStateType);
+ } catch (Exception e) {
+ shutdown();
+ throw new HelixException(String.format(
+ "Failed to attach CustomizedView Listener to HelixManager for type %s!",
+ customizedStateType), e);
+ }
+ }
+ break;
+ case TARGETEXTERNALVIEW:
+ // Check whether target external has been enabled or not
+ if (!_helixManager.getHelixDataAccessor().getBaseDataAccessor().exists(
+ _helixManager.getHelixDataAccessor().keyBuilder().targetExternalViews().getPath(),
+ 0)) {
+ shutdown();
+ throw new HelixException("Target External View is not enabled!");
+ }
+
+ try {
+ _helixManager.addTargetExternalViewChangeListener(this);
+ } catch (Exception e) {
+ shutdown();
+ throw new HelixException(
+ "Failed to attach TargetExternalView Listener to HelixManager!", e);
+ }
+ break;
+ case CURRENTSTATES:
+ // CurrentState change listeners will be added later in LiveInstanceChange call.
+ break;
+ default:
+ throw new HelixException(String.format("Unsupported source data type: %s", propertyType));
+ }
+ }
+ try {
+ _helixManager.addInstanceConfigChangeListener(this);
+ _helixManager.addLiveInstanceChangeListener(this);
+ } catch (Exception e) {
+ shutdown();
+ throw new HelixException(
+ "Failed to attach InstanceConfig and LiveInstance Change listeners to HelixManager!",
+ e);
+ }
+ }
+ }
+
+ /**
+ * Check and validate the input of the sourceDataTypeMap parameter
+ * @param sourceDataTypeMap
+ */
+ private void validateSourceDataTypeMap(Map<PropertyType, List<String>> sourceDataTypeMap) {
+ for (PropertyType propertyType : sourceDataTypeMap.keySet()) {
+ if (propertyType.equals(PropertyType.CUSTOMIZEDVIEW)
+ && sourceDataTypeMap.get(propertyType).size() == 0) {
+ logger.error("CustomizedView has been used without any aggregation type!");
+ throw new HelixException("CustomizedView has been used without any aggregation type!");
+ }
+ if (!propertyType.equals(PropertyType.CUSTOMIZEDVIEW)
+ && sourceDataTypeMap.get(propertyType).size() != 0) {
+ logger.error("Type has been used in addition to the propertyType {} !",
+ propertyType.name());
+ throw new HelixException(
+ String.format("Type %s has been used in addition to the propertyType %s !",
+ sourceDataTypeMap.get(propertyType), propertyType.name()));
+ }
+ }
+ }
+
+ /**
* Shutdown current RoutingTableProvider. Once it is shutdown, it should never be reused.
*/
public void shutdown() {
@@ -209,24 +310,36 @@ public class RoutingTableProvider
}
_routerUpdater.shutdown();
- _monitor.unregister();
+
+ for (PropertyType propertyType : _monitorMap.keySet()) {
+ _monitorMap.get(propertyType).unregister();
+ }
if (_helixManager != null) {
PropertyKey.Builder keyBuilder = _helixManager.getHelixDataAccessor().keyBuilder();
- switch (_sourceDataType) {
- case EXTERNALVIEW:
- _helixManager.removeListener(keyBuilder.externalViews(), this);
- break;
- case TARGETEXTERNALVIEW:
- _helixManager.removeListener(keyBuilder.targetExternalViews(), this);
- break;
- case CURRENTSTATES:
- NotificationContext context = new NotificationContext(_helixManager);
- context.setType(NotificationContext.Type.FINALIZE);
- updateCurrentStatesListeners(Collections.<LiveInstance> emptyList(), context);
- break;
- default:
- break;
+ for (PropertyType propertyType : _sourceDataTypeMap.keySet()) {
+ switch (propertyType) {
+ case EXTERNALVIEW:
+ _helixManager.removeListener(keyBuilder.externalViews(), this);
+ break;
+ case CUSTOMIZEDVIEW:
+ List<String> customizedStateTypes = _sourceDataTypeMap.get(propertyType);
+ // Remove listener on each individual customizedStateType
+ for (String customizedStateType : customizedStateTypes) {
+ _helixManager.removeListener(keyBuilder.customizedView(customizedStateType), this);
+ }
+ break;
+ case TARGETEXTERNALVIEW:
+ _helixManager.removeListener(keyBuilder.targetExternalViews(), this);
+ break;
+ case CURRENTSTATES:
+ NotificationContext context = new NotificationContext(_helixManager);
+ context.setType(NotificationContext.Type.FINALIZE);
+ updateCurrentStatesListeners(Collections.<LiveInstance> emptyList(), context);
+ break;
+ default:
+ break;
+ }
}
}
}
@@ -237,7 +350,46 @@ public class RoutingTableProvider
* @return snapshot of current routing table.
*/
public RoutingTableSnapshot getRoutingTableSnapshot() {
- return new RoutingTableSnapshot(_routingTableRef.get());
+ return new RoutingTableSnapshot(getRoutingTableRef(DEFAULT_PROPERTY_TYPE, DEFAULT_STATE_TYPE));
+ }
+
+ /**
+ * Get an snapshot of current RoutingTable information for specific PropertyType.
+ * The snapshot is immutable, it reflects the routing table information at the time this method is
+ * called.
+ * @return snapshot of current routing table.
+ */
+ public RoutingTableSnapshot getRoutingTableSnapshot(PropertyType propertyType) {
+ return new RoutingTableSnapshot(getRoutingTableRef(propertyType.name(), DEFAULT_STATE_TYPE));
+ }
+
+ /**
+ * Get an snapshot of all of the available RoutingTable information. The snapshot is immutable, it
+ * reflects the routing table information at the time this method is called.
+ * @return snapshot associated with specific propertyType and type.
+ */
+ public RoutingTableSnapshot getRoutingTableSnapshot(PropertyType propertyType, String stateType) {
+ return new RoutingTableSnapshot(getRoutingTableRef(propertyType.name(), stateType));
+ }
+
+ /**
+ * Get an snapshot of all of the available RoutingTable information. The snapshot is immutable, it
+ * reflects the routing table information at the time this method is called.
+ * @return all of the available snapshots of current routing table.
+ */
+ public Map<String, Map<String, RoutingTableSnapshot>> getRoutingTableSnapshots() {
+ Map<String, Map<String, RoutingTableSnapshot>> snapshots = new HashMap<>();
+ for (String key : _routingTableRefMap.keySet()) {
+ RoutingTable routingTable = _routingTableRefMap.get(key).get();
+ String propertyTypeName = routingTable.getPropertyType().name();
+ String customizedStateType = routingTable.getStateType();
+ if (!snapshots.containsKey(propertyTypeName)) {
+ snapshots.put(propertyTypeName, new HashMap<>());
+ }
+ snapshots.get(propertyTypeName).put(customizedStateType,
+ new RoutingTableSnapshot(routingTable));
+ }
+ return snapshots;
}
/**
@@ -264,12 +416,10 @@ public class RoutingTableProvider
}
/**
- * returns the instances for {resource,partition} pair that are in a specific
- * {state}
+ * returns the instances for {resource,partition} pair that are in a specific {state}.
* This method will be deprecated, please use the
* {@link #getInstancesForResource(String, String, String)} getInstancesForResource} method.
* @param resourceName
- * -
* @param partitionName
* @param state
* @return empty list if there is no instance in a given state
@@ -280,17 +430,16 @@ public class RoutingTableProvider
}
/**
- * returns the instances for {resource,partition} pair that are in a specific
- * {state}
+ * returns the instances for {resource,partition} pair that are in a specific {state}
* @param resourceName
- * -
* @param partitionName
* @param state
* @return empty list if there is no instance in a given state
*/
public List<InstanceConfig> getInstancesForResource(String resourceName, String partitionName,
String state) {
- return _routingTableRef.get().getInstancesForResource(resourceName, partitionName, state);
+ return getRoutingTableRef(DEFAULT_PROPERTY_TYPE, DEFAULT_STATE_TYPE)
+ .getInstancesForResource(resourceName, partitionName, state);
}
/**
@@ -305,8 +454,8 @@ public class RoutingTableProvider
*/
public List<InstanceConfig> getInstancesForResourceGroup(String resourceGroupName,
String partitionName, String state) {
- return _routingTableRef.get().getInstancesForResourceGroup(resourceGroupName, partitionName,
- state);
+ return getRoutingTableRef(DEFAULT_PROPERTY_TYPE, DEFAULT_STATE_TYPE)
+ .getInstancesForResourceGroup(resourceGroupName, partitionName, state);
}
/**
@@ -322,11 +471,12 @@ public class RoutingTableProvider
*/
public List<InstanceConfig> getInstancesForResourceGroup(String resourceGroupName,
String partitionName, String state, List<String> resourceTags) {
- return _routingTableRef.get().getInstancesForResourceGroup(resourceGroupName, partitionName,
- state, resourceTags);
+ return getRoutingTableRef(DEFAULT_PROPERTY_TYPE, DEFAULT_STATE_TYPE)
+ .getInstancesForResourceGroup(resourceGroupName, partitionName, state, resourceTags);
}
/**
+ * For specific routing table associated with {propertyType, stateType}
* returns all instances for {resource} that are in a specific {state}
* This method will be deprecated, please use the
* {@link #getInstancesForResource(String, String) getInstancesForResource} method.
@@ -345,7 +495,8 @@ public class RoutingTableProvider
* @return empty list if there is no instance in a given state
*/
public Set<InstanceConfig> getInstancesForResource(String resourceName, String state) {
- return _routingTableRef.get().getInstancesForResource(resourceName, state);
+ return getRoutingTableRef(DEFAULT_PROPERTY_TYPE, DEFAULT_STATE_TYPE)
+ .getInstancesForResource(resourceName, state);
}
/**
@@ -355,7 +506,8 @@ public class RoutingTableProvider
* @return empty list if there is no instance in a given state
*/
public Set<InstanceConfig> getInstancesForResourceGroup(String resourceGroupName, String state) {
- return _routingTableRef.get().getInstancesForResourceGroup(resourceGroupName, state);
+ return getRoutingTableRef(DEFAULT_PROPERTY_TYPE, DEFAULT_STATE_TYPE)
+ .getInstancesForResourceGroup(resourceGroupName, state);
}
/**
@@ -367,8 +519,8 @@ public class RoutingTableProvider
*/
public Set<InstanceConfig> getInstancesForResourceGroup(String resourceGroupName, String state,
List<String> resourceTags) {
- return _routingTableRef.get().getInstancesForResourceGroup(resourceGroupName, state,
- resourceTags);
+ return getRoutingTableRef(DEFAULT_PROPERTY_TYPE, DEFAULT_STATE_TYPE)
+ .getInstancesForResourceGroup(resourceGroupName, state, resourceTags);
}
/**
@@ -376,7 +528,11 @@ public class RoutingTableProvider
* @return
*/
public Collection<LiveInstance> getLiveInstances() {
- return _routingTableRef.get().getLiveInstances();
+ // Since line instances will be the same across all _routingTableRefMap, here one of the keys
+ // will be used without considering PropertyType
+ // TODO each table will keep a separate instance list.This can be improve by only keeping one
+ // copy of the data
+ return _routingTableRefMap.values().iterator().next().get().getLiveInstances();
}
/**
@@ -384,14 +540,58 @@ public class RoutingTableProvider
* @return
*/
public Collection<InstanceConfig> getInstanceConfigs() {
- return _routingTableRef.get().getInstanceConfigs();
+ // Since line instances will be the same across all _routingTableRefMap, here one of the keys
+ // will be used without considering PropertyType
+ // TODO each table will keep a separate instance list.This can be improve by only keeping one copy of the data
+ return _routingTableRefMap.values().iterator().next().get().getInstanceConfigs();
}
/**
- * Return names of all resources (shown in ExternalView) in this cluster.
+ * Return names of all resources (shown in ExternalView or CustomizedView) in this cluster.
*/
public Collection<String> getResources() {
- return _routingTableRef.get().getResources();
+ return getRoutingTableRef(DEFAULT_PROPERTY_TYPE, DEFAULT_STATE_TYPE).getResources();
+ }
+
+ /**
+ * Provide the key associated with specific PropertyType and StateType for _routingTableRefMap lookup.
+ * @param propertyTypeName
+ * @param stateType
+ * @return
+ */
+ private RoutingTable getRoutingTableRef(String propertyTypeName, String stateType) {
+ if (propertyTypeName.equals(DEFAULT_PROPERTY_TYPE)) {
+ // Check whether there exist only one snapshot (_routingTableRefMap)
+ if (_routingTableRefMap.keySet().size() == 1) {
+ String key = _routingTableRefMap.keySet().iterator().next();
+ if (!_routingTableRefMap.containsKey(key)) {
+ throw new HelixException(
+ String.format("Currently there is no snapshot available for PropertyType %s and stateType %s",
+ propertyTypeName, stateType));
+ }
+ return _routingTableRefMap.get(key).get();
+ } else {
+ throw new HelixException("There is none or more than one RoutingTableSnapshot");
+ }
+ }
+
+ if (stateType.equals(DEFAULT_STATE_TYPE)) {
+ if (propertyTypeName.equals(PropertyType.CUSTOMIZEDVIEW.name())) {
+ throw new HelixException("Specific type needs to be used for CUSTOMIZEDVIEW PropertyType");
+ }
+ }
+
+ String key = generateReferenceKey(propertyTypeName, stateType);
+ if (!_routingTableRefMap.containsKey(key)) {
+ throw new HelixException(
+ String.format("Currently there is no snapshot available for PropertyType %s and stateType %s",
+ propertyTypeName, stateType));
+ }
+ return _routingTableRefMap.get(key).get();
+ }
+
+ private String generateReferenceKey(String propertyType, String stateType) {
+ return propertyType + "_" + stateType;
}
@Override
@@ -399,27 +599,32 @@ public class RoutingTableProvider
public void onExternalViewChange(List<ExternalView> externalViewList,
NotificationContext changeContext) {
HelixConstants.ChangeType changeType = changeContext.getChangeType();
- if (changeType != null && !changeType.getPropertyType().equals(_sourceDataType)) {
+ if (changeType != null && !_sourceDataTypeMap.containsKey(changeType.getPropertyType())) {
logger.warn(
- "onExternalViewChange called with mismatched change types. Source data type {}, changed data type: {}",
- _sourceDataType, changeType);
+ "onExternalViewChange called with mismatched change types. Source data types does not contain changed data type: {}",
+ changeType);
return;
}
// Refresh with full list of external view.
if (externalViewList != null && externalViewList.size() > 0) {
// keep this here for back-compatibility, application can call onExternalViewChange directly
// with externalview list supplied.
- refresh(externalViewList, changeContext);
+ String keyReference = generateReferenceKey(PropertyType.EXTERNALVIEW.name(), DEFAULT_STATE_TYPE);
+ HelixDataAccessor accessor = changeContext.getManager().getHelixDataAccessor();
+ PropertyKey.Builder keyBuilder = accessor.keyBuilder();
+ List<InstanceConfig> configList = accessor.getChildValues(keyBuilder.instanceConfigs());
+ List<LiveInstance> liveInstances = accessor.getChildValues(keyBuilder.liveInstances());
+ refreshExternalView(externalViewList, configList, liveInstances, keyReference);
} else {
ClusterEventType eventType;
- if (_sourceDataType.equals(PropertyType.EXTERNALVIEW)) {
+ if (_sourceDataTypeMap.containsKey(PropertyType.EXTERNALVIEW)) {
eventType = ClusterEventType.ExternalViewChange;
- } else if (_sourceDataType.equals(PropertyType.TARGETEXTERNALVIEW)) {
+ } else if (_sourceDataTypeMap.containsKey(PropertyType.TARGETEXTERNALVIEW)) {
eventType = ClusterEventType.TargetExternalViewChange;
} else {
logger.warn(
- "onExternalViewChange called with mismatched change types. Source data type {}, change type: {}",
- _sourceDataType, changeType);
+ "onExternalViewChange called with mismatched change types. Source data types does not contain changed data type: {}",
+ changeType);
return;
}
_routerUpdater.queueEvent(changeContext, eventType, changeType);
@@ -444,11 +649,10 @@ public class RoutingTableProvider
@PreFetch(enabled = true)
public void onLiveInstanceChange(List<LiveInstance> liveInstances,
NotificationContext changeContext) {
- if (_sourceDataType.equals(PropertyType.CURRENTSTATES)) {
+ if (_sourceDataTypeMap.containsKey(PropertyType.CURRENTSTATES)) {
// Go though the live instance list and update CurrentState listeners
updateCurrentStatesListeners(liveInstances, changeContext);
}
-
_routerUpdater.queueEvent(changeContext, ClusterEventType.LiveInstanceChange,
HelixConstants.ChangeType.LIVE_INSTANCE);
}
@@ -457,7 +661,7 @@ public class RoutingTableProvider
@PreFetch(enabled = false)
public void onStateChange(String instanceName, List<CurrentState> statesInfo,
NotificationContext changeContext) {
- if (_sourceDataType.equals(PropertyType.CURRENTSTATES)) {
+ if (_sourceDataTypeMap.containsKey(PropertyType.CURRENTSTATES)) {
_routerUpdater.queueEvent(changeContext, ClusterEventType.CurrentStateChange,
HelixConstants.ChangeType.CURRENT_STATE);
} else {
@@ -466,6 +670,19 @@ public class RoutingTableProvider
}
}
+ @Override
+ @PreFetch(enabled = false)
+ public void onCustomizedViewChange(List<CustomizedView> customizedViewList,
+ NotificationContext changeContext) {
+ if (_sourceDataTypeMap.containsKey(PropertyType.CUSTOMIZEDVIEW)) {
+ _routerUpdater.queueEvent(changeContext, ClusterEventType.CustomizedViewChange,
+ HelixConstants.ChangeType.CUSTOMIZED_VIEW);
+ } else {
+ logger.warn(
+ "RoutingTableProvider does not use CurrentStates as source, ignore CurrentState changes!");
+ }
+ }
+
final AtomicReference<Map<String, LiveInstance>> _lastSeenSessions = new AtomicReference<>();
/**
@@ -529,43 +746,58 @@ public class RoutingTableProvider
private void reset() {
logger.info("Resetting the routing table.");
- RoutingTable newRoutingTable = new RoutingTable();
- _routingTableRef.set(newRoutingTable);
+ RoutingTable newRoutingTable;
+ for (String key: _routingTableRefMap.keySet()) {
+ PropertyType propertyType = _routingTableRefMap.get(key).get().getPropertyType();
+ if (propertyType == PropertyType.CUSTOMIZEDVIEW) {
+ String stateType = _routingTableRefMap.get(key).get().getStateType();
+ newRoutingTable = new CustomizedViewRoutingTable(propertyType, stateType);
+ } else {
+ newRoutingTable = new RoutingTable(propertyType);
+ }
+ _routingTableRefMap.get(key).set(newRoutingTable);
+ }
}
- protected void refresh(List<ExternalView> externalViewList, NotificationContext changeContext) {
- HelixDataAccessor accessor = changeContext.getManager().getHelixDataAccessor();
- PropertyKey.Builder keyBuilder = accessor.keyBuilder();
-
- List<InstanceConfig> configList = accessor.getChildValues(keyBuilder.instanceConfigs());
- List<LiveInstance> liveInstances = accessor.getChildValues(keyBuilder.liveInstances());
- refresh(externalViewList, configList, liveInstances);
+ protected void refreshExternalView(Collection<ExternalView> externalViews,
+ Collection<InstanceConfig> instanceConfigs, Collection<LiveInstance> liveInstances,
+ String referenceKey) {
+ long startTime = System.currentTimeMillis();
+ PropertyType propertyType = _routingTableRefMap.get(referenceKey).get().getPropertyType();
+ RoutingTable newRoutingTable =
+ new RoutingTable(externalViews, instanceConfigs, liveInstances, propertyType);
+ resetRoutingTableAndNotify(startTime, newRoutingTable, referenceKey);
}
- protected void refresh(Collection<ExternalView> externalViews,
- Collection<InstanceConfig> instanceConfigs, Collection<LiveInstance> liveInstances) {
+ protected void refreshCustomizedView(Collection<CustomizedView> customizedViews,
+ Collection<InstanceConfig> instanceConfigs, Collection<LiveInstance> liveInstances,
+ String referenceKey) {
long startTime = System.currentTimeMillis();
- RoutingTable newRoutingTable = new RoutingTable(externalViews, instanceConfigs, liveInstances);
- resetRoutingTableAndNotify(startTime, newRoutingTable);
+ PropertyType propertyType = _routingTableRefMap.get(referenceKey).get().getPropertyType();
+ String customizedStateType = _routingTableRefMap.get(referenceKey).get().getStateType();
+ RoutingTable newRoutingTable = new CustomizedViewRoutingTable(customizedViews, instanceConfigs,
+ liveInstances, propertyType, customizedStateType);
+ resetRoutingTableAndNotify(startTime, newRoutingTable, referenceKey);
}
- protected void refresh(Map<String, Map<String, Map<String, CurrentState>>> currentStateMap,
- Collection<InstanceConfig> instanceConfigs, Collection<LiveInstance> liveInstances) {
+ protected void refreshCurrentState(Map<String, Map<String, Map<String, CurrentState>>> currentStateMap,
+ Collection<InstanceConfig> instanceConfigs, Collection<LiveInstance> liveInstances,
+ String referenceKey) {
long startTime = System.currentTimeMillis();
RoutingTable newRoutingTable =
new RoutingTable(currentStateMap, instanceConfigs, liveInstances);
- resetRoutingTableAndNotify(startTime, newRoutingTable);
+ resetRoutingTableAndNotify(startTime, newRoutingTable, referenceKey);
}
- private void resetRoutingTableAndNotify(long startTime, RoutingTable newRoutingTable) {
- _routingTableRef.set(newRoutingTable);
+ private void resetRoutingTableAndNotify(long startTime, RoutingTable newRoutingTable, String referenceKey) {
+ _routingTableRefMap.get(referenceKey).set(newRoutingTable);
String clusterName = _helixManager != null ? _helixManager.getClusterName() : null;
logger.info("Refreshed the RoutingTable for cluster {}, took {} ms.", clusterName,
(System.currentTimeMillis() - startTime));
// TODO: move the callback user code logic to separate thread upon routing table statePropagation latency
// integration test result. If the latency is more than 2 secs, we need to change this part.
- notifyRoutingTableChange(clusterName);
+ notifyRoutingTableChange(clusterName, referenceKey);
// Update timestamp for last refresh
if (_isPeriodicRefreshEnabled) {
@@ -573,13 +805,16 @@ public class RoutingTableProvider
}
}
- private void notifyRoutingTableChange(String clusterName) {
- // This call back is called in the main event queue of RoutingTableProvider. We add log to record time spent
+ private void notifyRoutingTableChange(String clusterName, String referenceKey) {
+ // This call back is called in the main event queue of RoutingTableProvider. We add log to
+ // record time spent
// here. Potentially, we should call this callback in a separate thread if this is a bottleneck.
long startTime = System.currentTimeMillis();
- for (Map.Entry<RoutingTableChangeListener, ListenerContext> entry : _routingTableChangeListenerMap.entrySet()) {
- entry.getKey()
- .onRoutingTableChange(new RoutingTableSnapshot(_routingTableRef.get()), entry.getValue().getContext());
+ for (Map.Entry<RoutingTableChangeListener, ListenerContext> entry : _routingTableChangeListenerMap
+ .entrySet()) {
+ entry.getKey().onRoutingTableChange(
+ new RoutingTableSnapshot(_routingTableRefMap.get(referenceKey).get()),
+ entry.getValue().getContext());
}
logger.info("RoutingTableProvider user callback time for cluster {}, took {} ms.", clusterName,
(System.currentTimeMillis() - startTime));
@@ -587,10 +822,12 @@ public class RoutingTableProvider
private class RouterUpdater extends ClusterEventProcessor {
private final RoutingDataCache _dataCache;
+ private final Map<PropertyType, List<String>> _sourceDataTypeMap;
- public RouterUpdater(String clusterName, PropertyType sourceDataType) {
+ public RouterUpdater(String clusterName, Map<PropertyType, List<String>> sourceDataTypeMap) {
super(clusterName, "Helix-RouterUpdater-event_process");
- _dataCache = new RoutingDataCache(clusterName, sourceDataType);
+ _sourceDataTypeMap = sourceDataTypeMap;
+ _dataCache = new RoutingDataCache(clusterName, _sourceDataTypeMap);
}
@Override
@@ -618,36 +855,49 @@ public class RoutingTableProvider
throw new HelixException("HelixManager is null for router update event.");
}
if (!manager.isConnected()) {
- logger.error(
- String.format("HelixManager is not connected for router update event: %s", event));
+ logger.error(String.format("HelixManager is not connected for router update event: %s", event));
throw new HelixException("HelixManager is not connected for router update event.");
}
long startTime = System.currentTimeMillis();
_dataCache.refresh(manager.getHelixDataAccessor());
- switch (_sourceDataType) {
- case EXTERNALVIEW:
- refresh(_dataCache.getExternalViews().values(),
- _dataCache.getInstanceConfigMap().values(), _dataCache.getLiveInstances().values());
- break;
- case TARGETEXTERNALVIEW:
- refresh(_dataCache.getTargetExternalViews().values(),
- _dataCache.getInstanceConfigMap().values(), _dataCache.getLiveInstances().values());
- break;
- case CURRENTSTATES:
- refresh(_dataCache.getCurrentStatesMap(), _dataCache.getInstanceConfigMap().values(),
- _dataCache.getLiveInstances().values());
+ for (PropertyType propertyType : _sourceDataTypeMap.keySet()) {
+ switch (propertyType) {
+ case EXTERNALVIEW: {
+ String keyReference = generateReferenceKey(propertyType.name(), DEFAULT_STATE_TYPE);
+ refreshExternalView(_dataCache.getExternalViews().values(),
+ _dataCache.getInstanceConfigMap().values(), _dataCache.getLiveInstances().values(),
+ keyReference);
+ }
+ break;
+ case TARGETEXTERNALVIEW: {
+ String keyReference = generateReferenceKey(propertyType.name(), DEFAULT_STATE_TYPE);
+ refreshExternalView(_dataCache.getTargetExternalViews().values(),
+ _dataCache.getInstanceConfigMap().values(), _dataCache.getLiveInstances().values(),
+ keyReference);
+ }
+ break;
+ case CUSTOMIZEDVIEW:
+ for (String customizedStateType : _sourceDataTypeMap.getOrDefault(PropertyType.CUSTOMIZEDVIEW, Collections.emptyList())) {
+ String keyReference = generateReferenceKey(propertyType.name(), customizedStateType);
+ refreshCustomizedView(_dataCache.getCustomizedView(customizedStateType).values(),
+ _dataCache.getInstanceConfigMap().values(), _dataCache.getLiveInstances().values(), keyReference);
+ }
+ break;
+ case CURRENTSTATES: {
+ String keyReference = generateReferenceKey(propertyType.name(), DEFAULT_STATE_TYPE);;
+ refreshCurrentState(_dataCache.getCurrentStatesMap(), _dataCache.getInstanceConfigMap().values(),
+ _dataCache.getLiveInstances().values(), keyReference);
+ recordPropagationLatency(System.currentTimeMillis(), _dataCache.getCurrentStateSnapshot());
+ }
+ break;
+ default:
+ logger.warn("Unsupported source data type: {}, stop refreshing the routing table!", propertyType);
+ }
- recordPropagationLatency(System.currentTimeMillis(),
- _dataCache.getCurrentStateSnapshot());
- break;
- default:
- logger.warn("Unsupported source data type: {}, stop refreshing the routing table!",
- _sourceDataType);
+ _monitorMap.get(propertyType).increaseDataRefreshCounters(startTime);
}
-
- _monitor.increaseDataRefreshCounters(startTime);
}
}
@@ -673,10 +923,12 @@ public class RoutingTableProvider
for (String partition : partitionStateEndTimes.keySet()) {
long endTime = partitionStateEndTimes.get(partition);
if (currentTime >= endTime) {
- _monitor.recordStatePropagationLatency(currentTime - endTime);
- logger.debug(
- "CurrentState updated in the routing table. Node Key {}, Partition {}, end time {}, Propagation latency {}",
- key.toString(), partition, endTime, currentTime - endTime);
+ for (PropertyType propertyType : _sourceDataTypeMap.keySet()) {
+ _monitorMap.get(propertyType).recordStatePropagationLatency(currentTime - endTime);
+ logger.debug(
+ "CurrentState updated in the routing table. Node Key {}, Partition {}, end time {}, Propagation latency {}",
+ key.toString(), partition, endTime, currentTime - endTime);
+ }
} else {
// Verbose log in case currentTime < endTime. This could be the case that Router
// clock is slower than the participant clock.
@@ -692,6 +944,7 @@ public class RoutingTableProvider
}
}
+
public void queueEvent(NotificationContext context, ClusterEventType eventType,
HelixConstants.ChangeType changeType) {
ClusterEvent event = new ClusterEvent(_clusterName, eventType);
@@ -700,12 +953,17 @@ public class RoutingTableProvider
event.addAttribute(AttributeName.helixmanager.name(), context.getManager());
event.addAttribute(AttributeName.changeContext.name(), context);
queueEvent(event);
-
- _monitor.increaseCallbackCounters(_eventQueue.size());
+ // TODO: Split the monitor into. One is general router callback tracking. The other one is for
+ // each type of tracking.
+ // TODO: We may need to add more complexity to the customized view monitor for each state
+ // type.
+ for (PropertyType propertyType : _monitorMap.keySet()) {
+ _monitorMap.get(propertyType).increaseCallbackCounters(_eventQueue.size());
+ }
}
}
- private class ListenerContext {
+ protected class ListenerContext {
private Object _context;
public ListenerContext(Object context) {
diff --git a/helix-core/src/main/java/org/apache/helix/spectator/RoutingTableSnapshot.java b/helix-core/src/main/java/org/apache/helix/spectator/RoutingTableSnapshot.java
index abfc87b..e7cc6bb 100644
--- a/helix-core/src/main/java/org/apache/helix/spectator/RoutingTableSnapshot.java
+++ b/helix-core/src/main/java/org/apache/helix/spectator/RoutingTableSnapshot.java
@@ -20,9 +20,12 @@ package org.apache.helix.spectator;
*/
import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import java.util.Set;
+import org.apache.helix.PropertyType;
+import org.apache.helix.model.CustomizedView;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.InstanceConfig;
import org.apache.helix.model.LiveInstance;
@@ -33,9 +36,13 @@ import org.apache.helix.model.LiveInstance;
*/
public class RoutingTableSnapshot {
private final RoutingTable _routingTable;
+ private final PropertyType _propertyType;
+ private final String _stateType;
public RoutingTableSnapshot(RoutingTable routingTable) {
_routingTable = routingTable;
+ _propertyType = routingTable.getPropertyType();
+ _stateType = routingTable.getStateType();
}
/**
@@ -145,4 +152,34 @@ public class RoutingTableSnapshot {
public Collection<ExternalView> getExternalViews() {
return _routingTable.getExternalViews();
}
+
+ /**
+ * Returns a Collection of latest snapshot of CustomizedView. Note that if the RoutingTable is
+ * instantiated using CurrentStates, this Collection will be empty.
+ * @return
+ */
+ public Collection<CustomizedView> getCustomizeViews() {
+ if (_propertyType.equals(PropertyType.CUSTOMIZEDVIEW)){
+ CustomizedViewRoutingTable customizedViewRoutingTable =
+ (CustomizedViewRoutingTable) _routingTable;
+ return customizedViewRoutingTable.geCustomizedViews();
+ }
+ return Collections.emptySet();
+ }
+
+ /**
+ * Returns the PropertyType associated with this RoutingTableSnapshot
+ * @return
+ */
+ public PropertyType getPropertyType() {
+ return _propertyType;
+ }
+
+ /**
+ * Return the Type associated with the RoutingTableSnapshot (mainly used for CustomizedView)
+ * @return
+ */
+ public String getCustomizedStateType() {
+ return _stateType;
+ }
}
diff --git a/helix-core/src/test/java/org/apache/helix/controller/stages/DummyClusterManager.java b/helix-core/src/test/java/org/apache/helix/controller/stages/DummyClusterManager.java
index 772c0fa..655276b 100644
--- a/helix-core/src/test/java/org/apache/helix/controller/stages/DummyClusterManager.java
+++ b/helix-core/src/test/java/org/apache/helix/controller/stages/DummyClusterManager.java
@@ -147,7 +147,7 @@ public class DummyClusterManager implements HelixManager {
}
@Override
- public void addCustomizedViewChangeListener(CustomizedViewChangeListener listener, String aggregationType) throws Exception {
+ public void addCustomizedViewChangeListener(CustomizedViewChangeListener listener, String customizedStateType) throws Exception {
// TODO Auto-generated method stub
}
diff --git a/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProvider.java b/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProvider.java
index 6dc036a..d1a2b4c 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProvider.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProvider.java
@@ -1,7 +1,9 @@
package org.apache.helix.integration.spectator;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
@@ -9,15 +11,22 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.helix.AccessOption;
+import org.apache.helix.HelixException;
import org.apache.helix.HelixManager;
import org.apache.helix.HelixManagerFactory;
import org.apache.helix.InstanceType;
+import org.apache.helix.NotificationContext;
+import org.apache.helix.PropertyPathBuilder;
import org.apache.helix.PropertyType;
+import org.apache.helix.TestHelper;
import org.apache.helix.api.listeners.RoutingTableChangeListener;
import org.apache.helix.common.ZkTestBase;
import org.apache.helix.integration.manager.ClusterControllerManager;
import org.apache.helix.integration.manager.MockParticipantManager;
import org.apache.helix.model.BuiltInStateModelDefinitions;
+import org.apache.helix.model.CustomizedView;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.InstanceConfig;
import org.apache.helix.spectator.RoutingTableProvider;
@@ -52,6 +61,9 @@ public class TestRoutingTableProvider extends ZkTestBase {
private RoutingTableProvider _routingTableProvider_cs;
private boolean _listenerTestResult = true;
+
+ private static final AtomicBoolean customizedViewChangeCalled = new AtomicBoolean(false);
+
class MockRoutingTableChangeListener implements RoutingTableChangeListener {
@Override
public void onRoutingTableChange(RoutingTableSnapshot routingTableSnapshot, Object context) {
@@ -73,6 +85,19 @@ public class TestRoutingTableProvider extends ZkTestBase {
}
}
+ class MockRoutingTableProvider extends RoutingTableProvider {
+ MockRoutingTableProvider(HelixManager helixManager, Map<PropertyType, List<String>> sourceDataTypes) {
+ super(helixManager, sourceDataTypes);
+ }
+
+ @Override
+ public void onCustomizedViewChange(List<CustomizedView> customizedViewList,
+ NotificationContext changeContext){
+ customizedViewChangeCalled.getAndSet(true);
+ super.onCustomizedViewChange(customizedViewList, changeContext);
+ }
+ }
+
@BeforeClass
public void beforeClass() throws Exception {
System.out.println(
@@ -212,6 +237,90 @@ public class TestRoutingTableProvider extends ZkTestBase {
Sets.newSet(_instances.get(2)));
}
+ @Test(expectedExceptions = HelixException.class, dependsOnMethods = "testShutdownInstance")
+ public void testExternalViewWithType() {
+ Map<PropertyType, List<String>> sourceDataTypes = new HashMap<>();
+ sourceDataTypes.put(PropertyType.EXTERNALVIEW, Arrays.asList("typeA"));
+ sourceDataTypes.put(PropertyType.CUSTOMIZEDVIEW, Arrays.asList("typeA", "typeB"));
+ RoutingTableProvider routingTableProvider;
+ routingTableProvider = new RoutingTableProvider(_spectator, sourceDataTypes);
+ }
+
+ @Test(dependsOnMethods = "testExternalViewWithType", expectedExceptions = HelixException.class)
+ public void testCustomizedViewWithoutType() {
+ RoutingTableProvider routingTableProvider;
+ routingTableProvider = new RoutingTableProvider(_spectator, PropertyType.CUSTOMIZEDVIEW);
+ }
+
+ @Test(dependsOnMethods = "testCustomizedViewWithoutType")
+ public void testCustomizedViewCorrectConstructor() throws Exception {
+ Map<PropertyType, List<String>> sourceDataTypes = new HashMap<>();
+ sourceDataTypes.put(PropertyType.CUSTOMIZEDVIEW, Arrays.asList("typeA"));
+ MockRoutingTableProvider routingTableProvider =
+ new MockRoutingTableProvider(_spectator, sourceDataTypes);
+
+ CustomizedView customizedView = new CustomizedView(TEST_DB);
+ customizedView.setState("p1", "h1", "testState");
+
+ // Clear the flag before writing to the Customized View Path
+ customizedViewChangeCalled.getAndSet(false);
+ String customizedViewPath = PropertyPathBuilder.customizedView(CLUSTER_NAME, "typeA", TEST_DB);
+ _spectator.getHelixDataAccessor().getBaseDataAccessor().set(customizedViewPath,
+ customizedView.getRecord(), AccessOption.PERSISTENT);
+
+ boolean onCustomizedViewChangeCalled =
+ TestHelper.verify(() -> customizedViewChangeCalled.get(), TestHelper.WAIT_DURATION);
+ Assert.assertTrue(onCustomizedViewChangeCalled);
+
+ _spectator.getHelixDataAccessor().getBaseDataAccessor().remove(customizedViewPath,
+ AccessOption.PERSISTENT);
+ routingTableProvider.shutdown();
+ }
+
+ @Test(dependsOnMethods = "testCustomizedViewCorrectConstructor")
+ public void testGetRoutingTableSnapshot() {
+ Map<PropertyType, List<String>> sourceDataTypes = new HashMap<>();
+ sourceDataTypes.put(PropertyType.CUSTOMIZEDVIEW, Arrays.asList("typeA", "typeB"));
+ sourceDataTypes.put(PropertyType.EXTERNALVIEW, Collections.emptyList());
+ RoutingTableProvider routingTableProvider =
+ new RoutingTableProvider(_spectator, sourceDataTypes);
+
+ CustomizedView customizedView1 = new CustomizedView("Resource1");
+ customizedView1.setState("p1", "h1", "testState1");
+ customizedView1.setState("p1", "h2", "testState1");
+ customizedView1.setState("p2", "h1", "testState2");
+ customizedView1.setState("p3", "h2", "testState3");
+ String customizedViewPath1 =
+ PropertyPathBuilder.customizedView(CLUSTER_NAME, "typeA", "Resource1");
+
+ CustomizedView customizedView2 = new CustomizedView("Resource2");
+ customizedView2.setState("p1", "h3", "testState3");
+ customizedView2.setState("p1", "h4", "testState2");
+ String customizedViewPath2 =
+ PropertyPathBuilder.customizedView(CLUSTER_NAME, "typeB", "Resource2");
+
+ _spectator.getHelixDataAccessor().getBaseDataAccessor().set(customizedViewPath1,
+ customizedView1.getRecord(), AccessOption.PERSISTENT);
+ _spectator.getHelixDataAccessor().getBaseDataAccessor().set(customizedViewPath2,
+ customizedView2.getRecord(), AccessOption.PERSISTENT);
+
+ RoutingTableSnapshot routingTableSnapshot =
+ routingTableProvider.getRoutingTableSnapshot(PropertyType.CUSTOMIZEDVIEW, "typeA");
+ Assert.assertEquals(routingTableSnapshot.getPropertyType(), PropertyType.CUSTOMIZEDVIEW);
+ Assert.assertEquals(routingTableSnapshot.getCustomizedStateType(), "typeA");
+ routingTableSnapshot =
+ routingTableProvider.getRoutingTableSnapshot(PropertyType.CUSTOMIZEDVIEW, "typeB");
+ Assert.assertEquals(routingTableSnapshot.getPropertyType(), PropertyType.CUSTOMIZEDVIEW);
+ Assert.assertEquals(routingTableSnapshot.getCustomizedStateType(), "typeB");
+
+ Map<String, Map<String, RoutingTableSnapshot>> routingTableSnapshots =
+ routingTableProvider.getRoutingTableSnapshots();
+ Assert.assertEquals(routingTableSnapshots.size(), 2);
+ Assert.assertEquals(routingTableSnapshots.get(PropertyType.CUSTOMIZEDVIEW.name()).size(), 2);
+ routingTableProvider.shutdown();
+ }
+
+
private void validateRoutingTable(RoutingTableProvider routingTableProvider,
Set<String> masterNodes, Set<String> slaveNodes) {
IdealState is =
diff --git a/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProviderPeriodicRefresh.java b/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProviderPeriodicRefresh.java
index fdce32e..776fb95 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProviderPeriodicRefresh.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProviderPeriodicRefresh.java
@@ -2,6 +2,7 @@ package org.apache.helix.integration.spectator;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
@@ -16,6 +17,7 @@ import org.apache.helix.integration.manager.ClusterControllerManager;
import org.apache.helix.integration.manager.MockParticipantManager;
import org.apache.helix.model.BuiltInStateModelDefinitions;
import org.apache.helix.model.CurrentState;
+import org.apache.helix.model.CustomizedView;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.InstanceConfig;
import org.apache.helix.model.LiveInstance;
@@ -144,9 +146,10 @@ public class TestRoutingTableProviderPeriodicRefresh extends ZkTestBase {
}
@Override
- protected synchronized void refresh(List<ExternalView> externalViewList,
- NotificationContext changeContext) {
- super.refresh(externalViewList, changeContext);
+ protected synchronized void refreshExternalView(Collection<ExternalView> externalViews,
+ Collection<InstanceConfig> instanceConfigs, Collection<LiveInstance> liveInstances,
+ String referenceKey) {
+ super.refreshExternalView(externalViews, instanceConfigs, liveInstances, referenceKey);
_refreshCount++;
if (DEBUG) {
print();
@@ -154,20 +157,11 @@ public class TestRoutingTableProviderPeriodicRefresh extends ZkTestBase {
}
@Override
- protected synchronized void refresh(Collection<ExternalView> externalViews,
- Collection<InstanceConfig> instanceConfigs, Collection<LiveInstance> liveInstances) {
- super.refresh(externalViews, instanceConfigs, liveInstances);
- _refreshCount++;
- if (DEBUG) {
- print();
- }
- }
-
- @Override
- protected synchronized void refresh(
+ protected synchronized void refreshCurrentState(
Map<String, Map<String, Map<String, CurrentState>>> currentStateMap,
- Collection<InstanceConfig> instanceConfigs, Collection<LiveInstance> liveInstances) {
- super.refresh(currentStateMap, instanceConfigs, liveInstances);
+ Collection<InstanceConfig> instanceConfigs, Collection<LiveInstance> liveInstances,
+ String referenceKey) {
+ super.refreshCurrentState(currentStateMap, instanceConfigs, liveInstances, "Test");
_refreshCount++;
if (DEBUG) {
print();
diff --git a/helix-core/src/test/java/org/apache/helix/mock/MockManager.java b/helix-core/src/test/java/org/apache/helix/mock/MockManager.java
index 6b744b7..90c9bc1 100644
--- a/helix-core/src/test/java/org/apache/helix/mock/MockManager.java
+++ b/helix-core/src/test/java/org/apache/helix/mock/MockManager.java
@@ -151,7 +151,7 @@ public class MockManager implements HelixManager {
}
@Override
- public void addCustomizedViewChangeListener(CustomizedViewChangeListener listener, String aggregationType) {
+ public void addCustomizedViewChangeListener(CustomizedViewChangeListener listener, String customizedStateType) {
// TODO Auto-generated method stub
}
diff --git a/helix-core/src/test/java/org/apache/helix/participant/MockZKHelixManager.java b/helix-core/src/test/java/org/apache/helix/participant/MockZKHelixManager.java
index 9e50c2a..b31ba82 100644
--- a/helix-core/src/test/java/org/apache/helix/participant/MockZKHelixManager.java
+++ b/helix-core/src/test/java/org/apache/helix/participant/MockZKHelixManager.java
@@ -183,7 +183,7 @@ public class MockZKHelixManager implements HelixManager {
}
@Override
- public void addCustomizedViewChangeListener(CustomizedViewChangeListener listener, String aggregationType) throws Exception {
+ public void addCustomizedViewChangeListener(CustomizedViewChangeListener listener, String customizedStateType) throws Exception {
// TODO Auto-generated method stub
}
[helix] 13/23: Add two stages for customized state view
aggregation. (#888)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit cacbd85b0ef05f57084fa65d498858732b06d7ff
Author: zhangmeng916 <56...@users.noreply.github.com>
AuthorDate: Tue Mar 17 23:08:20 2020 -0700
Add two stages for customized state view aggregation. (#888)
1. One stage is the computation stage for customized state. It takes the Zookeeper data of customized states and converts them to the formatted output used by the other stage.
2. The other stage is customized view aggregation stage. It will take the output from the customized state computation stage, and output the customized view to Zookeeper.
3. The two stages together compute the customized view from the customized states.
4. Unit tests are added to verify the correctness of the two stages.
---
.../helix/common/caches/CustomizedStateCache.java | 6 +-
.../dataproviders/BaseControllerDataProvider.java | 6 +-
.../ResourceControllerDataProvider.java | 94 +++++++++++++
.../helix/controller/pipeline/AsyncWorkerType.java | 3 +-
.../stages/CustomizedStateComputationStage.java | 93 +++++++++++++
.../stages/CustomizedViewAggregationStage.java | 150 +++++++++++++++++++++
.../customizedstate/CustomizedStateProvider.java | 5 +-
.../org/apache/helix/manager/zk/ZKHelixAdmin.java | 8 +-
.../org/apache/helix/model/CustomizedState.java | 2 +-
.../apache/helix/model/CustomizedStateConfig.java | 2 +-
.../TestCustomizedStateComputationStage.java | 102 ++++++++++++++
.../controller/stages/TestCustomizedViewStage.java | 106 +++++++++++++++
.../TestControllerDataProviderSelectiveUpdate.java | 4 +-
.../apache/helix/manager/zk/TestZkHelixAdmin.java | 3 +-
.../server/resources/helix/ClusterAccessor.java | 6 +-
.../helix/rest/server/TestClusterAccessor.java | 9 +-
16 files changed, 578 insertions(+), 21 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedStateCache.java b/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedStateCache.java
index 0875b00..ba78953 100644
--- a/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedStateCache.java
+++ b/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedStateCache.java
@@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory;
public class CustomizedStateCache extends ParticipantStateCache<CustomizedState> {
private static final Logger LOG = LoggerFactory.getLogger(CurrentStateCache.class.getName());
- private final Set<String> _aggregationEnabledTypes;
+ private Set<String> _aggregationEnabledTypes;
public CustomizedStateCache(String clusterName, Set<String> aggregationEnabledTypes) {
this(createDefaultControlContextProvider(clusterName), aggregationEnabledTypes);
@@ -60,4 +60,8 @@ public class CustomizedStateCache extends ParticipantStateCache<CustomizedState>
}
return participantStateKeys;
}
+
+ public void setAggregationEnabledTypes(Set<String> aggregationEnabledTypes) {
+ _aggregationEnabledTypes = aggregationEnabledTypes;
+ }
}
diff --git a/helix-core/src/main/java/org/apache/helix/controller/dataproviders/BaseControllerDataProvider.java b/helix-core/src/main/java/org/apache/helix/controller/dataproviders/BaseControllerDataProvider.java
index 0cd9355..51b2e80 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/dataproviders/BaseControllerDataProvider.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/dataproviders/BaseControllerDataProvider.java
@@ -752,8 +752,12 @@ public class BaseControllerDataProvider implements ControlContextProvider {
return sb;
}
+ protected PropertyCache<LiveInstance> getLiveInstanceCache() {
+ return _liveInstanceCache;
+ }
+
@Override
public String toString() {
return genCacheContentStringBuilder().toString();
}
-}
+}
\ No newline at end of file
diff --git a/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java b/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java
index e8e5738..50a3ac9 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java
@@ -22,6 +22,7 @@ package org.apache.helix.controller.dataproviders;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -31,10 +32,14 @@ import org.apache.helix.HelixConstants;
import org.apache.helix.HelixDataAccessor;
import org.apache.helix.PropertyKey;
import org.apache.helix.common.caches.AbstractDataCache;
+import org.apache.helix.common.caches.CustomizedStateCache;
+import org.apache.helix.common.caches.CustomizedViewCache;
import org.apache.helix.common.caches.PropertyCache;
import org.apache.helix.controller.LogUtil;
import org.apache.helix.controller.pipeline.Pipeline;
import org.apache.helix.controller.stages.MissingTopStateRecord;
+import org.apache.helix.model.CustomizedState;
+import org.apache.helix.model.CustomizedStateConfig;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.ResourceAssignment;
@@ -55,6 +60,9 @@ public class ResourceControllerDataProvider extends BaseControllerDataProvider {
// Resource control specific property caches
private final PropertyCache<ExternalView> _externalViewCache;
private final PropertyCache<ExternalView> _targetExternalViewCache;
+ private final CustomizedStateCache _customizedStateCache;
+ // a map from customized state type to customized view cache
+ private final Map<String, CustomizedViewCache> _customizedViewCacheMap;
// maintain a cache of bestPossible assignment across pipeline runs
// TODO: this is only for customRebalancer, remove it and merge it with _idealMappingCache.
@@ -69,6 +77,7 @@ public class ResourceControllerDataProvider extends BaseControllerDataProvider {
// Maintain a set of all ChangeTypes for change detection
private Set<HelixConstants.ChangeType> _refreshedChangeTypes;
+ private Set<String> _aggregationEnabledTypes = new HashSet<>();
// CrushEd strategy needs to have a stable partition list input. So this cached list persist the
// previous seen partition lists. If the members in a list are not modified, the old list will be
@@ -122,6 +131,8 @@ public class ResourceControllerDataProvider extends BaseControllerDataProvider {
_missingTopStateMap = new HashMap<>();
_lastTopStateLocationMap = new HashMap<>();
_refreshedChangeTypes = ConcurrentHashMap.newKeySet();
+ _customizedStateCache = new CustomizedStateCache(this, _aggregationEnabledTypes);
+ _customizedViewCacheMap = new HashMap<>();
}
public synchronized void refresh(HelixDataAccessor accessor) {
@@ -141,8 +152,12 @@ public class ResourceControllerDataProvider extends BaseControllerDataProvider {
}
// Refresh resource controller specific property caches
+ refreshCustomizedStateConfig(accessor);
+ _customizedStateCache.setAggregationEnabledTypes(_aggregationEnabledTypes);
+ _customizedStateCache.refresh(accessor, getLiveInstanceCache().getPropertyMap());
refreshExternalViews(accessor);
refreshTargetExternalViews(accessor);
+ refreshCustomizedViewMap(accessor);
// This is part of the backward compatible workaround to fix
// https://github.com/apache/helix/issues/940.
@@ -164,6 +179,27 @@ public class ResourceControllerDataProvider extends BaseControllerDataProvider {
}
}
+ private void refreshCustomizedStateConfig(final HelixDataAccessor accessor) {
+ if (_propertyDataChangedMap.get(HelixConstants.ChangeType.CUSTOMIZED_STATE_CONFIG)
+ .getAndSet(false)) {
+ CustomizedStateConfig customizedStateConfig =
+ accessor.getProperty(accessor.keyBuilder().customizedStateConfig());
+ if (customizedStateConfig != null) {
+ _aggregationEnabledTypes =
+ new HashSet<>(customizedStateConfig.getAggregationEnabledTypes());
+ } else {
+ _aggregationEnabledTypes.clear();
+ }
+ LogUtil.logInfo(logger, getClusterEventId(), String
+ .format("Reloaded CustomizedStateConfig for cluster %s, %s pipeline.",
+ getClusterName(), getPipelineName()));
+ } else {
+ LogUtil.logInfo(logger, getClusterEventId(), String
+ .format("No customized state config change for %s cluster, %s pipeline",
+ getClusterName(), getPipelineName()));
+ }
+ }
+
private void refreshExternalViews(final HelixDataAccessor accessor) {
// As we are not listening on external view change, external view will be
// refreshed once during the cache's first refresh() call, or when full refresh is required
@@ -181,6 +217,45 @@ public class ResourceControllerDataProvider extends BaseControllerDataProvider {
}
}
+ public void refreshCustomizedViewMap(final HelixDataAccessor accessor) {
+ // As we are not listening on customized view change, customized view will be
+ // refreshed once during the cache's first refresh() call, or when full refresh is required
+ List<String> newStateTypes = accessor.getChildNames(accessor.keyBuilder().customizedViews());
+ if (_propertyDataChangedMap.get(HelixConstants.ChangeType.CUSTOMIZED_VIEW).getAndSet(false)) {
+ for (String stateType : newStateTypes) {
+ if (!_customizedViewCacheMap.containsKey(stateType)) {
+ CustomizedViewCache newCustomizedViewCache =
+ new CustomizedViewCache(getClusterName(), stateType);
+ _customizedViewCacheMap.put(stateType, newCustomizedViewCache);
+ }
+ _customizedViewCacheMap.get(stateType).refresh(accessor);
+ }
+ Set<String> previousCachedStateTypes = _customizedViewCacheMap.keySet();
+ previousCachedStateTypes.removeAll(newStateTypes);
+ logger.info("Remove customizedView for state: " + previousCachedStateTypes);
+ removeCustomizedViewTypes(new ArrayList<>(previousCachedStateTypes));
+ }
+ }
+
+ /**
+ * Provides the customized state of the node for a given state type (resource -> customizedState)
+ * @param instanceName
+ * @param customizedStateType
+ * @return
+ */
+ public Map<String, CustomizedState> getCustomizedState(String instanceName,
+ String customizedStateType) {
+ return _customizedStateCache.getParticipantState(instanceName, customizedStateType);
+ }
+
+ public Set<String> getAggregationEnabledCustomizedStateTypes() {
+ return _aggregationEnabledTypes;
+ }
+
+ protected void setAggregationEnabledCustomizedStateTypes(Set<String> aggregationEnabledTypes) {
+ _aggregationEnabledTypes = aggregationEnabledTypes;
+ }
+
public ExternalView getTargetExternalView(String resourceName) {
return _targetExternalViewCache.getPropertyByName(resourceName);
}
@@ -208,6 +283,14 @@ public class ResourceControllerDataProvider extends BaseControllerDataProvider {
}
/**
+ * Get local cached customized view map
+ * @return
+ */
+ public Map<String, CustomizedViewCache> getCustomizedViewCacheMap() {
+ return _customizedViewCacheMap;
+ }
+
+ /**
* Remove dead external views from map
* @param resourceNames
*/
@@ -218,6 +301,17 @@ public class ResourceControllerDataProvider extends BaseControllerDataProvider {
}
}
+ /**
+ * Remove dead customized views for certain state types from map
+ * @param stateTypeNames
+ */
+
+ private void removeCustomizedViewTypes(List<String> stateTypeNames) {
+ for (String stateType : stateTypeNames) {
+ _customizedViewCacheMap.remove(stateType);
+ }
+ }
+
public Map<String, Map<String, MissingTopStateRecord>> getMissingTopStateMap() {
return _missingTopStateMap;
}
diff --git a/helix-core/src/main/java/org/apache/helix/controller/pipeline/AsyncWorkerType.java b/helix-core/src/main/java/org/apache/helix/controller/pipeline/AsyncWorkerType.java
index ac938dc..e39050a 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/pipeline/AsyncWorkerType.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/pipeline/AsyncWorkerType.java
@@ -31,5 +31,6 @@ public enum AsyncWorkerType {
PersistAssignmentWorker,
ExternalViewComputeWorker,
MaintenanceRecoveryWorker,
- TaskJobPurgeWorker
+ TaskJobPurgeWorker,
+ CustomizedStateViewComputeWorker
}
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/CustomizedStateComputationStage.java b/helix-core/src/main/java/org/apache/helix/controller/stages/CustomizedStateComputationStage.java
new file mode 100644
index 0000000..cb5f1a3
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/controller/stages/CustomizedStateComputationStage.java
@@ -0,0 +1,93 @@
+package org.apache.helix.controller.stages;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.helix.controller.dataproviders.ResourceControllerDataProvider;
+import org.apache.helix.controller.pipeline.AbstractBaseStage;
+import org.apache.helix.controller.pipeline.StageException;
+import org.apache.helix.model.CustomizedState;
+import org.apache.helix.model.LiveInstance;
+import org.apache.helix.model.Partition;
+import org.apache.helix.model.Resource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class CustomizedStateComputationStage extends AbstractBaseStage {
+ private static Logger LOG = LoggerFactory.getLogger(CustomizedStateComputationStage.class);
+
+ @Override
+ public void process(ClusterEvent event) throws Exception {
+ _eventId = event.getEventId();
+ ResourceControllerDataProvider cache =
+ event.getAttribute(AttributeName.ControllerDataProvider.name());
+ final Map<String, Resource> resourceMap =
+ event.getAttribute(AttributeName.RESOURCES_TO_REBALANCE.name());
+ Set<String> aggregationEnabledTypes = cache.getAggregationEnabledCustomizedStateTypes();
+
+ if (cache == null || resourceMap == null) {
+ throw new StageException(
+ "Missing attributes in event:" + event + ". Requires DataCache|RESOURCE");
+ }
+
+ Map<String, LiveInstance> liveInstances = cache.getLiveInstances();
+ final CustomizedStateOutput customizedStateOutput = new CustomizedStateOutput();
+
+ for (LiveInstance instance : liveInstances.values()) {
+ String instanceName = instance.getInstanceName();
+ // update customized states.
+ for (String customizedStateType : aggregationEnabledTypes) {
+ Map<String, CustomizedState> customizedStateMap =
+ cache.getCustomizedState(instanceName, customizedStateType);
+ updateCustomizedStates(instanceName, customizedStateType, customizedStateMap,
+ customizedStateOutput, resourceMap);
+ }
+ }
+ event.addAttribute(AttributeName.CUSTOMIZED_STATE.name(), customizedStateOutput);
+ }
+
+ // update customized state in CustomizedStateOutput
+ private void updateCustomizedStates(String instanceName, String customizedStateType,
+ Map<String, CustomizedState> customizedStates, CustomizedStateOutput customizedStateOutput,
+ Map<String, Resource> resourceMap) {
+ // for each CustomizedState, update corresponding entry in CustomizedStateOutput
+ for (CustomizedState customizedState : customizedStates.values()) {
+ String resourceName = customizedState.getResourceName();
+ Resource resource = resourceMap.get(resourceName);
+ if (resource == null) {
+ continue;
+ }
+
+ Map<String, String> partitionStateMap = customizedState
+ .getPartitionStateMap(CustomizedState.CustomizedStateProperty.CURRENT_STATE);
+ for (String partitionName : partitionStateMap.keySet()) {
+ Partition partition = resource.getPartition(partitionName);
+ if (partition != null) {
+ customizedStateOutput
+ .setCustomizedState(customizedStateType, resourceName, partition, instanceName,
+ customizedState.getState(partitionName));
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/CustomizedViewAggregationStage.java b/helix-core/src/main/java/org/apache/helix/controller/stages/CustomizedViewAggregationStage.java
new file mode 100644
index 0000000..de0bc1c
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/controller/stages/CustomizedViewAggregationStage.java
@@ -0,0 +1,150 @@
+package org.apache.helix.controller.stages;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.HelixException;
+import org.apache.helix.HelixManager;
+import org.apache.helix.PropertyKey;
+import org.apache.helix.common.caches.CustomizedViewCache;
+import org.apache.helix.controller.LogUtil;
+import org.apache.helix.controller.dataproviders.ResourceControllerDataProvider;
+import org.apache.helix.controller.pipeline.AbstractAsyncBaseStage;
+import org.apache.helix.controller.pipeline.AsyncWorkerType;
+import org.apache.helix.controller.pipeline.StageException;
+import org.apache.helix.model.CustomizedView;
+import org.apache.helix.model.Partition;
+import org.apache.helix.model.Resource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class CustomizedViewAggregationStage extends AbstractAsyncBaseStage {
+ private static Logger LOG = LoggerFactory.getLogger(CustomizedViewAggregationStage.class);
+
+ @Override
+ public AsyncWorkerType getAsyncWorkerType() {
+ return AsyncWorkerType.CustomizedStateViewComputeWorker;
+ }
+
+ @Override
+ public void execute(final ClusterEvent event) throws Exception {
+ _eventId = event.getEventId();
+ HelixManager manager = event.getAttribute(AttributeName.helixmanager.name());
+ Map<String, Resource> resourceMap =
+ event.getAttribute(AttributeName.RESOURCES_TO_REBALANCE.name());
+ ResourceControllerDataProvider cache =
+ event.getAttribute(AttributeName.ControllerDataProvider.name());
+
+ if (manager == null || resourceMap == null || cache == null) {
+ throw new StageException(
+ "Missing attributes in event:" + event + ". Requires ClusterManager|RESOURCES|DataCache");
+ }
+
+ HelixDataAccessor dataAccessor = manager.getHelixDataAccessor();
+ PropertyKey.Builder keyBuilder = dataAccessor.keyBuilder();
+
+ CustomizedStateOutput customizedStateOutput =
+ event.getAttribute(AttributeName.CUSTOMIZED_STATE.name());
+
+ Map<String, CustomizedViewCache> customizedViewCacheMap = cache.getCustomizedViewCacheMap();
+
+ // remove stale customized view type from ZK and cache
+ List<String> customizedViewTypesToRemove = new ArrayList<>();
+ for (String stateType : customizedViewCacheMap.keySet()) {
+ if (!customizedStateOutput.getAllStateTypes().contains(stateType)) {
+ LogUtil.logInfo(LOG, _eventId, "Remove customizedView for stateType: " + stateType);
+ dataAccessor.removeProperty(keyBuilder.customizedView(stateType));
+ customizedViewTypesToRemove.add(stateType);
+ }
+ }
+
+ List<CustomizedView> updatedCustomizedViews = new ArrayList<>();
+ // update customized view
+ for (String stateType : customizedStateOutput.getAllStateTypes()) {
+ Map<String, CustomizedView> curCustomizedViews = new HashMap<>();
+ CustomizedViewCache customizedViewCache = customizedViewCacheMap.get(stateType);
+ if (customizedViewCache != null) {
+ curCustomizedViews = customizedViewCache.getCustomizedViewMap();
+ }
+
+ for (Resource resource : resourceMap.values()) {
+ try {
+ computeCustomizedStateView(resource, stateType, customizedStateOutput, curCustomizedViews,
+ updatedCustomizedViews);
+
+ List<PropertyKey> keys = new ArrayList<>();
+ for (Iterator<CustomizedView> it = updatedCustomizedViews.iterator(); it.hasNext(); ) {
+ CustomizedView view = it.next();
+ String resourceName = view.getResourceName();
+ keys.add(keyBuilder.customizedView(stateType, resourceName));
+ }
+ // add/update customized-views
+ if (updatedCustomizedViews.size() > 0) {
+ dataAccessor.setChildren(keys, updatedCustomizedViews);
+ }
+
+ // remove stale customized views
+ for (String resourceName : curCustomizedViews.keySet()) {
+ if (!resourceMap.keySet().contains(resourceName)) {
+ LogUtil.logInfo(LOG, _eventId, "Remove customizedView for resource: " + resourceName);
+ dataAccessor.removeProperty(keyBuilder.customizedView(stateType, resourceName));
+ }
+ }
+ } catch (HelixException ex) {
+ LogUtil.logError(LOG, _eventId,
+ "Failed to calculate customized view for resource " + resource.getResourceName(), ex);
+ }
+ }
+ }
+ }
+
+ private void computeCustomizedStateView(final Resource resource, final String stateType,
+ CustomizedStateOutput customizedStateOutput,
+ final Map<String, CustomizedView> curCustomizedViews,
+ List<CustomizedView> updatedCustomizedViews) {
+ String resourceName = resource.getResourceName();
+ CustomizedView view = new CustomizedView(resource.getResourceName());
+
+ for (Partition partition : resource.getPartitions()) {
+ Map<String, String> customizedStateMap =
+ customizedStateOutput.getPartitionCustomizedStateMap(stateType, resourceName, partition);
+ if (customizedStateMap != null && customizedStateMap.size() > 0) {
+ for (String instance : customizedStateMap.keySet()) {
+ view.setState(partition.getPartitionName(), instance, customizedStateMap.get(instance));
+ }
+ }
+ }
+
+ CustomizedView curCustomizedView = curCustomizedViews.get(resourceName);
+
+ // compare the new customized view with current one, set only on different
+ if (curCustomizedView == null || !curCustomizedView.getRecord().equals(view.getRecord())) {
+ // Add customized view to the list which will be written to ZK later.
+ updatedCustomizedViews.add(view);
+ }
+ }
+}
diff --git a/helix-core/src/main/java/org/apache/helix/customizedstate/CustomizedStateProvider.java b/helix-core/src/main/java/org/apache/helix/customizedstate/CustomizedStateProvider.java
index 80f4d91..7684e05 100644
--- a/helix-core/src/main/java/org/apache/helix/customizedstate/CustomizedStateProvider.java
+++ b/helix-core/src/main/java/org/apache/helix/customizedstate/CustomizedStateProvider.java
@@ -21,13 +21,14 @@ package org.apache.helix.customizedstate;
import java.util.HashMap;
import java.util.Map;
-import org.I0Itec.zkclient.DataUpdater;
+
import org.apache.helix.HelixDataAccessor;
import org.apache.helix.HelixException;
import org.apache.helix.HelixManager;
import org.apache.helix.PropertyKey;
-import org.apache.helix.ZNRecord;
import org.apache.helix.model.CustomizedState;
+import org.apache.helix.zookeeper.datamodel.ZNRecord;
+import org.apache.helix.zookeeper.zkclient.DataUpdater;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
index 7ece2c7..af037f0 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
@@ -1233,7 +1233,7 @@ public class ZKHelixAdmin implements HelixAdmin {
ZKHelixDataAccessor accessor =
new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
- Builder keyBuilder = accessor.keyBuilder();
+ PropertyKey.Builder keyBuilder = accessor.keyBuilder();
accessor.setProperty(keyBuilder.customizedStateConfig(),
customizedStateConfigFromBuilder);
}
@@ -1245,7 +1245,7 @@ public class ZKHelixAdmin implements HelixAdmin {
ZKHelixDataAccessor accessor =
new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
- Builder keyBuilder = accessor.keyBuilder();
+ PropertyKey.Builder keyBuilder = accessor.keyBuilder();
accessor.removeProperty(keyBuilder.customizedStateConfig());
}
@@ -1265,7 +1265,7 @@ public class ZKHelixAdmin implements HelixAdmin {
ZKHelixDataAccessor accessor =
new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
- Builder keyBuilder = accessor.keyBuilder();
+ PropertyKey.Builder keyBuilder = accessor.keyBuilder();
accessor.updateProperty(keyBuilder.customizedStateConfig(),
customizedStateConfigFromBuilder);
}
@@ -1292,7 +1292,7 @@ public class ZKHelixAdmin implements HelixAdmin {
CustomizedStateConfig customizedStateConfigFromBuilder = builder.build();
ZKHelixDataAccessor accessor =
new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
- Builder keyBuilder = accessor.keyBuilder();
+ PropertyKey.Builder keyBuilder = accessor.keyBuilder();
accessor.setProperty(keyBuilder.customizedStateConfig(),
customizedStateConfigFromBuilder);
}
diff --git a/helix-core/src/main/java/org/apache/helix/model/CustomizedState.java b/helix-core/src/main/java/org/apache/helix/model/CustomizedState.java
index 7b0b97d..925c3e0 100644
--- a/helix-core/src/main/java/org/apache/helix/model/CustomizedState.java
+++ b/helix-core/src/main/java/org/apache/helix/model/CustomizedState.java
@@ -24,7 +24,7 @@ import java.util.Map;
import java.util.TreeMap;
import org.apache.helix.HelixProperty;
-import org.apache.helix.ZNRecord;
+import org.apache.helix.zookeeper.datamodel.ZNRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/helix-core/src/main/java/org/apache/helix/model/CustomizedStateConfig.java b/helix-core/src/main/java/org/apache/helix/model/CustomizedStateConfig.java
index 0b13db1..6726ac4 100644
--- a/helix-core/src/main/java/org/apache/helix/model/CustomizedStateConfig.java
+++ b/helix-core/src/main/java/org/apache/helix/model/CustomizedStateConfig.java
@@ -28,7 +28,7 @@ import org.apache.helix.zookeeper.datamodel.ZNRecord;
/**
- * CustomizedStateAggregation configurations
+ * Customized state configurations
*/
public class CustomizedStateConfig extends HelixProperty {
diff --git a/helix-core/src/test/java/org/apache/helix/controller/stages/TestCustomizedStateComputationStage.java b/helix-core/src/test/java/org/apache/helix/controller/stages/TestCustomizedStateComputationStage.java
new file mode 100644
index 0000000..f8c1689
--- /dev/null
+++ b/helix-core/src/test/java/org/apache/helix/controller/stages/TestCustomizedStateComputationStage.java
@@ -0,0 +1,102 @@
+package org.apache.helix.controller.stages;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.helix.PropertyKey;
+import org.apache.helix.controller.dataproviders.ResourceControllerDataProvider;
+import org.apache.helix.model.CustomizedState;
+import org.apache.helix.model.CustomizedStateConfig;
+import org.apache.helix.model.Partition;
+import org.apache.helix.model.Resource;
+import org.testng.AssertJUnit;
+import org.testng.annotations.Test;
+
+
+public class TestCustomizedStateComputationStage extends BaseStageTest {
+ private final String RESOURCE_NAME = "testResourceName";
+ private final String PARTITION_NAME = "testResourceName_0";
+ private final String CUSTOMIZED_STATE_NAME = "customizedState1";
+ private final String INSTANCE_NAME = "localhost_1";
+
+ @Test
+ public void testEmptyCustomizedState() {
+ Map<String, Resource> resourceMap = getResourceMap();
+ event.addAttribute(AttributeName.RESOURCES.name(), resourceMap);
+ event.addAttribute(AttributeName.RESOURCES_TO_REBALANCE.name(), resourceMap);
+ event.addAttribute(AttributeName.ControllerDataProvider.name(),
+ new ResourceControllerDataProvider(_clusterName));
+ CustomizedStateComputationStage stage = new CustomizedStateComputationStage();
+ runStage(event, new ReadClusterDataStage());
+ runStage(event, stage);
+ CustomizedStateOutput output = event.getAttribute(AttributeName.CUSTOMIZED_STATE.name());
+ AssertJUnit.assertEquals(output
+ .getPartitionCustomizedStateMap(CUSTOMIZED_STATE_NAME, RESOURCE_NAME,
+ new Partition("testResourceName_0")).size(), 0);
+ }
+
+ @Test
+ public void testSimpleCustomizedState() {
+ // setup resource
+ Map<String, Resource> resourceMap = getResourceMap();
+
+ setupLiveInstances(5);
+
+ // Add CustomizedStateAggregation Config
+ CustomizedStateConfig config = new CustomizedStateConfig();
+ List<String> aggregationEnabledTypes = new ArrayList<>();
+ aggregationEnabledTypes.add(CUSTOMIZED_STATE_NAME);
+
+ config.setAggregationEnabledTypes(aggregationEnabledTypes);
+
+ PropertyKey.Builder keyBuilder = accessor.keyBuilder();
+ accessor.setProperty(keyBuilder.customizedStateConfig(), config);
+
+ event.addAttribute(AttributeName.RESOURCES.name(), resourceMap);
+ event.addAttribute(AttributeName.RESOURCES_TO_REBALANCE.name(), resourceMap);
+ event.addAttribute(AttributeName.ControllerDataProvider.name(),
+ new ResourceControllerDataProvider(_clusterName));
+ CustomizedStateComputationStage stage = new CustomizedStateComputationStage();
+ runStage(event, new ReadClusterDataStage());
+ runStage(event, stage);
+ CustomizedStateOutput output1 = event.getAttribute(AttributeName.CUSTOMIZED_STATE.name());
+ AssertJUnit.assertEquals(output1
+ .getPartitionCustomizedStateMap(CUSTOMIZED_STATE_NAME, RESOURCE_NAME,
+ new Partition(PARTITION_NAME)).size(), 0);
+
+ // Add a customized state
+ CustomizedState customizedState = new CustomizedState(RESOURCE_NAME);
+ customizedState.setState(PARTITION_NAME, "STARTED");
+ accessor.setProperty(
+ keyBuilder.customizedState(INSTANCE_NAME, CUSTOMIZED_STATE_NAME, RESOURCE_NAME),
+ customizedState);
+
+ runStage(event, new ReadClusterDataStage());
+ runStage(event, stage);
+ CustomizedStateOutput output2 = event.getAttribute(AttributeName.CUSTOMIZED_STATE.name());
+ Partition partition = new Partition(PARTITION_NAME);
+ AssertJUnit.assertEquals(output2
+ .getPartitionCustomizedState(CUSTOMIZED_STATE_NAME, RESOURCE_NAME, partition,
+ INSTANCE_NAME), "STARTED");
+ }
+}
\ No newline at end of file
diff --git a/helix-core/src/test/java/org/apache/helix/controller/stages/TestCustomizedViewStage.java b/helix-core/src/test/java/org/apache/helix/controller/stages/TestCustomizedViewStage.java
new file mode 100644
index 0000000..a8a167b
--- /dev/null
+++ b/helix-core/src/test/java/org/apache/helix/controller/stages/TestCustomizedViewStage.java
@@ -0,0 +1,106 @@
+package org.apache.helix.controller.stages;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.HelixManager;
+import org.apache.helix.PropertyKey;
+import org.apache.helix.TestHelper;
+import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.controller.dataproviders.ResourceControllerDataProvider;
+import org.apache.helix.controller.pipeline.Pipeline;
+import org.apache.helix.manager.zk.ZKHelixDataAccessor;
+import org.apache.helix.manager.zk.ZkBaseDataAccessor;
+import org.apache.helix.model.CustomizedState;
+import org.apache.helix.model.CustomizedStateConfig;
+import org.apache.helix.model.CustomizedView;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+
+public class TestCustomizedViewStage extends ZkUnitTestBase {
+ private final String RESOURCE_NAME = "testResourceName";
+ private final String PARTITION_NAME = "testResourceName_0";
+ private final String CUSTOMIZED_STATE_NAME = "customizedState1";
+ private final String INSTANCE_NAME = "localhost_1";
+
+ @Test
+ public void testCachedCustomizedViews() throws Exception {
+ String clusterName = "CLUSTER_" + TestHelper.getTestMethodName();
+
+ HelixDataAccessor accessor =
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<>(_gZkClient));
+ HelixManager manager = new DummyClusterManager(clusterName, accessor);
+
+ setupLiveInstances(clusterName, new int[]{0, 1});
+ setupStateModel(clusterName);
+
+ ClusterEvent event = new ClusterEvent(ClusterEventType.Unknown);
+ ResourceControllerDataProvider cache = new ResourceControllerDataProvider(clusterName);
+ event.addAttribute(AttributeName.helixmanager.name(), manager);
+ event.addAttribute(AttributeName.ControllerDataProvider.name(), cache);
+
+ CustomizedStateConfig config = new CustomizedStateConfig();
+ List<String> aggregationEnabledTypes = new ArrayList<>();
+ aggregationEnabledTypes.add(CUSTOMIZED_STATE_NAME);
+ config.setAggregationEnabledTypes(aggregationEnabledTypes);
+
+ PropertyKey.Builder keyBuilder = accessor.keyBuilder();
+ accessor.setProperty(keyBuilder.customizedStateConfig(), config);
+
+ CustomizedState customizedState = new CustomizedState(RESOURCE_NAME);
+ customizedState.setState(PARTITION_NAME, "STARTED");
+ accessor
+ .setProperty(keyBuilder.customizedState(INSTANCE_NAME, "customizedState1", RESOURCE_NAME),
+ customizedState);
+
+ CustomizedViewAggregationStage customizedViewComputeStage =
+ new CustomizedViewAggregationStage();
+ Pipeline dataRefresh = new Pipeline();
+ dataRefresh.addStage(new ReadClusterDataStage());
+ runPipeline(event, dataRefresh);
+ runStage(event, new ResourceComputationStage());
+ runStage(event, new CustomizedStateComputationStage());
+ runStage(event, customizedViewComputeStage);
+ Assert.assertEquals(cache.getCustomizedViewCacheMap().values(),
+ accessor.getChildValues(accessor.keyBuilder().customizedViews()));
+
+ // Assure there is no customized view got updated when running the stage again
+ List<CustomizedView> oldCustomizedViews =
+ accessor.getChildValues(accessor.keyBuilder().customizedViews());
+ runStage(event, customizedViewComputeStage);
+ List<CustomizedView> newCustomizedViews =
+ accessor.getChildValues(accessor.keyBuilder().customizedViews());
+ Assert.assertEquals(oldCustomizedViews, newCustomizedViews);
+ for (int i = 0; i < oldCustomizedViews.size(); i++) {
+ Assert.assertEquals(oldCustomizedViews.get(i).getStat().getVersion(),
+ newCustomizedViews.get(i).getStat().getVersion());
+ }
+
+ if (manager.isConnected()) {
+ manager.disconnect(); // For DummyClusterManager, this is not necessary
+ }
+ deleteLiveInstances(clusterName);
+ deleteCluster(clusterName);
+ }
+}
\ No newline at end of file
diff --git a/helix-core/src/test/java/org/apache/helix/integration/controller/TestControllerDataProviderSelectiveUpdate.java b/helix-core/src/test/java/org/apache/helix/integration/controller/TestControllerDataProviderSelectiveUpdate.java
index 9115e21..87b5904 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/controller/TestControllerDataProviderSelectiveUpdate.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/controller/TestControllerDataProviderSelectiveUpdate.java
@@ -49,7 +49,7 @@ public class TestControllerDataProviderSelectiveUpdate extends ZkStandAloneCMTes
Assert.assertEquals(accessor.getReadCount(PropertyType.EXTERNALVIEW), 1);
Assert.assertEquals(accessor.getReadCount(PropertyType.LIVEINSTANCES), NODE_NR);
Assert.assertEquals(accessor.getReadCount(PropertyType.CURRENTSTATES), NODE_NR);
- Assert.assertEquals(accessor.getReadCount(PropertyType.CONFIGS), NODE_NR + 1);
+ Assert.assertEquals(accessor.getReadCount(PropertyType.CONFIGS), NODE_NR + 2);
accessor.clearReadCounters();
@@ -94,7 +94,7 @@ public class TestControllerDataProviderSelectiveUpdate extends ZkStandAloneCMTes
Assert.assertEquals(accessor.getReadCount(PropertyType.EXTERNALVIEW), 1);
Assert.assertEquals(accessor.getReadCount(PropertyType.LIVEINSTANCES), NODE_NR);
Assert.assertEquals(accessor.getReadCount(PropertyType.CURRENTSTATES), NODE_NR);
- Assert.assertEquals(accessor.getReadCount(PropertyType.CONFIGS), NODE_NR + 1);
+ Assert.assertEquals(accessor.getReadCount(PropertyType.CONFIGS), NODE_NR + 2);
accessor.clearReadCounters();
diff --git a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
index f36d66c..5bc6d4f 100644
--- a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
+++ b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
@@ -44,16 +44,17 @@ import org.apache.helix.TestHelper;
import org.apache.helix.zookeeper.api.client.RealmAwareZkClient;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.controller.rebalancer.waged.WagedRebalancer;
import org.apache.helix.examples.MasterSlaveStateModelFactory;
import org.apache.helix.integration.manager.MockParticipantManager;
import org.apache.helix.model.ClusterConfig;
-import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ClusterConstraints.ConstraintAttribute;
import org.apache.helix.model.ClusterConstraints.ConstraintType;
import org.apache.helix.model.ConstraintItem;
+import org.apache.helix.model.CustomizedStateConfig;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.HelixConfigScope;
import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
index ca3c757..a5800b2 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
@@ -319,7 +319,7 @@ public class ClusterAccessor extends AbstractHelixResource {
new CustomizedStateConfig.Builder(record).build();
admin.addCustomizedStateConfig(clusterId, customizedStateConfig);
} catch (Exception ex) {
- _logger.error("Cannot add CustomizedStateConfig to cluster: {} Exception: {}",
+ LOG.error("Cannot add CustomizedStateConfig to cluster: {} Exception: {}",
clusterId, ex);
return serverError(ex);
}
@@ -338,7 +338,7 @@ public class ClusterAccessor extends AbstractHelixResource {
try {
admin.removeCustomizedStateConfig(clusterId);
} catch (Exception ex) {
- _logger.error(
+ LOG.error(
"Cannot remove CustomizedStateConfig from cluster: {}, Exception: {}",
clusterId, ex);
return serverError(ex);
@@ -398,7 +398,7 @@ public class ClusterAccessor extends AbstractHelixResource {
return badRequest("Unsupported command " + commandStr);
}
} catch (Exception ex) {
- _logger.error("Failed to {} CustomizedStateConfig for cluster {} new type: {}, Exception: {}", command, clusterId, type, ex);
+ LOG.error("Failed to {} CustomizedStateConfig for cluster {} new type: {}, Exception: {}", command, clusterId, type, ex);
return serverError(ex);
}
return OK();
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
index a39140d..279da84 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
@@ -37,23 +37,23 @@ import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixDataAccessor;
import org.apache.helix.PropertyKey;
import org.apache.helix.TestHelper;
-import org.apache.helix.model.RESTConfig;
-import org.apache.helix.zookeeper.datamodel.ZNRecord;
import org.apache.helix.cloud.azure.AzureConstants;
+import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.controller.rebalancer.DelayedAutoRebalancer;
import org.apache.helix.controller.rebalancer.strategy.CrushEdRebalanceStrategy;
import org.apache.helix.controller.rebalancer.waged.WagedRebalancer;
import org.apache.helix.integration.manager.ClusterDistributedController;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
import org.apache.helix.manager.zk.ZKUtil;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConfig;
+import org.apache.helix.model.CustomizedStateConfig;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.InstanceConfig;
import org.apache.helix.model.LiveInstance;
import org.apache.helix.model.MaintenanceSignal;
-import org.apache.helix.model.CloudConfig;
-import org.apache.helix.cloud.constants.CloudProvider;
+import org.apache.helix.model.RESTConfig;
import org.apache.helix.rest.common.HelixRestNamespace;
import org.apache.helix.rest.server.auditlog.AuditLog;
import org.apache.helix.rest.server.resources.AbstractResource;
@@ -61,6 +61,7 @@ import org.apache.helix.rest.server.resources.AbstractResource.Command;
import org.apache.helix.rest.server.resources.helix.ClusterAccessor;
import org.apache.helix.rest.server.util.JerseyUriRequestBuilder;
import org.apache.helix.tools.ClusterVerifiers.BestPossibleExternalViewVerifier;
+import org.apache.helix.zookeeper.datamodel.ZNRecord;
import org.codehaus.jackson.JsonNode;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.type.TypeReference;
[helix] 18/23: Add integration test to customized view aggregation
(#912)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 1c261220d4b211cf17e69da81b381a6cfa5b567b
Author: Molly Gao <31...@users.noreply.github.com>
AuthorDate: Tue Mar 31 10:49:52 2020 -0700
Add integration test to customized view aggregation (#912)
The integration test involves components: update customized state using customized view provider, and use routing table provider to get customized view snapshots which are aggregated in controller.
---
.../helix/controller/GenericHelixController.java | 3 +-
.../integration/TestCustomizedViewAggregation.java | 465 +++++++++++++++++++++
2 files changed, 467 insertions(+), 1 deletion(-)
diff --git a/helix-core/src/main/java/org/apache/helix/controller/GenericHelixController.java b/helix-core/src/main/java/org/apache/helix/controller/GenericHelixController.java
index 5678a5c..23f5285 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/GenericHelixController.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/GenericHelixController.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -892,7 +893,7 @@ public class GenericHelixController implements IdealStateChangeListener, LiveIns
}
if (!lastSeenCustomizedStateTypesMap.containsKey(instanceName)) {
- lastSeenCustomizedStateTypesMap.put(instanceName, Collections.emptySet());
+ lastSeenCustomizedStateTypesMap.put(instanceName, new HashSet<>());
}
Set<String> lastSeenCustomizedStateTypes = lastSeenCustomizedStateTypesMap.get(instanceName);
diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestCustomizedViewAggregation.java b/helix-core/src/test/java/org/apache/helix/integration/TestCustomizedViewAggregation.java
new file mode 100644
index 0000000..af24107
--- /dev/null
+++ b/helix-core/src/test/java/org/apache/helix/integration/TestCustomizedViewAggregation.java
@@ -0,0 +1,465 @@
+package org.apache.helix.integration;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import com.google.common.collect.Maps;
+import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.HelixManager;
+import org.apache.helix.HelixManagerFactory;
+import org.apache.helix.InstanceType;
+import org.apache.helix.PropertyType;
+import org.apache.helix.TestHelper;
+import org.apache.helix.ZNRecord;
+import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.customizedstate.CustomizedStateProvider;
+import org.apache.helix.customizedstate.CustomizedStateProviderFactory;
+import org.apache.helix.integration.manager.ClusterControllerManager;
+import org.apache.helix.integration.manager.MockParticipantManager;
+import org.apache.helix.model.CustomizedState;
+import org.apache.helix.model.CustomizedStateConfig;
+import org.apache.helix.model.CustomizedView;
+import org.apache.helix.spectator.RoutingTableProvider;
+import org.apache.helix.spectator.RoutingTableSnapshot;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+
+public class TestCustomizedViewAggregation extends ZkUnitTestBase {
+
+ private static CustomizedStateProvider _customizedStateProvider_participant0;
+ private static CustomizedStateProvider _customizedStateProvider_participant1;
+ private static RoutingTableProvider _routingTableProvider;
+ private static HelixManager _spectator;
+ private static HelixManager _manager;
+ // 1st key: customized state type, 2nd key: resource name, 3rd key: partition name, 4th key: instance name, value: state value
+ // This map contains all the customized state information that is updated to ZooKeeper
+ private static Map<String, Map<String, Map<String, Map<String, String>>>> _localCustomizedView;
+ // The set contains customized state types that are enabled for aggregation in config
+ private static Set<String> _aggregationEnabledTypes;
+ // The set contains customized state types that routing table provider shows to users
+ private static Set<String> _routingTableProviderDataSources;
+ private String INSTANCE_0;
+ private String INSTANCE_1;
+ private final String RESOURCE_0 = "TestDB0";
+ private final String RESOURCE_1 = "TestDB1";
+ private final String PARTITION_00 = "TestDB0_0";
+ private final String PARTITION_01 = "TestDB0_1";
+ private final String PARTITION_10 = "TestDB1_0";
+ private final String PARTITION_11 = "TestDB1_1";
+
+ // Customized state values used for test, TYPE_A_0 - TYPE_A_2 are values for Customized state TypeA, etc.
+ private enum CurrentStateValues {
+ TYPE_A_0, TYPE_A_1, TYPE_A_2, TYPE_B_0, TYPE_B_1, TYPE_B_2, TYPE_C_0, TYPE_C_1, TYPE_C_2
+ }
+
+ private enum CustomizedStateType {
+ TYPE_A, TYPE_B, TYPE_C
+ }
+
+ @BeforeClass
+ public void beforeClass() throws Exception {
+ super.beforeClass();
+
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+ int n = 2;
+
+ System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
+
+ TestHelper.setupCluster(clusterName, ZK_ADDR, 12918, // participant port
+ "localhost", // participant name prefix
+ "TestDB", // resource name prefix
+ 2, // resources
+ 2, // partitions per resource
+ n, // number of nodes
+ 2, // replicas
+ "MasterSlave", true); // do rebalance
+
+ ClusterControllerManager controller =
+ new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
+ controller.syncStart();
+
+ // start participants
+ MockParticipantManager[] participants = new MockParticipantManager[n];
+ for (int i = 0; i < n; i++) {
+ String instanceName = "localhost_" + (12918 + i);
+
+ participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
+ participants[i].syncStart();
+ }
+
+ INSTANCE_0 = participants[0].getInstanceName();
+ INSTANCE_1 = participants[1].getInstanceName();
+
+ _manager = HelixManagerFactory
+ .getZKHelixManager(clusterName, "admin", InstanceType.ADMINISTRATOR, ZK_ADDR);
+ _manager.connect();
+
+ _spectator = HelixManagerFactory
+ .getZKHelixManager(clusterName, "spectator", InstanceType.SPECTATOR, ZK_ADDR);
+ _spectator.connect();
+
+ // Initialize customized state provider
+ _customizedStateProvider_participant0 = CustomizedStateProviderFactory.getInstance()
+ .buildCustomizedStateProvider(_manager, participants[0].getInstanceName());
+ _customizedStateProvider_participant1 = CustomizedStateProviderFactory.getInstance()
+ .buildCustomizedStateProvider(_manager, participants[1].getInstanceName());
+
+ _localCustomizedView = new HashMap<>();
+ _routingTableProviderDataSources = new HashSet<>();
+ _aggregationEnabledTypes = new HashSet<>();
+ }
+
+ @AfterClass
+ public void afterClass() {
+ _routingTableProvider.shutdown();
+ _manager.disconnect();
+ _spectator.disconnect();
+ }
+
+ /**
+ * Compare the customized view values between ZK and local record
+ * @throws Exception thread interrupted exception
+ */
+ private void validateAggregationSnapshot() throws Exception {
+ boolean result = TestHelper.verify(new TestHelper.Verifier() {
+ @Override
+ public boolean verify() {
+ Map<String, Map<String, RoutingTableSnapshot>> routingTableSnapshots =
+ _routingTableProvider.getRoutingTableSnapshots();
+
+ // Get customized view snapshot
+ Map<String, RoutingTableSnapshot> fullCustomizedViewSnapshot =
+ routingTableSnapshots.get(PropertyType.CUSTOMIZEDVIEW.name());
+ boolean result = false;
+
+ if (fullCustomizedViewSnapshot.isEmpty() && !_routingTableProviderDataSources.isEmpty()) {
+ return false;
+ }
+
+ for (String customizedStateType : fullCustomizedViewSnapshot.keySet()) {
+ if (!_routingTableProviderDataSources.contains(customizedStateType)) {
+ return false;
+ }
+
+ // Get per customized state type snapshot
+ RoutingTableSnapshot customizedViewSnapshot =
+ fullCustomizedViewSnapshot.get(customizedStateType);
+
+ // local per customized state type map
+ Map<String, Map<String, Map<String, String>>> localSnapshot =
+ _localCustomizedView.getOrDefault(customizedStateType, Maps.newHashMap());
+
+ Collection<CustomizedView> customizedViews = customizedViewSnapshot.getCustomizeViews();
+
+ // If a customized state is not set to be aggregated in config, but is enabled in routing table provider, it will show up in customized view returned to user, but will be empty
+ if (!_aggregationEnabledTypes.contains(customizedStateType)
+ && customizedViews.size() != 0) {
+ return false;
+ }
+
+ // Get per resource snapshot
+ for (CustomizedView resourceCustomizedView : customizedViews) {
+ ZNRecord record = resourceCustomizedView.getRecord();
+ Map<String, Map<String, String>> resourceStateMap = record.getMapFields();
+
+ // Get local per resource map
+ Map<String, Map<String, String>> localPerResourceCustomizedView = localSnapshot
+ .getOrDefault(resourceCustomizedView.getResourceName(), Maps.newHashMap());
+
+ if (resourceStateMap.isEmpty() && !localPerResourceCustomizedView.isEmpty()) {
+ return false;
+ }
+
+ // Get per partition snapshot
+ for (String partitionName : resourceStateMap.keySet()) {
+ Map<String, String> stateMap =
+ resourceStateMap.getOrDefault(partitionName, Maps.newTreeMap());
+
+ // Get local per partition map
+ Map<String, String> localStateMap =
+ localPerResourceCustomizedView.getOrDefault(partitionName, Maps.newTreeMap());
+
+ if (stateMap.isEmpty() && !localStateMap.isEmpty()) {
+ return false;
+ }
+
+ for (String instanceName : stateMap.keySet()) {
+ // Per instance value
+ String stateMapValue = stateMap.get(instanceName);
+ String localStateMapValue = localStateMap.get(instanceName);
+ if (isEmptyValue(stateMapValue) && isEmptyValue(localStateMapValue)) {
+ } else if ((!isEmptyValue(stateMapValue) && !isEmptyValue(localStateMapValue)
+ && !stateMapValue.equals(localStateMapValue)) || (isEmptyValue(stateMapValue)
+ || isEmptyValue(localStateMapValue))) {
+ return false;
+ }
+ }
+ }
+ }
+ }
+ return true;
+ }
+ }, 12000);
+
+ Assert.assertTrue(result);
+ }
+
+ private boolean isEmptyValue(String value) {
+ return value == null || value.equals("");
+ }
+
+ /**
+ * Update the local record of customized view
+ * @param instanceName the instance to be updated
+ * @param customizedStateType the customized state type to be updated
+ * @param resourceName the resource to be updated
+ * @param partitionName the partition to be updated
+ * @param customizedStateValue if update, this will be the value to update; a null value indicate delete operation
+ */
+ private void updateLocalCustomizedViewMap(String instanceName,
+ CustomizedStateType customizedStateType, String resourceName, String partitionName,
+ CurrentStateValues customizedStateValue) {
+ _localCustomizedView.putIfAbsent(customizedStateType.name(), new TreeMap<>());
+ Map<String, Map<String, Map<String, String>>> localPerStateType =
+ _localCustomizedView.get(customizedStateType.name());
+ localPerStateType.putIfAbsent(resourceName, new TreeMap<>());
+ Map<String, Map<String, String>> localPerResource = localPerStateType.get(resourceName);
+ localPerResource.putIfAbsent(partitionName, new TreeMap<>());
+ Map<String, String> localPerPartition = localPerResource.get(partitionName);
+ if (customizedStateValue == null) {
+ localPerPartition.remove(instanceName);
+ if (localPerPartition.isEmpty()) {
+ localPerResource.remove(partitionName);
+ if (localPerResource.isEmpty()) {
+ localPerStateType.remove(resourceName);
+ if (localPerStateType.isEmpty()) {
+ _localCustomizedView.remove(customizedStateType.name());
+ }
+ }
+ }
+ } else {
+ localPerPartition.put(instanceName, customizedStateValue.name());
+ }
+ }
+
+ /**
+ * Call this method in the test for an update on customized view in both ZK and local map
+ * @param instanceName the instance to be updated
+ * @param customizedStateType the customized state type to be updated
+ * @param resourceName the resource to be updated
+ * @param partitionName the partition to be updated
+ * @param customizedStateValue if update, this will be the value to update; a null value indicate delete operation
+ * @throws Exception if the input instance name is not valid
+ */
+ private void update(String instanceName, CustomizedStateType customizedStateType,
+ String resourceName, String partitionName, CurrentStateValues customizedStateValue)
+ throws Exception {
+ if (instanceName.equals(INSTANCE_0)) {
+ _customizedStateProvider_participant0
+ .updateCustomizedState(customizedStateType.name(), resourceName, partitionName,
+ customizedStateValue.name());
+ updateLocalCustomizedViewMap(INSTANCE_0, customizedStateType, resourceName, partitionName,
+ customizedStateValue);
+ } else if (instanceName.equals(INSTANCE_1)) {
+ _customizedStateProvider_participant1
+ .updateCustomizedState(customizedStateType.name(), resourceName, partitionName,
+ customizedStateValue.name());
+ updateLocalCustomizedViewMap(INSTANCE_1, customizedStateType, resourceName, partitionName,
+ customizedStateValue);
+ } else {
+ throw new Exception("The input instance name is not valid.");
+ }
+ }
+
+ /**
+ *
+ * Call this method in the test for an delete on customized view in both ZK and local map
+ * @param instanceName the instance to be updated
+ * @param customizedStateType the customized state type to be updated
+ * @param resourceName the resource to be updated
+ * @param partitionName the partition to be updated
+ * @throws Exception if the input instance name is not valid
+ */
+ private void delete(String instanceName, CustomizedStateType customizedStateType,
+ String resourceName, String partitionName) throws Exception {
+ if (instanceName.equals(INSTANCE_0)) {
+ _customizedStateProvider_participant0
+ .deletePerPartitionCustomizedState(customizedStateType.name(), resourceName,
+ partitionName);
+ updateLocalCustomizedViewMap(INSTANCE_0, customizedStateType, resourceName, partitionName,
+ null);
+ } else if (instanceName.equals(INSTANCE_1)) {
+ _customizedStateProvider_participant1
+ .deletePerPartitionCustomizedState(customizedStateType.name(), resourceName,
+ partitionName);
+ updateLocalCustomizedViewMap(INSTANCE_1, customizedStateType, resourceName, partitionName,
+ null);
+ } else {
+ throw new Exception("The input instance name is not valid.");
+ }
+ }
+
+ /**
+ * Set the data sources (customized state types) for routing table provider
+ * @param customizedStateTypes list of customized state types that routing table provider will include in the snapshot shown to users
+ */
+ private void setRoutingTableProviderDataSources(List<CustomizedStateType> customizedStateTypes) {
+ List<String> customizedViewSources = new ArrayList<>();
+ _routingTableProviderDataSources.clear();
+ for (CustomizedStateType type : customizedStateTypes) {
+ customizedViewSources.add(type.name());
+ _routingTableProviderDataSources.add(type.name());
+ }
+ Map<PropertyType, List<String>> dataSource = new HashMap<>();
+ dataSource.put(PropertyType.CUSTOMIZEDVIEW, customizedViewSources);
+ _routingTableProvider = new RoutingTableProvider(_spectator, dataSource);
+ }
+
+ /**
+ * Set the customized view aggregation config in controller
+ * @param aggregationEnabledTypes list of customized state types that the controller will aggregate to customized view
+ */
+ private void setAggregationEnabledTypes(List<CustomizedStateType> aggregationEnabledTypes) {
+ List<String> enabledTypes = new ArrayList<>();
+ _aggregationEnabledTypes.clear();
+ for (CustomizedStateType type : aggregationEnabledTypes) {
+ enabledTypes.add(type.name());
+ _aggregationEnabledTypes.add(type.name());
+ }
+ CustomizedStateConfig.Builder customizedStateConfigBuilder =
+ new CustomizedStateConfig.Builder();
+ customizedStateConfigBuilder.setAggregationEnabledTypes(enabledTypes);
+ HelixDataAccessor accessor = _manager.getHelixDataAccessor();
+ accessor.setProperty(accessor.keyBuilder().customizedStateConfig(),
+ customizedStateConfigBuilder.build());
+ }
+
+ @Test
+ public void testCustomizedStateViewAggregation() throws Exception {
+ setAggregationEnabledTypes(
+ Arrays.asList(CustomizedStateType.TYPE_A, CustomizedStateType.TYPE_B));
+
+ update(INSTANCE_0, CustomizedStateType.TYPE_A, RESOURCE_0, PARTITION_00,
+ CurrentStateValues.TYPE_A_0);
+ update(INSTANCE_0, CustomizedStateType.TYPE_B, RESOURCE_0, PARTITION_00,
+ CurrentStateValues.TYPE_B_0);
+ update(INSTANCE_0, CustomizedStateType.TYPE_B, RESOURCE_0, PARTITION_01,
+ CurrentStateValues.TYPE_B_1);
+ update(INSTANCE_0, CustomizedStateType.TYPE_A, RESOURCE_1, PARTITION_11,
+ CurrentStateValues.TYPE_A_1);
+ update(INSTANCE_1, CustomizedStateType.TYPE_C, RESOURCE_0, PARTITION_00,
+ CurrentStateValues.TYPE_C_0);
+ update(INSTANCE_1, CustomizedStateType.TYPE_C, RESOURCE_0, PARTITION_01,
+ CurrentStateValues.TYPE_C_1);
+ update(INSTANCE_1, CustomizedStateType.TYPE_B, RESOURCE_1, PARTITION_10,
+ CurrentStateValues.TYPE_B_2);
+ update(INSTANCE_1, CustomizedStateType.TYPE_C, RESOURCE_1, PARTITION_10,
+ CurrentStateValues.TYPE_C_2);
+ update(INSTANCE_1, CustomizedStateType.TYPE_A, RESOURCE_1, PARTITION_11,
+ CurrentStateValues.TYPE_A_1);
+
+ // Aggregation enabled types: A, B; Routing table provider data sources: A, B, C; should show TypeA, TypeB customized views
+ setRoutingTableProviderDataSources(Arrays
+ .asList(CustomizedStateType.TYPE_A, CustomizedStateType.TYPE_B,
+ CustomizedStateType.TYPE_C));
+ validateAggregationSnapshot();
+
+ // Aggregation enabled types: A, B; Routing table provider data sources: A, B, C; should show TypeA, TypeB customized views
+ setAggregationEnabledTypes(Arrays.asList(CustomizedStateType.TYPE_A, CustomizedStateType.TYPE_B,
+ CustomizedStateType.TYPE_C));
+ validateAggregationSnapshot();
+
+ Assert.assertNull(_customizedStateProvider_participant0
+ .getCustomizedState(CustomizedStateType.TYPE_C.name(), RESOURCE_0));
+
+ // Test batch update API to update several customized state fields in the same customized state, but for now only CURRENT_STATE will be aggregated in customized view
+ Map<String, String> customizedStates = Maps.newHashMap();
+ customizedStates.put("CURRENT_STATE", CurrentStateValues.TYPE_A_2.name());
+ customizedStates.put("PREVIOUS_STATE", CurrentStateValues.TYPE_A_0.name());
+ _customizedStateProvider_participant1
+ .updateCustomizedState(CustomizedStateType.TYPE_A.name(), RESOURCE_1, PARTITION_10,
+ customizedStates);
+ updateLocalCustomizedViewMap(INSTANCE_1, CustomizedStateType.TYPE_A, RESOURCE_1, PARTITION_10,
+ CurrentStateValues.TYPE_A_2);
+
+ // Aggregation enabled types: A, B, C; Routing table provider data sources: A; should only show TypeA customized view
+ setRoutingTableProviderDataSources(Arrays.asList(CustomizedStateType.TYPE_A));
+ validateAggregationSnapshot();
+
+ // Test get customized state and get per partition customized state via customized state provider, this part of test doesn't change customized view
+ CustomizedState customizedState = _customizedStateProvider_participant1
+ .getCustomizedState(CustomizedStateType.TYPE_A.name(), RESOURCE_1);
+ Assert.assertEquals(customizedState.getState(PARTITION_10), CurrentStateValues.TYPE_A_2.name());
+ Assert.assertEquals(customizedState.getPreviousState(PARTITION_10),
+ CurrentStateValues.TYPE_A_0.name());
+ Assert.assertEquals(customizedState.getState(PARTITION_11), CurrentStateValues.TYPE_A_1.name());
+ Map<String, String> perPartitionCustomizedState = _customizedStateProvider_participant1
+ .getPerPartitionCustomizedState(CustomizedStateType.TYPE_A.name(), RESOURCE_1,
+ PARTITION_10);
+ Map<String, String> actualPerPartitionCustomizedState = Maps.newHashMap();
+ actualPerPartitionCustomizedState
+ .put(CustomizedState.CustomizedStateProperty.CURRENT_STATE.name(),
+ CurrentStateValues.TYPE_A_2.name());
+ actualPerPartitionCustomizedState
+ .put(CustomizedState.CustomizedStateProperty.PREVIOUS_STATE.name(),
+ CurrentStateValues.TYPE_A_0.name());
+ Assert.assertEquals(perPartitionCustomizedState, actualPerPartitionCustomizedState);
+
+ // Test delete per partition customized state via customized state provider.
+ _customizedStateProvider_participant1
+ .deletePerPartitionCustomizedState(CustomizedStateType.TYPE_A.name(), RESOURCE_1,
+ PARTITION_10);
+ customizedState = _customizedStateProvider_participant1
+ .getCustomizedState(CustomizedStateType.TYPE_A.name(), RESOURCE_1);
+ Assert.assertEquals(customizedState.getState(PARTITION_11), CurrentStateValues.TYPE_A_1.name());
+ Assert.assertNull(_customizedStateProvider_participant1
+ .getPerPartitionCustomizedState(CustomizedStateType.TYPE_A.name(), RESOURCE_1,
+ PARTITION_10));
+ // Customized view only reflect CURRENT_STATE field
+ updateLocalCustomizedViewMap(INSTANCE_1, CustomizedStateType.TYPE_A, RESOURCE_1, PARTITION_10,
+ null);
+ validateAggregationSnapshot();
+
+ //Aggregation enabled types: B; Routing table provider data sources: A, B, C; should show TypeB customized views
+ setAggregationEnabledTypes(Arrays.asList(CustomizedStateType.TYPE_B));
+ setRoutingTableProviderDataSources(Arrays
+ .asList(CustomizedStateType.TYPE_A, CustomizedStateType.TYPE_B,
+ CustomizedStateType.TYPE_C));
+ validateAggregationSnapshot();
+
+ //Aggregation enabled types: B; Routing table provider data sources: A; should show empty customized view
+ setRoutingTableProviderDataSources(Arrays.asList(CustomizedStateType.TYPE_A));
+ validateAggregationSnapshot();
+
+ // Update some customized states and verify
+ delete(INSTANCE_0, CustomizedStateType.TYPE_A, RESOURCE_0, PARTITION_00);
+ delete(INSTANCE_1, CustomizedStateType.TYPE_B, RESOURCE_1, PARTITION_10);
+
+ // delete a customize state that does not exist
+ delete(INSTANCE_1, CustomizedStateType.TYPE_A, RESOURCE_1, PARTITION_10);
+ validateAggregationSnapshot();
+
+ update(INSTANCE_0, CustomizedStateType.TYPE_B, RESOURCE_0, PARTITION_01,
+ CurrentStateValues.TYPE_B_2);
+ update(INSTANCE_1, CustomizedStateType.TYPE_B, RESOURCE_1, PARTITION_10,
+ CurrentStateValues.TYPE_B_1);
+ update(INSTANCE_1, CustomizedStateType.TYPE_C, RESOURCE_1, PARTITION_10,
+ CurrentStateValues.TYPE_C_0);
+ update(INSTANCE_0, CustomizedStateType.TYPE_A, RESOURCE_1, PARTITION_11,
+ CurrentStateValues.TYPE_A_0);
+ validateAggregationSnapshot();
+ }
+}
[helix] 08/23: Improve CustomizedStateProvider tests (#840)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 5074c1579d6601d05dff10dc7fff826d77dcb082
Author: mgao0 <31...@users.noreply.github.com>
AuthorDate: Tue Mar 3 10:56:16 2020 -0800
Improve CustomizedStateProvider tests (#840)
Add tests to make CustomizedStateProvider tests comprehensive.
---
.../paticipant/TestCustomizedStateUpdate.java | 224 ++++++++++++++++++---
1 file changed, 195 insertions(+), 29 deletions(-)
diff --git a/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestCustomizedStateUpdate.java b/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestCustomizedStateUpdate.java
index e0553cd..bc086ba 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestCustomizedStateUpdate.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestCustomizedStateUpdate.java
@@ -19,13 +19,22 @@ package org.apache.helix.integration.paticipant;
* under the License.
*/
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.stream.Collectors;
+
import org.apache.helix.HelixDataAccessor;
import org.apache.helix.HelixManager;
import org.apache.helix.HelixManagerFactory;
import org.apache.helix.InstanceType;
import org.apache.helix.PropertyKey;
+import org.apache.helix.TestHelper;
import org.apache.helix.customizedstate.CustomizedStateProvider;
import org.apache.helix.customizedstate.CustomizedStateProviderFactory;
import org.apache.helix.integration.common.ZkStandAloneCMTestBase;
@@ -33,40 +42,61 @@ import org.apache.helix.model.CustomizedState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
+
public class TestCustomizedStateUpdate extends ZkStandAloneCMTestBase {
private static Logger LOG = LoggerFactory.getLogger(TestCustomizedStateUpdate.class);
private final String CUSTOMIZE_STATE_NAME = "testState1";
private final String PARTITION_NAME1 = "testPartition1";
private final String PARTITION_NAME2 = "testPartition2";
private final String RESOURCE_NAME = "testResource1";
+ private final String PARTITION_STATE = "partitionState";
+ private static HelixManager _manager;
+ private static CustomizedStateProvider _mockProvider;
- @Test
- public void testUpdateCustomizedState() throws Exception {
- HelixManager manager = HelixManagerFactory.getZKHelixManager(CLUSTER_NAME, "admin",
- InstanceType.ADMINISTRATOR, ZK_ADDR);
- manager.connect();
+ @BeforeClass
+ public void beforeClass() throws Exception {
+ super.beforeClass();
+ _manager = HelixManagerFactory
+ .getZKHelixManager(CLUSTER_NAME, "admin", InstanceType.ADMINISTRATOR, ZK_ADDR);
+ _manager.connect();
_participants[0].connect();
+ _mockProvider = CustomizedStateProviderFactory.getInstance()
+ .buildCustomizedStateProvider(_manager, _participants[0].getInstanceName());
+ }
+
+ @AfterClass
+ public void afterClass() throws Exception {
+ super.afterClass();
+ _manager.disconnect();
+ }
- HelixDataAccessor dataAccessor = manager.getHelixDataAccessor();
+ @BeforeMethod
+ public void beforeMethod() {
+ HelixDataAccessor dataAccessor = _manager.getHelixDataAccessor();
PropertyKey propertyKey = dataAccessor.keyBuilder()
.customizedStates(_participants[0].getInstanceName(), CUSTOMIZE_STATE_NAME);
- CustomizedState customizedStates = manager.getHelixDataAccessor().getProperty(propertyKey);
+ dataAccessor.removeProperty(propertyKey);
+ CustomizedState customizedStates = dataAccessor.getProperty(propertyKey);
Assert.assertNull(customizedStates);
+ }
- CustomizedStateProvider mockProvider = CustomizedStateProviderFactory.getInstance()
- .buildCustomizedStateProvider(manager, _participants[0].getInstanceName());
+ @Test
+ public void testUpdateCustomizedState() {
// test adding customized state for a partition
Map<String, String> customizedStateMap = new HashMap<>();
customizedStateMap.put("PREVIOUS_STATE", "STARTED");
customizedStateMap.put("CURRENT_STATE", "END_OF_PUSH_RECEIVED");
- mockProvider.updateCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME1,
+ _mockProvider.updateCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME1,
customizedStateMap);
CustomizedState customizedState =
- mockProvider.getCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME);
+ _mockProvider.getCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME);
Assert.assertNotNull(customizedState);
Assert.assertEquals(customizedState.getId(), RESOURCE_NAME);
Map<String, Map<String, String>> mapView = customizedState.getRecord().getMapFields();
@@ -79,10 +109,10 @@ public class TestCustomizedStateUpdate extends ZkStandAloneCMTestBase {
// test partial update customized state for previous partition
Map<String, String> stateMap1 = new HashMap<>();
stateMap1.put("PREVIOUS_STATE", "END_OF_PUSH_RECEIVED");
- mockProvider.updateCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME1,
- stateMap1);
+ _mockProvider
+ .updateCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME1, stateMap1);
- customizedState = mockProvider.getCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME);
+ customizedState = _mockProvider.getCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME);
Assert.assertNotNull(customizedState);
Assert.assertEquals(customizedState.getId(), RESOURCE_NAME);
mapView = customizedState.getRecord().getMapFields();
@@ -96,10 +126,10 @@ public class TestCustomizedStateUpdate extends ZkStandAloneCMTestBase {
stateMap1 = new HashMap<>();
stateMap1.put("PREVIOUS_STATE", "END_OF_PUSH_RECEIVED");
stateMap1.put("CURRENT_STATE", "COMPLETED");
- mockProvider.updateCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME1,
- stateMap1);
+ _mockProvider
+ .updateCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME1, stateMap1);
- customizedState = mockProvider.getCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME);
+ customizedState = _mockProvider.getCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME);
Assert.assertNotNull(customizedState);
Assert.assertEquals(customizedState.getId(), RESOURCE_NAME);
mapView = customizedState.getRecord().getMapFields();
@@ -113,40 +143,176 @@ public class TestCustomizedStateUpdate extends ZkStandAloneCMTestBase {
Map<String, String> stateMap2 = new HashMap<>();
stateMap2.put("PREVIOUS_STATE", "STARTED");
stateMap2.put("CURRENT_STATE", "END_OF_PUSH_RECEIVED");
- mockProvider.updateCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME2,
- stateMap2);
+ _mockProvider
+ .updateCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME2, stateMap2);
- customizedState = mockProvider.getCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME);
+ customizedState = _mockProvider.getCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME);
Assert.assertNotNull(customizedState);
Assert.assertEquals(customizedState.getId(), RESOURCE_NAME);
mapView = customizedState.getRecord().getMapFields();
Assert.assertEquals(mapView.keySet().size(), 2);
- Assert.assertEqualsNoOrder(mapView.keySet().toArray(), new String[] {
- PARTITION_NAME1, PARTITION_NAME2
- });
+ Assert.assertEqualsNoOrder(mapView.keySet().toArray(),
+ new String[]{PARTITION_NAME1, PARTITION_NAME2});
- Map<String, String> partitionMap1 = mockProvider
+ Map<String, String> partitionMap1 = _mockProvider
.getPerPartitionCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME1);
Assert.assertEquals(partitionMap1.keySet().size(), 2);
Assert.assertEquals(partitionMap1.get("PREVIOUS_STATE"), "END_OF_PUSH_RECEIVED");
Assert.assertEquals(partitionMap1.get("CURRENT_STATE"), "COMPLETED");
- Map<String, String> partitionMap2 = mockProvider
+ Map<String, String> partitionMap2 = _mockProvider
.getPerPartitionCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME2);
Assert.assertEquals(partitionMap2.keySet().size(), 2);
Assert.assertEquals(partitionMap2.get("PREVIOUS_STATE"), "STARTED");
Assert.assertEquals(partitionMap2.get("CURRENT_STATE"), "END_OF_PUSH_RECEIVED");
// test delete customized state for a partition
- mockProvider.deletePerPartitionCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME,
- PARTITION_NAME1);
- customizedState = mockProvider.getCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME);
+ _mockProvider
+ .deletePerPartitionCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME1);
+ customizedState = _mockProvider.getCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME);
Assert.assertNotNull(customizedState);
Assert.assertEquals(customizedState.getId(), RESOURCE_NAME);
mapView = customizedState.getRecord().getMapFields();
Assert.assertEquals(mapView.keySet().size(), 1);
Assert.assertEquals(mapView.keySet().iterator().next(), PARTITION_NAME2);
+ }
+
+ @Test
+ public void testUpdateSinglePartitionCustomizedState() {
+ _mockProvider.updateCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME1,
+ PARTITION_STATE);
+
+ // get customized state
+ CustomizedState customizedState =
+ _mockProvider.getCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME);
+ Assert.assertEquals(
+ customizedState.getPartitionStateMap(CustomizedState.CustomizedStateProperty.CURRENT_STATE)
+ .size(), 1);
+ Map<String, String> map = new HashMap<>();
+ map.put(PARTITION_NAME1, null);
+ Assert.assertEquals(customizedState
+ .getPartitionStateMap(CustomizedState.CustomizedStateProperty.PREVIOUS_STATE), map);
+ Assert.assertEquals(
+ customizedState.getPartitionStateMap(CustomizedState.CustomizedStateProperty.START_TIME),
+ map);
+ Assert.assertEquals(
+ customizedState.getPartitionStateMap(CustomizedState.CustomizedStateProperty.END_TIME),
+ map);
+ Assert.assertEquals(customizedState.getState(PARTITION_NAME1), PARTITION_STATE);
+ Assert.assertNull(customizedState.getState(PARTITION_NAME2));
+ Assert.assertTrue(customizedState.isValid());
+
+ // get per partition customized state
+ map = new HashMap<>();
+ map.put(CustomizedState.CustomizedStateProperty.CURRENT_STATE.name(), PARTITION_STATE);
+ Map<String, String> partitionCustomizedState = _mockProvider
+ .getPerPartitionCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME1);
+ Assert.assertEquals(partitionCustomizedState, map);
+ Assert.assertNull(_mockProvider
+ .getPerPartitionCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME2));
+ }
+
+ @Test
+ public void testUpdateSinglePartitionCustomizedStateWithNullField() {
+ _mockProvider
+ .updateCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME1, (String) null);
+
+ // get customized state
+ CustomizedState customizedState =
+ _mockProvider.getCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME);
+ Map<String, String> map = new HashMap<>();
+ map.put(PARTITION_NAME1, null);
+ Assert.assertEquals(
+ customizedState.getPartitionStateMap(CustomizedState.CustomizedStateProperty.CURRENT_STATE),
+ map);
+ Assert.assertEquals(customizedState.getState(PARTITION_NAME1), null);
+ Assert.assertTrue(customizedState.isValid());
+
+ // get per partition customized state
+ map = new HashMap<>();
+ map.put(CustomizedState.CustomizedStateProperty.CURRENT_STATE.name(), null);
+ Map<String, String> partitionCustomizedState = _mockProvider
+ .getPerPartitionCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME1);
+ Assert.assertEquals(partitionCustomizedState, map);
+ Assert.assertNull(_mockProvider
+ .getPerPartitionCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME2));
+ }
+
+ @Test
+ public void testUpdateCustomizedStateWithEmptyMap() {
+ _mockProvider.updateCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME1,
+ new HashMap<>());
+
+ // get customized state
+ CustomizedState customizedState =
+ _mockProvider.getCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME);
+ Assert.assertNull(customizedState.getState(PARTITION_NAME1));
+ Map<String, String> partitionStateMap =
+ customizedState.getPartitionStateMap(CustomizedState.CustomizedStateProperty.CURRENT_STATE);
+ Assert.assertNotNull(partitionStateMap);
+ Assert.assertTrue(partitionStateMap.containsKey(PARTITION_NAME1));
+ Assert.assertNull(partitionStateMap.get(PARTITION_NAME1));
+ Assert.assertNull(customizedState.getState(PARTITION_NAME1));
+ Assert.assertFalse(partitionStateMap.containsKey(PARTITION_NAME2));
+ Assert.assertTrue(customizedState.isValid());
+
+ // get per partition customized state
+ Map<String, String> partitionCustomizedState = _mockProvider
+ .getPerPartitionCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME1);
+ Assert.assertEquals(partitionCustomizedState.size(), 0);
+ Assert.assertNull(_mockProvider
+ .getPerPartitionCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME2));
+ }
+
+ @Test
+ public void testDeleteNonExistingPerPartitionCustomizedState() {
+ _mockProvider.updateCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME1,
+ PARTITION_STATE);
+ _mockProvider
+ .deletePerPartitionCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME2);
+ Assert.assertNotNull(_mockProvider
+ .getPerPartitionCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME1));
+ Assert.assertNull(_mockProvider
+ .getPerPartitionCustomizedState(CUSTOMIZE_STATE_NAME, RESOURCE_NAME, PARTITION_NAME2));
+ }
+
+ @Test
+ public void testSimultaneousUpdateCustomizedState() {
+ int n = 10;
+
+ List<Callable<Boolean>> threads = new ArrayList<>();
+ for (int i = 0; i < n; i++) {
+ threads.add(new TestSimultaneousUpdate());
+ }
+ Map<String, Boolean> resultMap = TestHelper.startThreadsConcurrently(threads, 1000);
+ Assert.assertEquals(resultMap.size(), n);
+ Boolean[] results = new Boolean[n];
+ Arrays.fill(results, true);
+ Assert.assertEqualsNoOrder(resultMap.values().toArray(), results);
+ }
+
+ private static class TestSimultaneousUpdate implements Callable<Boolean> {
+ private Random rand = new Random();
- manager.disconnect();
+ @Override
+ public Boolean call() {
+ String customizedStateName = "testState";
+ String resourceName = "resource" + String.valueOf(rand.nextInt(10));
+ String partitionName = "partition" + String.valueOf(rand.nextInt(10));
+ String partitionState = "Updated";
+ try {
+ _mockProvider.updateCustomizedState(customizedStateName, resourceName, partitionName,
+ partitionState);
+ } catch (Exception e) {
+ return false;
+ }
+ Map<String, String> states = _mockProvider
+ .getPerPartitionCustomizedState(customizedStateName, resourceName, partitionName);
+ if (states == null) {
+ return false;
+ }
+ return states.get(CustomizedState.CustomizedStateProperty.CURRENT_STATE.name())
+ .equals(partitionState);
+ }
}
}
[helix] 10/23: rename custmized state aggregation config to
customized state config (#885)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 21a47c5473f14f70d9e1419ffb522aa3905bb49a
Author: zhangmeng916 <56...@users.noreply.github.com>
AuthorDate: Tue Mar 10 22:45:05 2020 -0700
rename custmized state aggregation config to customized state config (#885)
Rename customized state aggregation config to customized state config for future extendibility.
Co-authored-by: Meng Zhang <mn...@mnzhang-mn1.linkedin.biz>
---
.../main/java/org/apache/helix/ConfigAccessor.java | 10 +--
.../src/main/java/org/apache/helix/HelixAdmin.java | 22 +++----
.../main/java/org/apache/helix/HelixConstants.java | 2 +-
.../main/java/org/apache/helix/HelixManager.java | 10 +--
.../main/java/org/apache/helix/PropertyKey.java | 6 +-
...va => CustomizedStateConfigChangeListener.java} | 8 +--
.../helix/common/caches/CustomizedStateCache.java | 6 +-
.../org/apache/helix/manager/zk/ZKHelixAdmin.java | 56 ++++++++--------
.../apache/helix/manager/zk/ZKHelixManager.java | 10 +--
...ationConfig.java => CustomizedStateConfig.java} | 58 ++++++++---------
.../controller/stages/DummyClusterManager.java | 4 +-
.../apache/helix/manager/zk/TestZkHelixAdmin.java | 9 ++-
.../java/org/apache/helix/mock/MockHelixAdmin.java | 12 ++--
.../java/org/apache/helix/mock/MockManager.java | 4 +-
...nConfig.java => TestCustomizedStateConfig.java} | 74 +++++++++++-----------
.../helix/participant/MockZKHelixManager.java | 4 +-
.../server/resources/helix/ClusterAccessor.java | 36 +++++------
.../helix/rest/server/TestClusterAccessor.java | 5 +-
18 files changed, 168 insertions(+), 168 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
index 63f232e..30a9a8f 100644
--- a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
+++ b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
@@ -33,7 +33,7 @@ import org.apache.helix.manager.zk.ZKUtil;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ConfigScope;
-import org.apache.helix.model.CustomizedStateAggregationConfig;
+import org.apache.helix.model.CustomizedStateConfig;
import org.apache.helix.model.HelixConfigScope;
import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
import org.apache.helix.model.InstanceConfig;
@@ -589,11 +589,11 @@ public class ConfigAccessor {
}
/**
- * Get CustomizedStateAggregationConfig of the given cluster.
+ * Get CustomizedStateConfig of the given cluster.
* @param clusterName
- * @return The instance of {@link CustomizedStateAggregationConfig}
+ * @return The instance of {@link CustomizedStateConfig}
*/
- public CustomizedStateAggregationConfig getCustomizedStateAggregationConfig(String clusterName) {
+ public CustomizedStateConfig getCustomizedStateConfig(String clusterName) {
if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
throw new HelixException(String.format("Failed to get config. cluster: %s is not setup.", clusterName));
}
@@ -606,7 +606,7 @@ public class ConfigAccessor {
return null;
}
- return new CustomizedStateAggregationConfig.Builder(record).build();
+ return new CustomizedStateConfig.Builder(record).build();
}
/**
diff --git a/helix-core/src/main/java/org/apache/helix/HelixAdmin.java b/helix-core/src/main/java/org/apache/helix/HelixAdmin.java
index 985d00f..11b8667 100644
--- a/helix-core/src/main/java/org/apache/helix/HelixAdmin.java
+++ b/helix-core/src/main/java/org/apache/helix/HelixAdmin.java
@@ -26,7 +26,7 @@ import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ClusterConstraints.ConstraintType;
import org.apache.helix.model.ConstraintItem;
-import org.apache.helix.model.CustomizedStateAggregationConfig;
+import org.apache.helix.model.CustomizedStateConfig;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.HelixConfigScope;
import org.apache.helix.model.IdealState;
@@ -106,30 +106,30 @@ public interface HelixAdmin {
*/
void addClusterToGrandCluster(String clusterName, String grandCluster);
- /** Add a CustomizedStateAggregationConfig to a cluster
+ /** Add a CustomizedStateConfig to a cluster
* @param clusterName
- * @param customizedStateAggregationConfig
+ * @param customizedStateConfig
*/
- void addCustomizedStateAggregationConfig(String clusterName,
- CustomizedStateAggregationConfig customizedStateAggregationConfig);
+ void addCustomizedStateConfig(String clusterName,
+ CustomizedStateConfig customizedStateConfig);
/**
- * Remove CustomizedStateAggregationConfig from specific cluster
+ * Remove CustomizedStateConfig from specific cluster
* @param clusterName
*/
- void removeCustomizedStateAggregationConfig(String clusterName);
+ void removeCustomizedStateConfig(String clusterName);
/**
- * Add a type to CustomizedStateAggregationConfig of specific cluster
+ * Add a type to CustomizedStateConfig of specific cluster
* @param clusterName
*/
- void addTypeToCustomizedStateAggregationConfig(String clusterName, String type);
+ void addTypeToCustomizedStateConfig(String clusterName, String type);
/**
- * Remove a type from CustomizedStateAggregationConfig of specific cluster
+ * Remove a type from CustomizedStateConfig of specific cluster
* @param clusterName
*/
- void removeTypeFromCustomizedStateAggregationConfig(String clusterName, String type);
+ void removeTypeFromCustomizedStateConfig(String clusterName, String type);
/**
* Add a resource to a cluster, using the default ideal state mode AUTO
diff --git a/helix-core/src/main/java/org/apache/helix/HelixConstants.java b/helix-core/src/main/java/org/apache/helix/HelixConstants.java
index 27f82c6..c71f340 100644
--- a/helix-core/src/main/java/org/apache/helix/HelixConstants.java
+++ b/helix-core/src/main/java/org/apache/helix/HelixConstants.java
@@ -30,7 +30,7 @@ public interface HelixConstants {
CONFIG (PropertyType.CONFIGS),
INSTANCE_CONFIG (PropertyType.CONFIGS),
RESOURCE_CONFIG (PropertyType.CONFIGS),
- CUSTOMIZED_STATE_AGGREGATION_CONFIG (PropertyType.CONFIGS),
+ CUSTOMIZED_STATE_CONFIG (PropertyType.CONFIGS),
CLUSTER_CONFIG (PropertyType.CONFIGS),
LIVE_INSTANCE (PropertyType.LIVEINSTANCES),
CURRENT_STATE (PropertyType.CURRENTSTATES),
diff --git a/helix-core/src/main/java/org/apache/helix/HelixManager.java b/helix-core/src/main/java/org/apache/helix/HelixManager.java
index a499b14..30f6126 100644
--- a/helix-core/src/main/java/org/apache/helix/HelixManager.java
+++ b/helix-core/src/main/java/org/apache/helix/HelixManager.java
@@ -26,7 +26,7 @@ import org.apache.helix.api.listeners.ClusterConfigChangeListener;
import org.apache.helix.api.listeners.ConfigChangeListener;
import org.apache.helix.api.listeners.ControllerChangeListener;
import org.apache.helix.api.listeners.CurrentStateChangeListener;
-import org.apache.helix.api.listeners.CustomizedStateAggregationConfigChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateConfigChangeListener;
import org.apache.helix.api.listeners.CustomizedStateChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.CustomizedViewChangeListener;
@@ -41,7 +41,7 @@ import org.apache.helix.controller.pipeline.Pipeline;
import org.apache.helix.healthcheck.ParticipantHealthReportCollector;
import org.apache.helix.manager.zk.ZKHelixManager;
import org.apache.helix.model.ClusterConfig;
-import org.apache.helix.model.CustomizedStateAggregationConfig;
+import org.apache.helix.model.CustomizedStateConfig;
import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
import org.apache.helix.participant.HelixStateMachineEngine;
import org.apache.helix.participant.StateMachineEngine;
@@ -161,13 +161,13 @@ public interface HelixManager {
void addResourceConfigChangeListener(ResourceConfigChangeListener listener) throws Exception;
/**
- * @see CustomizedStateAggregationConfigChangeListener#onCustomizedStateAggregationConfigChange(CustomizedStateAggregationConfig,
+ * @see CustomizedStateConfigChangeListener#onCustomizedStateConfigChange(CustomizedStateConfig,
* NotificationContext)
* @param listener
*/
- void addCustomizedStateAggregationConfigChangeListener(
- CustomizedStateAggregationConfigChangeListener listener) throws Exception;
+ void addCustomizedStateConfigChangeListener(
+ CustomizedStateConfigChangeListener listener) throws Exception;
/**
* @see ClusterConfigChangeListener#onClusterConfigChange(ClusterConfig, NotificationContext)
diff --git a/helix-core/src/main/java/org/apache/helix/PropertyKey.java b/helix-core/src/main/java/org/apache/helix/PropertyKey.java
index fc343a8..29e8d65 100644
--- a/helix-core/src/main/java/org/apache/helix/PropertyKey.java
+++ b/helix-core/src/main/java/org/apache/helix/PropertyKey.java
@@ -27,7 +27,7 @@ import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ControllerHistory;
import org.apache.helix.model.CurrentState;
-import org.apache.helix.model.CustomizedStateAggregationConfig;
+import org.apache.helix.model.CustomizedStateConfig;
import org.apache.helix.model.CustomizedState;
import org.apache.helix.model.CustomizedView;
import org.apache.helix.model.Error;
@@ -254,9 +254,9 @@ public class PropertyKey {
* Get a property key associated with this customized state aggregation configuration
* @return {@link PropertyKey}
*/
- public PropertyKey customizedStateAggregationConfig() {
+ public PropertyKey customizedStateConfig() {
return new PropertyKey(CONFIGS, ConfigScopeProperty.CUSTOMIZED_STATE_AGGREGATION,
- CustomizedStateAggregationConfig.class, _clusterName,
+ CustomizedStateConfig.class, _clusterName,
ConfigScopeProperty.CUSTOMIZED_STATE_AGGREGATION.name(), _clusterName);
}
diff --git a/helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedStateAggregationConfigChangeListener.java b/helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedStateConfigChangeListener.java
similarity index 79%
rename from helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedStateAggregationConfigChangeListener.java
rename to helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedStateConfigChangeListener.java
index 2617ef3..4d31d87 100644
--- a/helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedStateAggregationConfigChangeListener.java
+++ b/helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedStateConfigChangeListener.java
@@ -20,17 +20,17 @@ package org.apache.helix.api.listeners;
*/
import org.apache.helix.NotificationContext;
-import org.apache.helix.model.CustomizedStateAggregationConfig;
+import org.apache.helix.model.CustomizedStateConfig;
/**
* Interface to implement to listen for changes to customized state aggregation configurations.
*/
-public interface CustomizedStateAggregationConfigChangeListener {
+public interface CustomizedStateConfigChangeListener {
/**
* Invoked when customized state aggregation config changes
- * @param customizedStateAggregationConfig
+ * @param customizedStateConfig
* @param context
*/
- void onCustomizedStateAggregationConfigChange(CustomizedStateAggregationConfig customizedStateAggregationConfig,
+ void onCustomizedStateConfigChange(CustomizedStateConfig customizedStateConfig,
NotificationContext context);
}
\ No newline at end of file
diff --git a/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedStateCache.java b/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedStateCache.java
index 1f33948..2964b13 100644
--- a/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedStateCache.java
+++ b/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedStateCache.java
@@ -27,7 +27,7 @@ import org.apache.helix.HelixDataAccessor;
import org.apache.helix.PropertyKey;
import org.apache.helix.common.controllers.ControlContextProvider;
import org.apache.helix.model.CustomizedState;
-import org.apache.helix.model.CustomizedStateAggregationConfig;
+import org.apache.helix.model.CustomizedStateConfig;
import org.apache.helix.model.LiveInstance;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,9 +50,9 @@ public class CustomizedStateCache extends ParticipantStateCache<CustomizedState>
Set<PropertyKey> participantStateKeys = new HashSet<>();
PropertyKey.Builder keyBuilder = accessor.keyBuilder();
Set<String> restrictedKeys = new HashSet<>(
- accessor.getProperty(accessor.keyBuilder().customizedStateAggregationConfig()).getRecord()
+ accessor.getProperty(accessor.keyBuilder().customizedStateConfig()).getRecord()
.getListFields().get(
- CustomizedStateAggregationConfig.CustomizedStateAggregationProperty.AGGREGATION_ENABLED_TYPES
+ CustomizedStateConfig.CustomizedStateProperty.AGGREGATION_ENABLED_TYPES
.name()));
for (String instanceName : liveInstanceMap.keySet()) {
for (String customizedStateType : restrictedKeys) {
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
index 3eb4d55..7ece2c7 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
@@ -63,7 +63,7 @@ import org.apache.helix.model.ClusterConstraints.ConstraintType;
import org.apache.helix.model.ConstraintItem;
import org.apache.helix.model.ControllerHistory;
import org.apache.helix.model.CurrentState;
-import org.apache.helix.model.CustomizedStateAggregationConfig;
+import org.apache.helix.model.CustomizedStateConfig;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.HelixConfigScope;
import org.apache.helix.model.IdealState;
@@ -1217,84 +1217,84 @@ public class ZKHelixAdmin implements HelixAdmin {
}
@Override
- public void addCustomizedStateAggregationConfig(String clusterName,
- CustomizedStateAggregationConfig customizedStateAggregationConfig) {
+ public void addCustomizedStateConfig(String clusterName,
+ CustomizedStateConfig customizedStateConfig) {
logger.info(
- "Add CustomizedStateAggregationConfig to cluster {}, CustomizedStateAggregationConfig is {}",
- clusterName, customizedStateAggregationConfig.toString());
+ "Add CustomizedStateConfig to cluster {}, CustomizedStateConfig is {}",
+ clusterName, customizedStateConfig.toString());
if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
throw new HelixException("cluster " + clusterName + " is not setup yet");
}
- CustomizedStateAggregationConfig.Builder builder =
- new CustomizedStateAggregationConfig.Builder(customizedStateAggregationConfig);
- CustomizedStateAggregationConfig customizedStateAggregationConfigFromBuilder = builder.build();
+ CustomizedStateConfig.Builder builder =
+ new CustomizedStateConfig.Builder(customizedStateConfig);
+ CustomizedStateConfig customizedStateConfigFromBuilder = builder.build();
ZKHelixDataAccessor accessor =
new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
Builder keyBuilder = accessor.keyBuilder();
- accessor.setProperty(keyBuilder.customizedStateAggregationConfig(),
- customizedStateAggregationConfigFromBuilder);
+ accessor.setProperty(keyBuilder.customizedStateConfig(),
+ customizedStateConfigFromBuilder);
}
@Override
- public void removeCustomizedStateAggregationConfig(String clusterName) {
+ public void removeCustomizedStateConfig(String clusterName) {
logger.info(
- "Remove CustomizedStateAggregationConfig from cluster {}.", clusterName);
+ "Remove CustomizedStateConfig from cluster {}.", clusterName);
ZKHelixDataAccessor accessor =
new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
Builder keyBuilder = accessor.keyBuilder();
- accessor.removeProperty(keyBuilder.customizedStateAggregationConfig());
+ accessor.removeProperty(keyBuilder.customizedStateConfig());
}
@Override
- public void addTypeToCustomizedStateAggregationConfig(String clusterName, String type) {
- logger.info("Add type {} to CustomizedStateAggregationConfig of cluster {}", type, clusterName);
+ public void addTypeToCustomizedStateConfig(String clusterName, String type) {
+ logger.info("Add type {} to CustomizedStateConfig of cluster {}", type, clusterName);
if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
throw new HelixException("cluster " + clusterName + " is not setup yet");
}
- CustomizedStateAggregationConfig.Builder builder =
- new CustomizedStateAggregationConfig.Builder();
+ CustomizedStateConfig.Builder builder =
+ new CustomizedStateConfig.Builder();
builder.addAggregationEnabledType(type);
- CustomizedStateAggregationConfig customizedStateAggregationConfigFromBuilder = builder.build();
+ CustomizedStateConfig customizedStateConfigFromBuilder = builder.build();
ZKHelixDataAccessor accessor =
new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
Builder keyBuilder = accessor.keyBuilder();
- accessor.updateProperty(keyBuilder.customizedStateAggregationConfig(),
- customizedStateAggregationConfigFromBuilder);
+ accessor.updateProperty(keyBuilder.customizedStateConfig(),
+ customizedStateConfigFromBuilder);
}
@Override
- public void removeTypeFromCustomizedStateAggregationConfig(String clusterName, String type) {
- logger.info("Remove type {} to CustomizedStateAggregationConfig of cluster {}", type,
+ public void removeTypeFromCustomizedStateConfig(String clusterName, String type) {
+ logger.info("Remove type {} to CustomizedStateConfig of cluster {}", type,
clusterName);
if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
throw new HelixException("cluster " + clusterName + " is not setup yet");
}
- CustomizedStateAggregationConfig.Builder builder = new CustomizedStateAggregationConfig.Builder(
- _configAccessor.getCustomizedStateAggregationConfig(clusterName));
+ CustomizedStateConfig.Builder builder = new CustomizedStateConfig.Builder(
+ _configAccessor.getCustomizedStateConfig(clusterName));
if (!builder.getAggregationEnabledTypes().contains(type)) {
throw new HelixException("Type " + type
- + " is missing from the CustomizedStateAggregationConfig of cluster " + clusterName);
+ + " is missing from the CustomizedStateConfig of cluster " + clusterName);
}
builder.removeAggregationEnabledType(type);
- CustomizedStateAggregationConfig customizedStateAggregationConfigFromBuilder = builder.build();
+ CustomizedStateConfig customizedStateConfigFromBuilder = builder.build();
ZKHelixDataAccessor accessor =
new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
Builder keyBuilder = accessor.keyBuilder();
- accessor.setProperty(keyBuilder.customizedStateAggregationConfig(),
- customizedStateAggregationConfigFromBuilder);
+ accessor.setProperty(keyBuilder.customizedStateConfig(),
+ customizedStateConfigFromBuilder);
}
@Override
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
index a6425fa..b731635 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
@@ -58,7 +58,7 @@ import org.apache.helix.api.listeners.ConfigChangeListener;
import org.apache.helix.api.listeners.ControllerChangeListener;
import org.apache.helix.api.listeners.CurrentStateChangeListener;
import org.apache.helix.api.listeners.CustomizedViewChangeListener;
-import org.apache.helix.api.listeners.CustomizedStateAggregationConfigChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateConfigChangeListener;
import org.apache.helix.api.listeners.CustomizedStateChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
@@ -489,10 +489,10 @@ public class ZKHelixManager implements HelixManager, IZkStateListener {
}
@Override
- public void addCustomizedStateAggregationConfigChangeListener(
- CustomizedStateAggregationConfigChangeListener listener) throws Exception {
- addListener(listener, new Builder(_clusterName).customizedStateAggregationConfig(),
- ChangeType.CUSTOMIZED_STATE_AGGREGATION_CONFIG, new EventType[] {
+ public void addCustomizedStateConfigChangeListener(
+ CustomizedStateConfigChangeListener listener) throws Exception {
+ addListener(listener, new Builder(_clusterName).customizedStateConfig(),
+ ChangeType.CUSTOMIZED_STATE_CONFIG, new EventType[] {
EventType.NodeDataChanged
});
}
diff --git a/helix-core/src/main/java/org/apache/helix/model/CustomizedStateAggregationConfig.java b/helix-core/src/main/java/org/apache/helix/model/CustomizedStateConfig.java
similarity index 60%
rename from helix-core/src/main/java/org/apache/helix/model/CustomizedStateAggregationConfig.java
rename to helix-core/src/main/java/org/apache/helix/model/CustomizedStateConfig.java
index 4acd0e6..0b13db1 100644
--- a/helix-core/src/main/java/org/apache/helix/model/CustomizedStateAggregationConfig.java
+++ b/helix-core/src/main/java/org/apache/helix/model/CustomizedStateConfig.java
@@ -30,33 +30,33 @@ import org.apache.helix.zookeeper.datamodel.ZNRecord;
/**
* CustomizedStateAggregation configurations
*/
-public class CustomizedStateAggregationConfig extends HelixProperty {
+public class CustomizedStateConfig extends HelixProperty {
- public static final String CUSTOMIZED_STATE_AGGREGATION_CONFIG_KW =
- "CustomizedStateAggregationConfig";
+ public static final String CUSTOMIZED_STATE_CONFIG_KW =
+ "CustomizedStateConfig";
/**
* Indicate which customized states will be aggregated.
* NOTE: Do NOT use this field name directly, use its corresponding getter/setter in the
- * CustomizedStateAggregationConfig.
+ * CustomizedStateConfig.
*/
- public enum CustomizedStateAggregationProperty {
+ public enum CustomizedStateProperty {
AGGREGATION_ENABLED_TYPES,
}
/**
- * Instantiate the CustomizedStateAggregationConfig
+ * Instantiate the CustomizedStateConfig
*/
- public CustomizedStateAggregationConfig() {
- super(CUSTOMIZED_STATE_AGGREGATION_CONFIG_KW);
+ public CustomizedStateConfig() {
+ super(CUSTOMIZED_STATE_CONFIG_KW);
}
/**
* Instantiate with a pre-populated record
- * @param record a ZNRecord corresponding to a CustomizedStateAggregationConfig
+ * @param record a ZNRecord corresponding to a CustomizedStateConfig
*/
- public CustomizedStateAggregationConfig(ZNRecord record) {
- super(CUSTOMIZED_STATE_AGGREGATION_CONFIG_KW);
+ public CustomizedStateConfig(ZNRecord record) {
+ super(CUSTOMIZED_STATE_CONFIG_KW);
_record.setSimpleFields(record.getSimpleFields());
_record.setListFields(record.getListFields());
_record.setMapFields(record.getMapFields());
@@ -67,7 +67,7 @@ public class CustomizedStateAggregationConfig extends HelixProperty {
* @param aggregationEnabledTypes
*/
public void setAggregationEnabledTypes(List<String> aggregationEnabledTypes) {
- _record.setListField(CustomizedStateAggregationProperty.AGGREGATION_ENABLED_TYPES.name(),
+ _record.setListField(CustomizedStateProperty.AGGREGATION_ENABLED_TYPES.name(),
aggregationEnabledTypes);
}
@@ -77,22 +77,22 @@ public class CustomizedStateAggregationConfig extends HelixProperty {
*/
public List<String> getAggregationEnabledTypes() {
return _record
- .getListField(CustomizedStateAggregationProperty.AGGREGATION_ENABLED_TYPES.name());
+ .getListField(CustomizedStateProperty.AGGREGATION_ENABLED_TYPES.name());
}
public static class Builder {
private ZNRecord _record;
- public CustomizedStateAggregationConfig build() {
- return new CustomizedStateAggregationConfig(_record);
+ public CustomizedStateConfig build() {
+ return new CustomizedStateConfig(_record);
}
/**
* Default constructor
*/
public Builder() {
- _record = new ZNRecord(CUSTOMIZED_STATE_AGGREGATION_CONFIG_KW);
+ _record = new ZNRecord(CUSTOMIZED_STATE_CONFIG_KW);
}
/**
@@ -104,43 +104,43 @@ public class CustomizedStateAggregationConfig extends HelixProperty {
}
/**
- * Constructor with CustomizedStateAggregationConfig as input
- * @param customizedStateAggregationConfig
+ * Constructor with CustomizedStateConfig as input
+ * @param customizedStateConfig
*/
- public Builder(CustomizedStateAggregationConfig customizedStateAggregationConfig) {
- _record = customizedStateAggregationConfig.getRecord();
+ public Builder(CustomizedStateConfig customizedStateConfig) {
+ _record = customizedStateConfig.getRecord();
}
public Builder setAggregationEnabledTypes(List<String> aggregationEnabledTypes) {
- _record.setListField(CustomizedStateAggregationProperty.AGGREGATION_ENABLED_TYPES.name(), aggregationEnabledTypes);
+ _record.setListField(CustomizedStateProperty.AGGREGATION_ENABLED_TYPES.name(), aggregationEnabledTypes);
return this;
}
public Builder addAggregationEnabledType(String type) {
if (_record.getListField(
- CustomizedStateAggregationProperty.AGGREGATION_ENABLED_TYPES.name()) == null) {
- _record.setListField(CustomizedStateAggregationProperty.AGGREGATION_ENABLED_TYPES.name(), new ArrayList<String>());
+ CustomizedStateProperty.AGGREGATION_ENABLED_TYPES.name()) == null) {
+ _record.setListField(CustomizedStateProperty.AGGREGATION_ENABLED_TYPES.name(), new ArrayList<String>());
}
- List<String> aggregationEnabledTypes = _record.getListField(CustomizedStateAggregationProperty.AGGREGATION_ENABLED_TYPES.name());
+ List<String> aggregationEnabledTypes = _record.getListField(CustomizedStateProperty.AGGREGATION_ENABLED_TYPES.name());
aggregationEnabledTypes.add(type);
- _record.setListField(CustomizedStateAggregationProperty.AGGREGATION_ENABLED_TYPES.name(), aggregationEnabledTypes);
+ _record.setListField(CustomizedStateProperty.AGGREGATION_ENABLED_TYPES.name(), aggregationEnabledTypes);
return this;
}
public Builder removeAggregationEnabledType(String type) {
- if (!_record.getListField(CustomizedStateAggregationProperty.AGGREGATION_ENABLED_TYPES.name())
+ if (!_record.getListField(CustomizedStateProperty.AGGREGATION_ENABLED_TYPES.name())
.contains(type)) {
throw new HelixException(
- "Type " + type + " is missing from the CustomizedStateAggregationConfig");
+ "Type " + type + " is missing from the CustomizedStateConfig");
}
- _record.getListField(CustomizedStateAggregationProperty.AGGREGATION_ENABLED_TYPES.name())
+ _record.getListField(CustomizedStateProperty.AGGREGATION_ENABLED_TYPES.name())
.remove(type);
return this;
}
public List<String> getAggregationEnabledTypes() {
return _record
- .getListField(CustomizedStateAggregationProperty.AGGREGATION_ENABLED_TYPES.name());
+ .getListField(CustomizedStateProperty.AGGREGATION_ENABLED_TYPES.name());
}
}
}
diff --git a/helix-core/src/test/java/org/apache/helix/controller/stages/DummyClusterManager.java b/helix-core/src/test/java/org/apache/helix/controller/stages/DummyClusterManager.java
index 84d413b..772c0fa 100644
--- a/helix-core/src/test/java/org/apache/helix/controller/stages/DummyClusterManager.java
+++ b/helix-core/src/test/java/org/apache/helix/controller/stages/DummyClusterManager.java
@@ -37,7 +37,7 @@ import org.apache.helix.api.listeners.ClusterConfigChangeListener;
import org.apache.helix.api.listeners.ConfigChangeListener;
import org.apache.helix.api.listeners.ControllerChangeListener;
import org.apache.helix.api.listeners.CurrentStateChangeListener;
-import org.apache.helix.api.listeners.CustomizedStateAggregationConfigChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateConfigChangeListener;
import org.apache.helix.api.listeners.CustomizedStateChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.CustomizedViewChangeListener;
@@ -302,7 +302,7 @@ public class DummyClusterManager implements HelixManager {
}
@Override
- public void addCustomizedStateAggregationConfigChangeListener(CustomizedStateAggregationConfigChangeListener listener)
+ public void addCustomizedStateConfigChangeListener(CustomizedStateConfigChangeListener listener)
throws Exception {
// TODO Auto-generated method stub
diff --git a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
index da2145b..f36d66c 100644
--- a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
+++ b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
@@ -54,7 +54,6 @@ import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ClusterConstraints.ConstraintAttribute;
import org.apache.helix.model.ClusterConstraints.ConstraintType;
import org.apache.helix.model.ConstraintItem;
-import org.apache.helix.model.CustomizedStateAggregationConfig;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.HelixConfigScope;
import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
@@ -548,12 +547,12 @@ public class TestZkHelixAdmin extends ZkUnitTestBase {
instanceConfig.setInstanceEnabledForPartition(testResourcePrefix, "2", false);
Assert.assertEquals(instanceConfig.getDisabledPartitions(testResourcePrefix).size(), 3);
Assert.assertEquals(instanceConfig.getRecord()
- .getListField(InstanceConfig.InstanceConfigProperty.HELIX_DISABLED_PARTITION.name()).size(),
+ .getListField(InstanceConfig.InstanceConfigProperty.HELIX_DISABLED_PARTITION.name()).size(),
3);
instanceConfig.setInstanceEnabledForPartition(testResourcePrefix, "2", true);
Assert.assertEquals(instanceConfig.getDisabledPartitions(testResourcePrefix).size(), 2);
Assert.assertEquals(instanceConfig.getRecord()
- .getListField(InstanceConfig.InstanceConfigProperty.HELIX_DISABLED_PARTITION.name()).size(),
+ .getListField(InstanceConfig.InstanceConfigProperty.HELIX_DISABLED_PARTITION.name()).size(),
2);
}
@@ -600,7 +599,6 @@ public class TestZkHelixAdmin extends ZkUnitTestBase {
// OK since resourceConfig is empty
}
-<<<<<<< HEAD
// Set PARTITION_CAPACITY_MAP
Map<String, String> capacityDataMap =
ImmutableMap.of("WCU", "1", "RCU", "2", "STORAGE", "3");
@@ -686,6 +684,7 @@ public class TestZkHelixAdmin extends ZkUnitTestBase {
Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor");
}
+
@Test
public void testRemoveCloudConfig() throws Exception {
String className = TestHelper.getTestClassName();
@@ -811,4 +810,4 @@ public class TestZkHelixAdmin extends ZkUnitTestBase {
Assert.assertEquals(listTypesFromZk.get(0), "mockType2");
Assert.assertEquals(listTypesFromZk.get(1), "mockType3");
}
-}
\ No newline at end of file
+}
diff --git a/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java b/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java
index 02bd639..29e64f6 100644
--- a/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java
+++ b/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java
@@ -35,7 +35,7 @@ import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ConstraintItem;
-import org.apache.helix.model.CustomizedStateAggregationConfig;
+import org.apache.helix.model.CustomizedStateConfig;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.HelixConfigScope;
import org.apache.helix.model.IdealState;
@@ -145,23 +145,23 @@ public class MockHelixAdmin implements HelixAdmin {
}
@Override
- public void addCustomizedStateAggregationConfig(String clusterName,
- CustomizedStateAggregationConfig customizedStateAggregationConfig) {
+ public void addCustomizedStateConfig(String clusterName,
+ CustomizedStateConfig customizedStateConfig) {
}
@Override
- public void removeCustomizedStateAggregationConfig(String clusterName) {
+ public void removeCustomizedStateConfig(String clusterName) {
}
@Override
- public void addTypeToCustomizedStateAggregationConfig(String clusterName, String type) {
+ public void addTypeToCustomizedStateConfig(String clusterName, String type) {
}
@Override
- public void removeTypeFromCustomizedStateAggregationConfig(String clusterName, String type) {
+ public void removeTypeFromCustomizedStateConfig(String clusterName, String type) {
}
diff --git a/helix-core/src/test/java/org/apache/helix/mock/MockManager.java b/helix-core/src/test/java/org/apache/helix/mock/MockManager.java
index f32160b..6b744b7 100644
--- a/helix-core/src/test/java/org/apache/helix/mock/MockManager.java
+++ b/helix-core/src/test/java/org/apache/helix/mock/MockManager.java
@@ -40,7 +40,7 @@ import org.apache.helix.api.listeners.ConfigChangeListener;
import org.apache.helix.api.listeners.ControllerChangeListener;
import org.apache.helix.api.listeners.CurrentStateChangeListener;
import org.apache.helix.api.listeners.CustomizedViewChangeListener;
-import org.apache.helix.api.listeners.CustomizedStateAggregationConfigChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateConfigChangeListener;
import org.apache.helix.api.listeners.CustomizedStateChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
@@ -173,7 +173,7 @@ public class MockManager implements HelixManager {
}
@Override
- public void addCustomizedStateAggregationConfigChangeListener(CustomizedStateAggregationConfigChangeListener listener)
+ public void addCustomizedStateConfigChangeListener(CustomizedStateConfigChangeListener listener)
throws Exception {
}
diff --git a/helix-core/src/test/java/org/apache/helix/model/TestCustomizedStateAggregationConfig.java b/helix-core/src/test/java/org/apache/helix/model/TestCustomizedStateConfig.java
similarity index 56%
rename from helix-core/src/test/java/org/apache/helix/model/TestCustomizedStateAggregationConfig.java
rename to helix-core/src/test/java/org/apache/helix/model/TestCustomizedStateConfig.java
index bda6a50..13948e4 100644
--- a/helix-core/src/test/java/org/apache/helix/model/TestCustomizedStateAggregationConfig.java
+++ b/helix-core/src/test/java/org/apache/helix/model/TestCustomizedStateConfig.java
@@ -31,70 +31,70 @@ import org.apache.helix.manager.zk.ZkBaseDataAccessor;
import org.testng.Assert;
import org.testng.annotations.Test;
-public class TestCustomizedStateAggregationConfig extends ZkUnitTestBase {
+public class TestCustomizedStateConfig extends ZkUnitTestBase {
@Test(expectedExceptions = HelixException.class)
- public void TestCustomizedStateAggregationConfigNonExistentCluster() {
+ public void TestCustomizedStateConfigNonExistentCluster() {
String className = getShortClassName();
String clusterName = "CLUSTER_" + className;
- // Read CustomizedStateAggregationConfig from Zookeeper and get exception since cluster in not setup yet
+ // Read CustomizedStateConfig from Zookeeper and get exception since cluster in not setup yet
ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
- CustomizedStateAggregationConfig customizedStateAggregationConfig =
- _configAccessor.getCustomizedStateAggregationConfig(clusterName);
+ CustomizedStateConfig customizedStateConfig =
+ _configAccessor.getCustomizedStateConfig(clusterName);
}
- @Test(dependsOnMethods = "TestCustomizedStateAggregationConfigNonExistentCluster")
- public void testCustomizedStateAggregationConfigNull() {
+ @Test(dependsOnMethods = "TestCustomizedStateConfigNonExistentCluster")
+ public void testCustomizedStateConfigNull() {
String className = getShortClassName();
String clusterName = "CLUSTER_" + className;
TestHelper.setupEmptyCluster(_gZkClient, clusterName);
- // Read CustomizedStateAggregationConfig from Zookeeper
+ // Read CustomizedStateConfig from Zookeeper
ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
- CustomizedStateAggregationConfig customizedStateAggregationConfigFromZk =
- _configAccessor.getCustomizedStateAggregationConfig(clusterName);
- Assert.assertNull(customizedStateAggregationConfigFromZk);
+ CustomizedStateConfig customizedStateConfigFromZk =
+ _configAccessor.getCustomizedStateConfig(clusterName);
+ Assert.assertNull(customizedStateConfigFromZk);
}
- @Test(dependsOnMethods = "testCustomizedStateAggregationConfigNull")
- public void testCustomizedStateAggregationConfig() {
+ @Test(dependsOnMethods = "testCustomizedStateConfigNull")
+ public void testCustomizedStateConfig() {
String className = getShortClassName();
String clusterName = "CLUSTER_" + className;
TestHelper.setupEmptyCluster(_gZkClient, clusterName);
- // Create dummy CustomizedStateAggregationConfig object
- CustomizedStateAggregationConfig.Builder customizedStateAggregationConfigBuilder =
- new CustomizedStateAggregationConfig.Builder();
+ // Create dummy CustomizedStateConfig object
+ CustomizedStateConfig.Builder customizedStateConfigBuilder =
+ new CustomizedStateConfig.Builder();
List<String> aggregationEnabledTypes = new ArrayList<String>();
aggregationEnabledTypes.add("mockType1");
aggregationEnabledTypes.add("mockType2");
- customizedStateAggregationConfigBuilder.setAggregationEnabledTypes(aggregationEnabledTypes);
- CustomizedStateAggregationConfig customizedStateAggregationConfig =
- customizedStateAggregationConfigBuilder.build();
+ customizedStateConfigBuilder.setAggregationEnabledTypes(aggregationEnabledTypes);
+ CustomizedStateConfig customizedStateConfig =
+ customizedStateConfigBuilder.build();
- // Write the CustomizedStateAggregationConfig to Zookeeper
+ // Write the CustomizedStateConfig to Zookeeper
ZKHelixDataAccessor accessor =
new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor(ZK_ADDR));
Builder keyBuilder = accessor.keyBuilder();
- accessor.setProperty(keyBuilder.customizedStateAggregationConfig(),
- customizedStateAggregationConfig);
+ accessor.setProperty(keyBuilder.customizedStateConfig(),
+ customizedStateConfig);
- // Read CustomizedStateAggregationConfig from Zookeeper and check the content
+ // Read CustomizedStateConfig from Zookeeper and check the content
ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
- CustomizedStateAggregationConfig customizedStateAggregationConfigFromZk =
- _configAccessor.getCustomizedStateAggregationConfig(clusterName);
- Assert.assertEquals(customizedStateAggregationConfigFromZk.getAggregationEnabledTypes().size(),
+ CustomizedStateConfig customizedStateConfigFromZk =
+ _configAccessor.getCustomizedStateConfig(clusterName);
+ Assert.assertEquals(customizedStateConfigFromZk.getAggregationEnabledTypes().size(),
2);
Assert.assertEquals(aggregationEnabledTypes.get(0), "mockType1");
Assert.assertEquals(aggregationEnabledTypes.get(1), "mockType2");
}
- @Test(dependsOnMethods = "testCustomizedStateAggregationConfig")
- public void testCustomizedStateAggregationConfigBuilder() {
+ @Test(dependsOnMethods = "testCustomizedStateConfig")
+ public void testCustomizedStateConfigBuilder() {
String className = getShortClassName();
String clusterName = "CLUSTER_" + className;
TestHelper.setupEmptyCluster(_gZkClient, clusterName);
- CustomizedStateAggregationConfig.Builder builder =
- new CustomizedStateAggregationConfig.Builder();
+ CustomizedStateConfig.Builder builder =
+ new CustomizedStateConfig.Builder();
builder.addAggregationEnabledType("mockType1");
builder.addAggregationEnabledType("mockType2");
@@ -104,20 +104,20 @@ public class TestCustomizedStateAggregationConfig extends ZkUnitTestBase {
Assert.assertEquals(aggregationEnabledTypes.get(0), "mockType1");
Assert.assertEquals(aggregationEnabledTypes.get(1), "mockType2");
- CustomizedStateAggregationConfig customizedStateAggregationConfig = builder.build();
+ CustomizedStateConfig customizedStateConfig = builder.build();
ZKHelixDataAccessor accessor =
new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor(ZK_ADDR));
Builder keyBuilder = accessor.keyBuilder();
- accessor.setProperty(keyBuilder.customizedStateAggregationConfig(),
- customizedStateAggregationConfig);
+ accessor.setProperty(keyBuilder.customizedStateConfig(),
+ customizedStateConfig);
- // Read CustomizedStateAggregationConfig from Zookeeper and check the content
+ // Read CustomizedStateConfig from Zookeeper and check the content
ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
- CustomizedStateAggregationConfig customizedStateAggregationConfigFromZk =
- _configAccessor.getCustomizedStateAggregationConfig(clusterName);
+ CustomizedStateConfig customizedStateConfigFromZk =
+ _configAccessor.getCustomizedStateConfig(clusterName);
List<String> aggregationEnabledTypesFromZk =
- customizedStateAggregationConfigFromZk.getAggregationEnabledTypes();
+ customizedStateConfigFromZk.getAggregationEnabledTypes();
Assert.assertEquals(aggregationEnabledTypesFromZk.get(0), "mockType1");
Assert.assertEquals(aggregationEnabledTypesFromZk.get(1), "mockType2");
}
diff --git a/helix-core/src/test/java/org/apache/helix/participant/MockZKHelixManager.java b/helix-core/src/test/java/org/apache/helix/participant/MockZKHelixManager.java
index 4abe7c6..9e50c2a 100644
--- a/helix-core/src/test/java/org/apache/helix/participant/MockZKHelixManager.java
+++ b/helix-core/src/test/java/org/apache/helix/participant/MockZKHelixManager.java
@@ -39,7 +39,7 @@ import org.apache.helix.api.listeners.ConfigChangeListener;
import org.apache.helix.api.listeners.ControllerChangeListener;
import org.apache.helix.api.listeners.CurrentStateChangeListener;
import org.apache.helix.api.listeners.CustomizedViewChangeListener;
-import org.apache.helix.api.listeners.CustomizedStateAggregationConfigChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateConfigChangeListener;
import org.apache.helix.api.listeners.CustomizedStateChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
@@ -126,7 +126,7 @@ public class MockZKHelixManager implements HelixManager {
}
@Override
- public void addCustomizedStateAggregationConfigChangeListener(CustomizedStateAggregationConfigChangeListener listener)
+ public void addCustomizedStateConfigChangeListener(CustomizedStateConfigChangeListener listener)
throws Exception {
// TODO Auto-generated method stub
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
index 07838e3..ca3c757 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
@@ -48,7 +48,7 @@ import org.apache.helix.manager.zk.ZKUtil;
import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.ControllerHistory;
-import org.apache.helix.model.CustomizedStateAggregationConfig;
+import org.apache.helix.model.CustomizedStateConfig;
import org.apache.helix.model.HelixConfigScope;
import org.apache.helix.model.LiveInstance;
import org.apache.helix.model.MaintenanceSignal;
@@ -300,7 +300,7 @@ public class ClusterAccessor extends AbstractHelixResource {
@PUT
@Path("{clusterId}/customized-state-aggregation-config")
- public Response addCustomizedStateAggregationConfig(@PathParam("clusterId") String clusterId,
+ public Response addCustomizedStateConfig(@PathParam("clusterId") String clusterId,
String content) {
if (!doesClusterExist(clusterId)) {
return notFound(String.format("Cluster %s does not exist", clusterId));
@@ -315,11 +315,11 @@ public class ClusterAccessor extends AbstractHelixResource {
}
try {
- CustomizedStateAggregationConfig customizedStateAggregationConfig =
- new CustomizedStateAggregationConfig.Builder(record).build();
- admin.addCustomizedStateAggregationConfig(clusterId, customizedStateAggregationConfig);
+ CustomizedStateConfig customizedStateConfig =
+ new CustomizedStateConfig.Builder(record).build();
+ admin.addCustomizedStateConfig(clusterId, customizedStateConfig);
} catch (Exception ex) {
- _logger.error("Cannot add CustomizedStateAggregationConfig to cluster: {} Exception: {}",
+ _logger.error("Cannot add CustomizedStateConfig to cluster: {} Exception: {}",
clusterId, ex);
return serverError(ex);
}
@@ -329,17 +329,17 @@ public class ClusterAccessor extends AbstractHelixResource {
@DELETE
@Path("{clusterId}/customized-state-aggregation-config")
- public Response removeCustomizedStateAggregationConfig(@PathParam("clusterId") String clusterId) {
+ public Response removeCustomizedStateConfig(@PathParam("clusterId") String clusterId) {
if (!doesClusterExist(clusterId)) {
return notFound(String.format("Cluster %s does not exist", clusterId));
}
HelixAdmin admin = getHelixAdmin();
try {
- admin.removeCustomizedStateAggregationConfig(clusterId);
+ admin.removeCustomizedStateConfig(clusterId);
} catch (Exception ex) {
_logger.error(
- "Cannot remove CustomizedStateAggregationConfig from cluster: {}, Exception: {}",
+ "Cannot remove CustomizedStateConfig from cluster: {}, Exception: {}",
clusterId, ex);
return serverError(ex);
}
@@ -349,17 +349,17 @@ public class ClusterAccessor extends AbstractHelixResource {
@GET
@Path("{clusterId}/customized-state-aggregation-config")
- public Response getCustomizedStateAggregationConfig(@PathParam("clusterId") String clusterId) {
+ public Response getCustomizedStateConfig(@PathParam("clusterId") String clusterId) {
if (!doesClusterExist(clusterId)) {
return notFound(String.format("Cluster %s does not exist", clusterId));
}
ConfigAccessor configAccessor = getConfigAccessor();
- CustomizedStateAggregationConfig customizedStateAggregationConfig =
- configAccessor.getCustomizedStateAggregationConfig(clusterId);
+ CustomizedStateConfig customizedStateConfig =
+ configAccessor.getCustomizedStateConfig(clusterId);
- if (customizedStateAggregationConfig != null) {
- return JSONRepresentation(customizedStateAggregationConfig.getRecord());
+ if (customizedStateConfig != null) {
+ return JSONRepresentation(customizedStateConfig.getRecord());
}
return notFound();
@@ -367,7 +367,7 @@ public class ClusterAccessor extends AbstractHelixResource {
@POST
@Path("{clusterId}/customized-state-aggregation-config")
- public Response updateCustomizedStateAggregationConfig(@PathParam("clusterId") String clusterId,
+ public Response updateCustomizedStateConfig(@PathParam("clusterId") String clusterId,
@QueryParam("command") String commandStr, @QueryParam("type") String type) {
if (!doesClusterExist(clusterId)) {
return notFound(String.format("Cluster %s does not exist", clusterId));
@@ -389,16 +389,16 @@ public class ClusterAccessor extends AbstractHelixResource {
try {
switch (command) {
case delete:
- admin.removeTypeFromCustomizedStateAggregationConfig(clusterId, type);
+ admin.removeTypeFromCustomizedStateConfig(clusterId, type);
break;
case add:
- admin.addTypeToCustomizedStateAggregationConfig(clusterId, type);
+ admin.addTypeToCustomizedStateConfig(clusterId, type);
break;
default:
return badRequest("Unsupported command " + commandStr);
}
} catch (Exception ex) {
- _logger.error("Failed to {} CustomizedStateAggregationConfig for cluster {} new type: {}, Exception: {}", command, clusterId, type, ex);
+ _logger.error("Failed to {} CustomizedStateConfig for cluster {} new type: {}, Exception: {}", command, clusterId, type, ex);
return serverError(ex);
}
return OK();
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
index fd343ae..a39140d 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
@@ -47,7 +47,6 @@ import org.apache.helix.integration.manager.ClusterDistributedController;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
import org.apache.helix.manager.zk.ZKUtil;
import org.apache.helix.model.ClusterConfig;
-import org.apache.helix.model.CustomizedStateAggregationConfig;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.InstanceConfig;
@@ -784,6 +783,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
}
+
@Test(dependsOnMethods = "testAddClusterWithCloudConfigDisabledCloud")
public void testAddCloudConfigNonExistedCluster() throws IOException {
System.out.println("Start test :" + TestHelper.getTestMethodName());
@@ -898,6 +898,7 @@ public class TestClusterAccessor extends AbstractTestClass {
System.out.println("End test :" + TestHelper.getTestMethodName());
}
+
@Test(dependsOnMethods = "testDeleteCloudConfig")
public void testPartialDeleteCloudConfig() throws IOException {
System.out.println("Start test :" + TestHelper.getTestMethodName());
@@ -1213,4 +1214,4 @@ public class TestClusterAccessor extends AbstractTestClass {
Assert.assertEquals(auditLog.getResponseCode(), statusCode);
Assert.assertEquals(auditLog.getResponseEntity(), responseEntity);
}
-}
+}
\ No newline at end of file
[helix] 22/23: Move routing table provider initialization (#946)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 25e47262e5a4ca7d9c6302890f2bf8aa120d4722
Author: Molly Gao <31...@users.noreply.github.com>
AuthorDate: Mon Apr 13 10:56:48 2020 -0700
Move routing table provider initialization (#946)
This PR moves the initialization of routing table provider to before class so it is initialized before any updates. Also, added several checks into the validation method to cover some edge cases.
---
.../integration/TestCustomizedViewAggregation.java | 92 +++++++++++-----------
1 file changed, 48 insertions(+), 44 deletions(-)
diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestCustomizedViewAggregation.java b/helix-core/src/test/java/org/apache/helix/integration/TestCustomizedViewAggregation.java
index af24107..e270e9f 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/TestCustomizedViewAggregation.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/TestCustomizedViewAggregation.java
@@ -71,9 +71,7 @@ public class TestCustomizedViewAggregation extends ZkUnitTestBase {
public void beforeClass() throws Exception {
super.beforeClass();
- String className = TestHelper.getTestClassName();
- String methodName = TestHelper.getTestMethodName();
- String clusterName = className + "_" + methodName;
+ String clusterName = TestHelper.getTestClassName();
int n = 2;
System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
@@ -120,6 +118,23 @@ public class TestCustomizedViewAggregation extends ZkUnitTestBase {
_localCustomizedView = new HashMap<>();
_routingTableProviderDataSources = new HashSet<>();
_aggregationEnabledTypes = new HashSet<>();
+
+ List<String> customizedStateTypes = Arrays
+ .asList(CustomizedStateType.TYPE_A.name(), CustomizedStateType.TYPE_B.name(),
+ CustomizedStateType.TYPE_C.name());
+
+ CustomizedStateConfig.Builder customizedStateConfigBuilder =
+ new CustomizedStateConfig.Builder();
+ customizedStateConfigBuilder.setAggregationEnabledTypes(customizedStateTypes);
+ HelixDataAccessor accessor = _manager.getHelixDataAccessor();
+ accessor.setProperty(accessor.keyBuilder().customizedStateConfig(),
+ customizedStateConfigBuilder.build());
+ _aggregationEnabledTypes.addAll(customizedStateTypes);
+
+ Map<PropertyType, List<String>> dataSource = new HashMap<>();
+ dataSource.put(PropertyType.CUSTOMIZEDVIEW, customizedStateTypes);
+ _routingTableProvider = new RoutingTableProvider(_spectator, dataSource);
+ _routingTableProviderDataSources.addAll(customizedStateTypes);
}
@AfterClass
@@ -143,7 +158,6 @@ public class TestCustomizedViewAggregation extends ZkUnitTestBase {
// Get customized view snapshot
Map<String, RoutingTableSnapshot> fullCustomizedViewSnapshot =
routingTableSnapshots.get(PropertyType.CUSTOMIZEDVIEW.name());
- boolean result = false;
if (fullCustomizedViewSnapshot.isEmpty() && !_routingTableProviderDataSources.isEmpty()) {
return false;
@@ -170,6 +184,11 @@ public class TestCustomizedViewAggregation extends ZkUnitTestBase {
return false;
}
+ if (_aggregationEnabledTypes.contains(customizedStateType)
+ && customizedViews.size() != localSnapshot.size()) {
+ return false;
+ }
+
// Get per resource snapshot
for (CustomizedView resourceCustomizedView : customizedViews) {
ZNRecord record = resourceCustomizedView.getRecord();
@@ -200,10 +219,7 @@ public class TestCustomizedViewAggregation extends ZkUnitTestBase {
// Per instance value
String stateMapValue = stateMap.get(instanceName);
String localStateMapValue = localStateMap.get(instanceName);
- if (isEmptyValue(stateMapValue) && isEmptyValue(localStateMapValue)) {
- } else if ((!isEmptyValue(stateMapValue) && !isEmptyValue(localStateMapValue)
- && !stateMapValue.equals(localStateMapValue)) || (isEmptyValue(stateMapValue)
- || isEmptyValue(localStateMapValue))) {
+ if (!stateMapValue.equals(localStateMapValue)) {
return false;
}
}
@@ -217,10 +233,6 @@ public class TestCustomizedViewAggregation extends ZkUnitTestBase {
Assert.assertTrue(result);
}
- private boolean isEmptyValue(String value) {
- return value == null || value.equals("");
- }
-
/**
* Update the local record of customized view
* @param instanceName the instance to be updated
@@ -243,12 +255,6 @@ public class TestCustomizedViewAggregation extends ZkUnitTestBase {
localPerPartition.remove(instanceName);
if (localPerPartition.isEmpty()) {
localPerResource.remove(partitionName);
- if (localPerResource.isEmpty()) {
- localPerStateType.remove(resourceName);
- if (localPerStateType.isEmpty()) {
- _localCustomizedView.remove(customizedStateType.name());
- }
- }
}
} else {
localPerPartition.put(instanceName, customizedStateValue.name());
@@ -348,9 +354,10 @@ public class TestCustomizedViewAggregation extends ZkUnitTestBase {
}
@Test
- public void testCustomizedStateViewAggregation() throws Exception {
- setAggregationEnabledTypes(
- Arrays.asList(CustomizedStateType.TYPE_A, CustomizedStateType.TYPE_B));
+ public void testCustomizedViewAggregation() throws Exception {
+
+ // Aggregating: Type A, Type B, Type C
+ // Routing table: Type A, Type B, Type C
update(INSTANCE_0, CustomizedStateType.TYPE_A, RESOURCE_0, PARTITION_00,
CurrentStateValues.TYPE_A_0);
@@ -370,16 +377,6 @@ public class TestCustomizedViewAggregation extends ZkUnitTestBase {
CurrentStateValues.TYPE_C_2);
update(INSTANCE_1, CustomizedStateType.TYPE_A, RESOURCE_1, PARTITION_11,
CurrentStateValues.TYPE_A_1);
-
- // Aggregation enabled types: A, B; Routing table provider data sources: A, B, C; should show TypeA, TypeB customized views
- setRoutingTableProviderDataSources(Arrays
- .asList(CustomizedStateType.TYPE_A, CustomizedStateType.TYPE_B,
- CustomizedStateType.TYPE_C));
- validateAggregationSnapshot();
-
- // Aggregation enabled types: A, B; Routing table provider data sources: A, B, C; should show TypeA, TypeB customized views
- setAggregationEnabledTypes(Arrays.asList(CustomizedStateType.TYPE_A, CustomizedStateType.TYPE_B,
- CustomizedStateType.TYPE_C));
validateAggregationSnapshot();
Assert.assertNull(_customizedStateProvider_participant0
@@ -395,8 +392,18 @@ public class TestCustomizedViewAggregation extends ZkUnitTestBase {
updateLocalCustomizedViewMap(INSTANCE_1, CustomizedStateType.TYPE_A, RESOURCE_1, PARTITION_10,
CurrentStateValues.TYPE_A_2);
- // Aggregation enabled types: A, B, C; Routing table provider data sources: A; should only show TypeA customized view
- setRoutingTableProviderDataSources(Arrays.asList(CustomizedStateType.TYPE_A));
+ validateAggregationSnapshot();
+
+ // Set the routing table provider data sources to only Type A and Type B, so users won't see Type C customized view
+ // Aggregating: Type A, Type B, Type C
+ // Routing table: Type A, Type B
+ setRoutingTableProviderDataSources(
+ Arrays.asList(CustomizedStateType.TYPE_A, CustomizedStateType.TYPE_B));
+ validateAggregationSnapshot();
+
+ // Aggregating: Type A
+ // Routing table: Type A, Type B
+ setAggregationEnabledTypes(Arrays.asList(CustomizedStateType.TYPE_A));
validateAggregationSnapshot();
// Test get customized state and get per partition customized state via customized state provider, this part of test doesn't change customized view
@@ -433,17 +440,6 @@ public class TestCustomizedViewAggregation extends ZkUnitTestBase {
null);
validateAggregationSnapshot();
- //Aggregation enabled types: B; Routing table provider data sources: A, B, C; should show TypeB customized views
- setAggregationEnabledTypes(Arrays.asList(CustomizedStateType.TYPE_B));
- setRoutingTableProviderDataSources(Arrays
- .asList(CustomizedStateType.TYPE_A, CustomizedStateType.TYPE_B,
- CustomizedStateType.TYPE_C));
- validateAggregationSnapshot();
-
- //Aggregation enabled types: B; Routing table provider data sources: A; should show empty customized view
- setRoutingTableProviderDataSources(Arrays.asList(CustomizedStateType.TYPE_A));
- validateAggregationSnapshot();
-
// Update some customized states and verify
delete(INSTANCE_0, CustomizedStateType.TYPE_A, RESOURCE_0, PARTITION_00);
delete(INSTANCE_1, CustomizedStateType.TYPE_B, RESOURCE_1, PARTITION_10);
@@ -452,6 +448,14 @@ public class TestCustomizedViewAggregation extends ZkUnitTestBase {
delete(INSTANCE_1, CustomizedStateType.TYPE_A, RESOURCE_1, PARTITION_10);
validateAggregationSnapshot();
+ // Aggregating: Type A, Type B, Type C
+ // Routing table: Type A, Type B, Type C
+ setRoutingTableProviderDataSources(Arrays
+ .asList(CustomizedStateType.TYPE_A, CustomizedStateType.TYPE_B, CustomizedStateType.TYPE_C));
+ setAggregationEnabledTypes(Arrays.asList(CustomizedStateType.TYPE_A, CustomizedStateType.TYPE_B,
+ CustomizedStateType.TYPE_C));
+ validateAggregationSnapshot();
+
update(INSTANCE_0, CustomizedStateType.TYPE_B, RESOURCE_0, PARTITION_01,
CurrentStateValues.TYPE_B_2);
update(INSTANCE_1, CustomizedStateType.TYPE_B, RESOURCE_1, PARTITION_10,
[helix] 07/23: add listener and config for customized view
aggregation (#815)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit ca7af925154747e900e1a2370472b1e348ec7ee9
Author: zhangmeng916 <56...@users.noreply.github.com>
AuthorDate: Wed Feb 26 16:34:27 2020 -0800
add listener and config for customized view aggregation (#815)
Add listeners for customized state and customized state aggregation config in Helix managers
---
.../main/java/org/apache/helix/HelixConstants.java | 2 +
.../main/java/org/apache/helix/HelixManager.java | 22 ++++++++++-
.../main/java/org/apache/helix/PropertyType.java | 2 +-
...mizedStateAggregationConfigChangeListener.java} | 41 +++++++++-----------
.../listeners/CustomizedStateChangeListener.java} | 44 ++++++++++------------
.../helix/controller/stages/AttributeName.java | 1 +
.../apache/helix/manager/zk/ZKHelixManager.java | 19 ++++++++++
.../controller/stages/DummyClusterManager.java | 16 ++++++++
.../manager/TestParticipantManager.java | 2 +-
.../java/org/apache/helix/mock/MockManager.java | 15 ++++++++
.../helix/participant/MockZKHelixManager.java | 16 ++++++++
11 files changed, 129 insertions(+), 51 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/HelixConstants.java b/helix-core/src/main/java/org/apache/helix/HelixConstants.java
index b0783bf..27f82c6 100644
--- a/helix-core/src/main/java/org/apache/helix/HelixConstants.java
+++ b/helix-core/src/main/java/org/apache/helix/HelixConstants.java
@@ -30,9 +30,11 @@ public interface HelixConstants {
CONFIG (PropertyType.CONFIGS),
INSTANCE_CONFIG (PropertyType.CONFIGS),
RESOURCE_CONFIG (PropertyType.CONFIGS),
+ CUSTOMIZED_STATE_AGGREGATION_CONFIG (PropertyType.CONFIGS),
CLUSTER_CONFIG (PropertyType.CONFIGS),
LIVE_INSTANCE (PropertyType.LIVEINSTANCES),
CURRENT_STATE (PropertyType.CURRENTSTATES),
+ CUSTOMIZED_STATE (PropertyType.CUSTOMIZEDSTATES),
MESSAGE (PropertyType.MESSAGES),
EXTERNAL_VIEW (PropertyType.EXTERNALVIEW),
CUSTOMIZED_VIEW (PropertyType.CUSTOMIZEDVIEW),
diff --git a/helix-core/src/main/java/org/apache/helix/HelixManager.java b/helix-core/src/main/java/org/apache/helix/HelixManager.java
index 2be5ce4..a499b14 100644
--- a/helix-core/src/main/java/org/apache/helix/HelixManager.java
+++ b/helix-core/src/main/java/org/apache/helix/HelixManager.java
@@ -26,6 +26,8 @@ import org.apache.helix.api.listeners.ClusterConfigChangeListener;
import org.apache.helix.api.listeners.ConfigChangeListener;
import org.apache.helix.api.listeners.ControllerChangeListener;
import org.apache.helix.api.listeners.CurrentStateChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateAggregationConfigChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.CustomizedViewChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
@@ -39,6 +41,7 @@ import org.apache.helix.controller.pipeline.Pipeline;
import org.apache.helix.healthcheck.ParticipantHealthReportCollector;
import org.apache.helix.manager.zk.ZKHelixManager;
import org.apache.helix.model.ClusterConfig;
+import org.apache.helix.model.CustomizedStateAggregationConfig;
import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
import org.apache.helix.participant.HelixStateMachineEngine;
import org.apache.helix.participant.StateMachineEngine;
@@ -158,6 +161,15 @@ public interface HelixManager {
void addResourceConfigChangeListener(ResourceConfigChangeListener listener) throws Exception;
/**
+ * @see CustomizedStateAggregationConfigChangeListener#onCustomizedStateAggregationConfigChange(CustomizedStateAggregationConfig,
+ * NotificationContext)
+ * @param listener
+ */
+
+ void addCustomizedStateAggregationConfigChangeListener(
+ CustomizedStateAggregationConfigChangeListener listener) throws Exception;
+
+ /**
* @see ClusterConfigChangeListener#onClusterConfigChange(ClusterConfig, NotificationContext)
* @param listener
*/
@@ -215,6 +227,14 @@ public interface HelixManager {
String sessionId) throws Exception;
/**
+ * @see CustomizedStateChangeListener#onCustomizedStateChange(String, List, NotificationContext)
+ * @param listener
+ * @param instanceName
+ */
+ void addCustomizedStateChangeListener(CustomizedStateChangeListener listener, String instanceName,
+ String stateName) throws Exception;
+
+ /**
* @see ExternalViewChangeListener#onExternalViewChange(List, NotificationContext)
* @param listener
*/
@@ -425,4 +445,4 @@ public interface HelixManager {
* @return ParticipantHealthReportCollector
*/
ParticipantHealthReportCollector getHealthReportCollector();
-}
+}
\ No newline at end of file
diff --git a/helix-core/src/main/java/org/apache/helix/PropertyType.java b/helix-core/src/main/java/org/apache/helix/PropertyType.java
index 363db21..03a6ce8 100644
--- a/helix-core/src/main/java/org/apache/helix/PropertyType.java
+++ b/helix-core/src/main/java/org/apache/helix/PropertyType.java
@@ -216,4 +216,4 @@ public enum PropertyType {
public boolean isCached() {
return isCached;
}
-}
+}
\ No newline at end of file
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/AttributeName.java b/helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedStateAggregationConfigChangeListener.java
similarity index 53%
copy from helix-core/src/main/java/org/apache/helix/controller/stages/AttributeName.java
copy to helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedStateAggregationConfigChangeListener.java
index b570568..2617ef3 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/stages/AttributeName.java
+++ b/helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedStateAggregationConfigChangeListener.java
@@ -1,4 +1,4 @@
-package org.apache.helix.controller.stages;
+package org.apache.helix.api.listeners;
/*
* Licensed to the Apache Software Foundation (ASF) under one
@@ -9,7 +9,7 @@ package org.apache.helix.controller.stages;
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
@@ -19,25 +19,18 @@ package org.apache.helix.controller.stages;
* under the License.
*/
-public enum AttributeName {
- RESOURCES,
- RESOURCES_TO_REBALANCE,
- BEST_POSSIBLE_STATE,
- CURRENT_STATE,
- INTERMEDIATE_STATE,
- MESSAGES_ALL,
- MESSAGES_SELECTED,
- MESSAGES_THROTTLE,
- LOCAL_STATE,
- EVENT_CREATE_TIME,
- helixmanager,
- clusterStatusMonitor,
- changeContext,
- instanceName,
- eventData,
- AsyncFIFOWorkerPool,
- PipelineType,
- LastRebalanceFinishTimeStamp,
- ControllerDataProvider,
- STATEFUL_REBALANCER
-}
+import org.apache.helix.NotificationContext;
+import org.apache.helix.model.CustomizedStateAggregationConfig;
+
+/**
+ * Interface to implement to listen for changes to customized state aggregation configurations.
+ */
+public interface CustomizedStateAggregationConfigChangeListener {
+ /**
+ * Invoked when customized state aggregation config changes
+ * @param customizedStateAggregationConfig
+ * @param context
+ */
+ void onCustomizedStateAggregationConfigChange(CustomizedStateAggregationConfig customizedStateAggregationConfig,
+ NotificationContext context);
+}
\ No newline at end of file
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/AttributeName.java b/helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedStateChangeListener.java
similarity index 51%
copy from helix-core/src/main/java/org/apache/helix/controller/stages/AttributeName.java
copy to helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedStateChangeListener.java
index b570568..0753f67 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/stages/AttributeName.java
+++ b/helix-core/src/main/java/org/apache/helix/api/listeners/CustomizedStateChangeListener.java
@@ -1,4 +1,4 @@
-package org.apache.helix.controller.stages;
+package org.apache.helix.api.listeners;
/*
* Licensed to the Apache Software Foundation (ASF) under one
@@ -9,7 +9,7 @@ package org.apache.helix.controller.stages;
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
@@ -19,25 +19,21 @@ package org.apache.helix.controller.stages;
* under the License.
*/
-public enum AttributeName {
- RESOURCES,
- RESOURCES_TO_REBALANCE,
- BEST_POSSIBLE_STATE,
- CURRENT_STATE,
- INTERMEDIATE_STATE,
- MESSAGES_ALL,
- MESSAGES_SELECTED,
- MESSAGES_THROTTLE,
- LOCAL_STATE,
- EVENT_CREATE_TIME,
- helixmanager,
- clusterStatusMonitor,
- changeContext,
- instanceName,
- eventData,
- AsyncFIFOWorkerPool,
- PipelineType,
- LastRebalanceFinishTimeStamp,
- ControllerDataProvider,
- STATEFUL_REBALANCER
-}
+import java.util.List;
+import org.apache.helix.NotificationContext;
+import org.apache.helix.model.CustomizedState;
+
+/**
+ * Interface to implement to respond to changes in the current state
+ */
+public interface CustomizedStateChangeListener {
+
+ /**
+ * Invoked when customized state changes
+ * @param instanceName name of the instance whose state changed
+ * @param customizedStatesInfo a list of the customized states
+ * @param changeContext the change event and state
+ */
+ void onCustomizedStateChange(String instanceName, List<CustomizedState> customizedStatesInfo,
+ NotificationContext changeContext);
+}
\ No newline at end of file
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/AttributeName.java b/helix-core/src/main/java/org/apache/helix/controller/stages/AttributeName.java
index b570568..0af0ee5 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/stages/AttributeName.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/stages/AttributeName.java
@@ -24,6 +24,7 @@ public enum AttributeName {
RESOURCES_TO_REBALANCE,
BEST_POSSIBLE_STATE,
CURRENT_STATE,
+ CUSTOMIZED_STATE,
INTERMEDIATE_STATE,
MESSAGES_ALL,
MESSAGES_SELECTED,
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
index c06ddf1..a6425fa 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
@@ -58,6 +58,8 @@ import org.apache.helix.api.listeners.ConfigChangeListener;
import org.apache.helix.api.listeners.ControllerChangeListener;
import org.apache.helix.api.listeners.CurrentStateChangeListener;
import org.apache.helix.api.listeners.CustomizedViewChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateAggregationConfigChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
import org.apache.helix.api.listeners.InstanceConfigChangeListener;
@@ -487,6 +489,15 @@ public class ZKHelixManager implements HelixManager, IZkStateListener {
}
@Override
+ public void addCustomizedStateAggregationConfigChangeListener(
+ CustomizedStateAggregationConfigChangeListener listener) throws Exception {
+ addListener(listener, new Builder(_clusterName).customizedStateAggregationConfig(),
+ ChangeType.CUSTOMIZED_STATE_AGGREGATION_CONFIG, new EventType[] {
+ EventType.NodeDataChanged
+ });
+ }
+
+ @Override
public void addClusterfigChangeListener(ClusterConfigChangeListener listener) throws Exception{
addListener(listener, new Builder(_clusterName).clusterConfig(), ChangeType.CLUSTER_CONFIG,
new EventType[] { EventType.NodeDataChanged
@@ -576,6 +587,14 @@ public class ZKHelixManager implements HelixManager, IZkStateListener {
}
@Override
+ public void addCustomizedStateChangeListener(CustomizedStateChangeListener listener,
+ String instanceName, String customizedStateName) throws Exception {
+ addListener(listener, new Builder(_clusterName).customizedStates(instanceName, customizedStateName),
+ ChangeType.CUSTOMIZED_STATE, new EventType[] { EventType.NodeChildrenChanged
+ });
+ }
+
+ @Override
public void addExternalViewChangeListener(ExternalViewChangeListener listener) throws Exception {
addListener(listener, new Builder(_clusterName).externalViews(), ChangeType.EXTERNAL_VIEW,
new EventType[] { EventType.NodeChildrenChanged });
diff --git a/helix-core/src/test/java/org/apache/helix/controller/stages/DummyClusterManager.java b/helix-core/src/test/java/org/apache/helix/controller/stages/DummyClusterManager.java
index 076f115..84d413b 100644
--- a/helix-core/src/test/java/org/apache/helix/controller/stages/DummyClusterManager.java
+++ b/helix-core/src/test/java/org/apache/helix/controller/stages/DummyClusterManager.java
@@ -37,6 +37,8 @@ import org.apache.helix.api.listeners.ClusterConfigChangeListener;
import org.apache.helix.api.listeners.ConfigChangeListener;
import org.apache.helix.api.listeners.ControllerChangeListener;
import org.apache.helix.api.listeners.CurrentStateChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateAggregationConfigChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.CustomizedViewChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
@@ -132,6 +134,13 @@ public class DummyClusterManager implements HelixManager {
}
@Override
+ public void addCustomizedStateChangeListener(CustomizedStateChangeListener listener,
+ String instanceName, String sessionId) throws Exception {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
public void addExternalViewChangeListener(ExternalViewChangeListener listener) throws Exception {
// TODO Auto-generated method stub
@@ -293,6 +302,13 @@ public class DummyClusterManager implements HelixManager {
}
@Override
+ public void addCustomizedStateAggregationConfigChangeListener(CustomizedStateAggregationConfigChangeListener listener)
+ throws Exception {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
public void addClusterfigChangeListener(ClusterConfigChangeListener listener)
throws Exception {
// TODO Auto-generated method stub
diff --git a/helix-core/src/test/java/org/apache/helix/integration/manager/TestParticipantManager.java b/helix-core/src/test/java/org/apache/helix/integration/manager/TestParticipantManager.java
index 6e972c5..c7a3752 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/manager/TestParticipantManager.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/manager/TestParticipantManager.java
@@ -151,7 +151,7 @@ public class TestParticipantManager extends ZkTestBase {
// check HelixCallback Monitor
Set<ObjectInstance> objs =
_server.queryMBeans(buildCallbackMonitorObjectName(type, clusterName, instanceName), null);
- Assert.assertEquals(objs.size(), 14);
+ Assert.assertEquals(objs.size(), 16);
// check HelixZkClient Monitors
objs =
diff --git a/helix-core/src/test/java/org/apache/helix/mock/MockManager.java b/helix-core/src/test/java/org/apache/helix/mock/MockManager.java
index 01e6b9e..f32160b 100644
--- a/helix-core/src/test/java/org/apache/helix/mock/MockManager.java
+++ b/helix-core/src/test/java/org/apache/helix/mock/MockManager.java
@@ -40,6 +40,8 @@ import org.apache.helix.api.listeners.ConfigChangeListener;
import org.apache.helix.api.listeners.ControllerChangeListener;
import org.apache.helix.api.listeners.CurrentStateChangeListener;
import org.apache.helix.api.listeners.CustomizedViewChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateAggregationConfigChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
import org.apache.helix.api.listeners.InstanceConfigChangeListener;
@@ -136,6 +138,13 @@ public class MockManager implements HelixManager {
}
@Override
+ public void addCustomizedStateChangeListener(CustomizedStateChangeListener listener,
+ String instanceName, String sessionId) {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
public void addExternalViewChangeListener(ExternalViewChangeListener listener) {
// TODO Auto-generated method stub
@@ -164,6 +173,12 @@ public class MockManager implements HelixManager {
}
@Override
+ public void addCustomizedStateAggregationConfigChangeListener(CustomizedStateAggregationConfigChangeListener listener)
+ throws Exception {
+
+ }
+
+ @Override
public void addClusterfigChangeListener(ClusterConfigChangeListener listener)
throws Exception {
diff --git a/helix-core/src/test/java/org/apache/helix/participant/MockZKHelixManager.java b/helix-core/src/test/java/org/apache/helix/participant/MockZKHelixManager.java
index 67efbd5..4abe7c6 100644
--- a/helix-core/src/test/java/org/apache/helix/participant/MockZKHelixManager.java
+++ b/helix-core/src/test/java/org/apache/helix/participant/MockZKHelixManager.java
@@ -39,6 +39,8 @@ import org.apache.helix.api.listeners.ConfigChangeListener;
import org.apache.helix.api.listeners.ControllerChangeListener;
import org.apache.helix.api.listeners.CurrentStateChangeListener;
import org.apache.helix.api.listeners.CustomizedViewChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateAggregationConfigChangeListener;
+import org.apache.helix.api.listeners.CustomizedStateChangeListener;
import org.apache.helix.api.listeners.ExternalViewChangeListener;
import org.apache.helix.api.listeners.IdealStateChangeListener;
import org.apache.helix.api.listeners.InstanceConfigChangeListener;
@@ -124,6 +126,13 @@ public class MockZKHelixManager implements HelixManager {
}
@Override
+ public void addCustomizedStateAggregationConfigChangeListener(CustomizedStateAggregationConfigChangeListener listener)
+ throws Exception {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
public void addClusterfigChangeListener(ClusterConfigChangeListener listener)
throws Exception {
// TODO Auto-generated method stub
@@ -161,6 +170,13 @@ public class MockZKHelixManager implements HelixManager {
}
@Override
+ public void addCustomizedStateChangeListener(CustomizedStateChangeListener listener,
+ String instanceName, String sessionId) throws Exception {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
public void addExternalViewChangeListener(ExternalViewChangeListener listener) throws Exception {
// TODO Auto-generated method stub
[helix] 19/23: fix customized state provider (#928)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 15ad8a08f666e45e29344a2bb6341e1903b395ad
Author: meng <mn...@linkedin.com>
AuthorDate: Tue Apr 7 11:04:40 2020 -0700
fix customized state provider (#928)
Modify customized state provider factory. The new factory can build a customized state provider with either Helix own manager or a customer input manager.
---
.../CustomizedStateProviderFactory.java | 43 +++++++++-------------
1 file changed, 17 insertions(+), 26 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/customizedstate/CustomizedStateProviderFactory.java b/helix-core/src/main/java/org/apache/helix/customizedstate/CustomizedStateProviderFactory.java
index 3e60522..5190a82 100644
--- a/helix-core/src/main/java/org/apache/helix/customizedstate/CustomizedStateProviderFactory.java
+++ b/helix-core/src/main/java/org/apache/helix/customizedstate/CustomizedStateProviderFactory.java
@@ -19,9 +19,9 @@ package org.apache.helix.customizedstate;
* under the License.
*/
-import java.util.HashMap;
-import org.apache.helix.HelixException;
import org.apache.helix.HelixManager;
+import org.apache.helix.HelixManagerFactory;
+import org.apache.helix.InstanceType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,9 +30,6 @@ import org.slf4j.LoggerFactory;
*/
public class CustomizedStateProviderFactory {
private static Logger LOG = LoggerFactory.getLogger(CustomizedStateProvider.class);
- private final HashMap<String, CustomizedStateProvider> _customizedStateProviderMap =
- new HashMap<>();
- private HelixManager _helixManager;
protected CustomizedStateProviderFactory() {
}
@@ -46,34 +43,28 @@ public class CustomizedStateProviderFactory {
return SingletonHelper.INSTANCE;
}
- public CustomizedStateProvider buildCustomizedStateProvider(String instanceName) {
- if (_helixManager == null) {
- throw new HelixException("Helix Manager has not been set yet.");
- }
- return buildCustomizedStateProvider(_helixManager, instanceName);
+ /**
+ * Build a customized state provider based on user input.
+ * @param instanceName The name of the instance
+ * @param clusterName The name of the cluster that has the instance
+ * @param zkAddress The zookeeper address of the cluster
+ * @return CustomizedStateProvider
+ */
+ public CustomizedStateProvider buildCustomizedStateProvider(String instanceName,
+ String clusterName, String zkAddress) {
+ HelixManager helixManager = HelixManagerFactory
+ .getZKHelixManager(clusterName, instanceName, InstanceType.ADMINISTRATOR, zkAddress);
+ return new CustomizedStateProvider(helixManager, instanceName);
}
/**
- * Build a customized state provider based on the specified input. If the instance already has a
- * provider, return it. Otherwise, build a new one and put it in the map.
- * @param helixManager The helix manager that belongs to the instance
+ * Build a customized state provider based on user input.
* @param instanceName The name of the instance
+ * @param helixManager Helix manager provided by user
* @return CustomizedStateProvider
*/
public CustomizedStateProvider buildCustomizedStateProvider(HelixManager helixManager,
String instanceName) {
- synchronized (_customizedStateProviderMap) {
- if (_customizedStateProviderMap.get(instanceName) != null) {
- return _customizedStateProviderMap.get(instanceName);
- }
- CustomizedStateProvider customizedStateProvider =
- new CustomizedStateProvider(helixManager, instanceName);
- _customizedStateProviderMap.put(instanceName, customizedStateProvider);
- return customizedStateProvider;
- }
- }
-
- public void setHelixManager(HelixManager helixManager) {
- _helixManager = helixManager;
+ return new CustomizedStateProvider(helixManager, instanceName);
}
}
[helix] 23/23: use new ZNRecord and update test
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 4c65bfc8472a4b29747860109bd51e2b952b7f49
Author: Meng Zhang <mn...@linkedin.com>
AuthorDate: Mon Apr 13 13:53:18 2020 -0700
use new ZNRecord and update test
---
.../ResourceControllerDataProvider.java | 1 +
.../org/apache/helix/model/CustomizedView.java | 3 ++-
.../TestComputeAndCleanupCustomizedView.java | 2 +-
.../integration/TestCustomizedViewAggregation.java | 24 ++--------------------
.../apache/helix/manager/zk/TestZkHelixAdmin.java | 1 +
.../server/resources/helix/ClusterAccessor.java | 8 ++++----
.../helix/rest/server/TestClusterAccessor.java | 6 +++---
7 files changed, 14 insertions(+), 31 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java b/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java
index a904e80..0f564fb 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java
@@ -19,6 +19,7 @@ package org.apache.helix.controller.dataproviders;
* under the License.
*/
+import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
diff --git a/helix-core/src/main/java/org/apache/helix/model/CustomizedView.java b/helix-core/src/main/java/org/apache/helix/model/CustomizedView.java
index 791829a..4e472ea 100644
--- a/helix-core/src/main/java/org/apache/helix/model/CustomizedView.java
+++ b/helix-core/src/main/java/org/apache/helix/model/CustomizedView.java
@@ -24,7 +24,8 @@ import java.util.Set;
import java.util.TreeMap;
import org.apache.helix.HelixProperty;
-import org.apache.helix.ZNRecord;
+import org.apache.helix.zookeeper.datamodel.ZNRecord;
+
/**
* Customized view is an aggregation (across all instances)
diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestComputeAndCleanupCustomizedView.java b/helix-core/src/test/java/org/apache/helix/integration/TestComputeAndCleanupCustomizedView.java
index 6de2521..061dfcf 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/TestComputeAndCleanupCustomizedView.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/TestComputeAndCleanupCustomizedView.java
@@ -26,7 +26,6 @@ import java.util.Map;
import org.apache.helix.PropertyKey;
import org.apache.helix.TestHelper;
-import org.apache.helix.ZNRecord;
import org.apache.helix.ZkTestHelper;
import org.apache.helix.ZkUnitTestBase;
import org.apache.helix.integration.manager.ClusterControllerManager;
@@ -37,6 +36,7 @@ import org.apache.helix.manager.zk.ZkBaseDataAccessor;
import org.apache.helix.model.CustomizedState;
import org.apache.helix.model.CustomizedStateConfig;
import org.apache.helix.model.CustomizedView;
+import org.apache.helix.zookeeper.datamodel.ZNRecord;
import org.testng.Assert;
import org.testng.annotations.Test;
diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestCustomizedViewAggregation.java b/helix-core/src/test/java/org/apache/helix/integration/TestCustomizedViewAggregation.java
index e270e9f..45be8fe 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/TestCustomizedViewAggregation.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/TestCustomizedViewAggregation.java
@@ -18,7 +18,6 @@ import org.apache.helix.HelixManagerFactory;
import org.apache.helix.InstanceType;
import org.apache.helix.PropertyType;
import org.apache.helix.TestHelper;
-import org.apache.helix.ZNRecord;
import org.apache.helix.ZkUnitTestBase;
import org.apache.helix.customizedstate.CustomizedStateProvider;
import org.apache.helix.customizedstate.CustomizedStateProviderFactory;
@@ -29,6 +28,7 @@ import org.apache.helix.model.CustomizedStateConfig;
import org.apache.helix.model.CustomizedView;
import org.apache.helix.spectator.RoutingTableProvider;
import org.apache.helix.spectator.RoutingTableSnapshot;
+import org.apache.helix.zookeeper.datamodel.ZNRecord;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
@@ -322,17 +322,6 @@ public class TestCustomizedViewAggregation extends ZkUnitTestBase {
* Set the data sources (customized state types) for routing table provider
* @param customizedStateTypes list of customized state types that routing table provider will include in the snapshot shown to users
*/
- private void setRoutingTableProviderDataSources(List<CustomizedStateType> customizedStateTypes) {
- List<String> customizedViewSources = new ArrayList<>();
- _routingTableProviderDataSources.clear();
- for (CustomizedStateType type : customizedStateTypes) {
- customizedViewSources.add(type.name());
- _routingTableProviderDataSources.add(type.name());
- }
- Map<PropertyType, List<String>> dataSource = new HashMap<>();
- dataSource.put(PropertyType.CUSTOMIZEDVIEW, customizedViewSources);
- _routingTableProvider = new RoutingTableProvider(_spectator, dataSource);
- }
/**
* Set the customized view aggregation config in controller
@@ -394,15 +383,8 @@ public class TestCustomizedViewAggregation extends ZkUnitTestBase {
validateAggregationSnapshot();
- // Set the routing table provider data sources to only Type A and Type B, so users won't see Type C customized view
- // Aggregating: Type A, Type B, Type C
- // Routing table: Type A, Type B
- setRoutingTableProviderDataSources(
- Arrays.asList(CustomizedStateType.TYPE_A, CustomizedStateType.TYPE_B));
- validateAggregationSnapshot();
-
// Aggregating: Type A
- // Routing table: Type A, Type B
+ // Routing table: Type A, Type B, Type C
setAggregationEnabledTypes(Arrays.asList(CustomizedStateType.TYPE_A));
validateAggregationSnapshot();
@@ -450,8 +432,6 @@ public class TestCustomizedViewAggregation extends ZkUnitTestBase {
// Aggregating: Type A, Type B, Type C
// Routing table: Type A, Type B, Type C
- setRoutingTableProviderDataSources(Arrays
- .asList(CustomizedStateType.TYPE_A, CustomizedStateType.TYPE_B, CustomizedStateType.TYPE_C));
setAggregationEnabledTypes(Arrays.asList(CustomizedStateType.TYPE_A, CustomizedStateType.TYPE_B,
CustomizedStateType.TYPE_C));
validateAggregationSnapshot();
diff --git a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
index ee64524..2cb8fa3 100644
--- a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
+++ b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
@@ -78,6 +78,7 @@ import org.testng.AssertJUnit;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
+
public class TestZkHelixAdmin extends ZkUnitTestBase {
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
index a5800b2..cbdf82a 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
@@ -299,7 +299,7 @@ public class ClusterAccessor extends AbstractHelixResource {
@PUT
- @Path("{clusterId}/customized-state-aggregation-config")
+ @Path("{clusterId}/customized-state-config")
public Response addCustomizedStateConfig(@PathParam("clusterId") String clusterId,
String content) {
if (!doesClusterExist(clusterId)) {
@@ -328,7 +328,7 @@ public class ClusterAccessor extends AbstractHelixResource {
}
@DELETE
- @Path("{clusterId}/customized-state-aggregation-config")
+ @Path("{clusterId}/customized-state-config")
public Response removeCustomizedStateConfig(@PathParam("clusterId") String clusterId) {
if (!doesClusterExist(clusterId)) {
return notFound(String.format("Cluster %s does not exist", clusterId));
@@ -348,7 +348,7 @@ public class ClusterAccessor extends AbstractHelixResource {
}
@GET
- @Path("{clusterId}/customized-state-aggregation-config")
+ @Path("{clusterId}/customized-state-config")
public Response getCustomizedStateConfig(@PathParam("clusterId") String clusterId) {
if (!doesClusterExist(clusterId)) {
return notFound(String.format("Cluster %s does not exist", clusterId));
@@ -366,7 +366,7 @@ public class ClusterAccessor extends AbstractHelixResource {
}
@POST
- @Path("{clusterId}/customized-state-aggregation-config")
+ @Path("{clusterId}/customized-state-config")
public Response updateCustomizedStateConfig(@PathParam("clusterId") String clusterId,
@QueryParam("command") String commandStr, @QueryParam("type") String type) {
if (!doesClusterExist(clusterId)) {
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
index 279da84..555f094 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
@@ -1028,7 +1028,7 @@ public class TestClusterAccessor extends AbstractTestClass {
public void testAddCustomizedConfig() throws Exception {
System.out.println("Start test :" + TestHelper.getTestMethodName());
_gSetupTool.addCluster("TestClusterCustomized", true);
- String urlBase = "clusters/TestClusterCustomized/customizedstateconfig/";
+ String urlBase = "clusters/TestClusterCustomized/customized-state-config/";
ZNRecord record = new ZNRecord("TestCustomizedStateConfig");
List<String> testList = new ArrayList<String>();
testList.add("mockType1");
@@ -1071,7 +1071,7 @@ public class TestClusterAccessor extends AbstractTestClass {
public void testDeleteCustomizedConfig() throws IOException {
System.out.println("Start test :" + TestHelper.getTestMethodName());
_gSetupTool.addCluster("TestClusterCustomized", true);
- String urlBase = "clusters/TestClusterCustomized/customizedstateconfig/";
+ String urlBase = "clusters/TestClusterCustomized/customized-state-config/";
ZNRecord record = new ZNRecord("TestCustomizedStateConfig");
List<String> testList = new ArrayList<String>();
testList.add("mockType1");
@@ -1101,7 +1101,7 @@ public class TestClusterAccessor extends AbstractTestClass {
public void testUpdateCustomizedConfig() throws IOException {
System.out.println("Start test :" + TestHelper.getTestMethodName());
_gSetupTool.addCluster("TestClusterCustomized", true);
- String urlBase = "clusters/TestClusterCustomized/customizedstateconfig/";
+ String urlBase = "clusters/TestClusterCustomized/customized-state-config/";
ZNRecord record = new ZNRecord("TestCustomizedStateConfig");
List<String> testList = new ArrayList<String>();
testList.add("mockType1");