You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by hu...@apache.org on 2020/04/10 07:16:56 UTC
[helix] branch customizeView updated: Add cache update/delete in
customized view aggregation stage (#934)
This is an automated email from the ASF dual-hosted git repository.
hulee pushed a commit to branch customizeView
in repository https://gitbox.apache.org/repos/asf/helix.git
The following commit(s) were added to refs/heads/customizeView by this push:
new 4c530a6 Add cache update/delete in customized view aggregation stage (#934)
4c530a6 is described below
commit 4c530a64eda5347035fe184244806058ec8a9294
Author: Meng Zhang <mn...@linkedin.com>
AuthorDate: Fri Apr 10 00:16:47 2020 -0700
Add cache update/delete in customized view aggregation stage (#934)
Add customized view cache update in customized view aggregation stage. This is to ensure that the cache does not have stale data
---
.../helix/common/caches/CustomizedViewCache.java | 8 +++++
.../ResourceControllerDataProvider.java | 35 +++++++++++++++++++++-
.../stages/CustomizedViewAggregationStage.java | 15 ++++++++--
3 files changed, 54 insertions(+), 4 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java b/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java
index cdbc0f8..a62e34b 100644
--- a/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java
+++ b/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java
@@ -86,4 +86,12 @@ public class CustomizedViewCache extends AbstractDataCache<CustomizedView> {
public Map<String, CustomizedView> getCustomizedViewMap() {
return Collections.unmodifiableMap(_customizedViewCache.getPropertyMap());
}
+
+ /**
+ * Return customized view cache as a property cache.
+ * @return
+ */
+ public PropertyCache<CustomizedView> getCustomizedViewCache() {
+ return _customizedViewCache;
+ }
}
diff --git a/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java b/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java
index 57f477f..ebe820c 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java
@@ -39,6 +39,7 @@ import org.apache.helix.controller.pipeline.Pipeline;
import org.apache.helix.controller.stages.MissingTopStateRecord;
import org.apache.helix.model.CustomizedState;
import org.apache.helix.model.CustomizedStateConfig;
+import org.apache.helix.model.CustomizedView;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.ResourceAssignment;
import org.slf4j.Logger;
@@ -258,6 +259,22 @@ public class ResourceControllerDataProvider extends BaseControllerDataProvider {
}
/**
+ * Update the cached customized view map
+ * @param customizedViews
+ */
+ public void updateCustomizedViews(String customizedStateType,
+ List<CustomizedView> customizedViews) {
+ if (!_customizedViewCacheMap.containsKey(customizedStateType)) {
+ CustomizedViewCache customizedViewCache =
+ new CustomizedViewCache(getClusterName(), customizedStateType);
+ _customizedViewCacheMap.put(customizedStateType, customizedViewCache);
+ }
+ for (CustomizedView cv : customizedViews) {
+ _customizedViewCacheMap.get(customizedStateType).getCustomizedViewCache().setProperty(cv);
+ }
+ }
+
+ /**
* Get local cached customized view map
* @return
*/
@@ -281,12 +298,28 @@ public class ResourceControllerDataProvider extends BaseControllerDataProvider {
* @param stateTypeNames
*/
- private void removeCustomizedViewTypes(Set<String> stateTypeNames) {
+ public void removeCustomizedViewTypes(Set<String> stateTypeNames) {
for (String stateType : stateTypeNames) {
_customizedViewCacheMap.remove(stateType);
}
}
+ /**
+ * Remove dead customized views for a certain state type from customized view cache
+ * @param stateType a specific customized state type
+ * @param resourceNames the names of resources whose customized view is stale
+ */
+ public void removeCustomizedViews(String stateType, List<String> resourceNames) {
+ if (!_customizedViewCacheMap.containsKey(stateType)) {
+ logger.warn(String.format("The customized state type : %s is not in the cache", stateType));
+ return;
+ }
+ for (String resourceName : resourceNames) {
+ _customizedViewCacheMap.get(stateType).getCustomizedViewCache()
+ .deletePropertyByName(resourceName);
+ }
+ }
+
public Map<String, Map<String, MissingTopStateRecord>> getMissingTopStateMap() {
return _missingTopStateMap;
}
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/CustomizedViewAggregationStage.java b/helix-core/src/main/java/org/apache/helix/controller/stages/CustomizedViewAggregationStage.java
index cdcdfd2..58888ed 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/stages/CustomizedViewAggregationStage.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/stages/CustomizedViewAggregationStage.java
@@ -21,9 +21,11 @@ package org.apache.helix.controller.stages;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.helix.HelixDataAccessor;
import org.apache.helix.HelixException;
@@ -72,13 +74,16 @@ public class CustomizedViewAggregationStage extends AbstractAsyncBaseStage {
Map<String, CustomizedViewCache> customizedViewCacheMap = cache.getCustomizedViewCacheMap();
- // remove stale customized view type from ZK
+ // remove stale customized view type from ZK and cache
+ Set<String> customizedTypesToRemove = new HashSet<>();
for (String stateType : customizedViewCacheMap.keySet()) {
if (!customizedStateOutput.getAllStateTypes().contains(stateType)) {
LogUtil.logInfo(LOG, _eventId, "Remove customizedView for stateType: " + stateType);
dataAccessor.removeProperty(keyBuilder.customizedView(stateType));
+ customizedTypesToRemove.add(stateType);
}
}
+ cache.removeCustomizedViewTypes(customizedTypesToRemove);
// update customized view
for (String stateType : customizedStateOutput.getAllStateTypes()) {
@@ -100,18 +105,22 @@ public class CustomizedViewAggregationStage extends AbstractAsyncBaseStage {
String resourceName = view.getResourceName();
keys.add(keyBuilder.customizedView(stateType, resourceName));
}
- // add/update customized-views
+ // add/update customized-views from zk and cache
if (updatedCustomizedViews.size() > 0) {
dataAccessor.setChildren(keys, updatedCustomizedViews);
+ cache.updateCustomizedViews(stateType, updatedCustomizedViews);
}
- // remove stale customized views
+ // remove stale customized views from zk and cache
+ List<String> customizedViewToRemove = new ArrayList<>();
for (String resourceName : curCustomizedViews.keySet()) {
if (!resourceMap.keySet().contains(resourceName)) {
LogUtil.logInfo(LOG, _eventId, "Remove customizedView for resource: " + resourceName);
dataAccessor.removeProperty(keyBuilder.customizedView(stateType, resourceName));
+ customizedViewToRemove.add(resourceName);
}
}
+ cache.removeCustomizedViews(stateType, customizedViewToRemove);
} catch (HelixException ex) {
LogUtil.logError(LOG, _eventId,
"Failed to calculate customized view for resource " + resource.getResourceName(), ex);