You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by jx...@apache.org on 2021/12/03 23:32:29 UTC
[helix] branch master updated: Implement RestSnapShot and substitute the kv maps in HelixDataAccessorWrapper to a RestSnapShot object (#1913)
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
The following commit(s) were added to refs/heads/master by this push:
new bf04419 Implement RestSnapShot and substitute the kv maps in HelixDataAccessorWrapper to a RestSnapShot object (#1913)
bf04419 is described below
commit bf04419940855b78cacdffc01b2cf1392d38a5cd
Author: xyuanlu <xy...@gmail.com>
AuthorDate: Fri Dec 3 15:32:21 2021 -0800
Implement RestSnapShot and substitute the kv maps in HelixDataAccessorWrapper to a RestSnapShot object (#1913)
* implement RestSnapShot and substitute the kv maps in HelixDataAccessorWrapper with RestSnapShot object
---
.../java/org/apache/helix/datamodel/Snapshot.java | 25 ++++++++
.../helix/manager/zk/ZKHelixDataAccessor.java | 2 +-
.../api/OperationInterface.java | 8 +++
.../rest/common/HelixDataAccessorWrapper.java | 74 +++++++++++++++++-----
.../helix/rest/common/RestSnapShotSimpleImpl.java | 29 +++++++++
.../helix/rest/common/datamodel/RestSnapShot.java | 47 +++++++++++---
6 files changed, 159 insertions(+), 26 deletions(-)
diff --git a/helix-common/src/main/java/org/apache/helix/datamodel/Snapshot.java b/helix-common/src/main/java/org/apache/helix/datamodel/Snapshot.java
new file mode 100644
index 0000000..bb2858a
--- /dev/null
+++ b/helix-common/src/main/java/org/apache/helix/datamodel/Snapshot.java
@@ -0,0 +1,25 @@
+package org.apache.helix.datamodel;
+
+import java.util.HashMap;
+import java.util.Map;
+
+
+public abstract class Snapshot<K, V> {
+ protected Map<K, V> _valueCache;
+
+ public Snapshot() {
+ _valueCache = new HashMap<>();
+ }
+
+ public V getValue(K key) {
+ return _valueCache.get(key);
+ }
+
+ public void updateValue(K key, V value) {
+ _valueCache.put(key, value);
+ }
+
+ public boolean containsKey(K key) {
+ return _valueCache.containsKey(key);
+ }
+}
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixDataAccessor.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixDataAccessor.java
index 3424871..7e571c8 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixDataAccessor.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixDataAccessor.java
@@ -55,7 +55,7 @@ import org.slf4j.LoggerFactory;
public class ZKHelixDataAccessor implements HelixDataAccessor {
private static Logger LOG = LoggerFactory.getLogger(ZKHelixDataAccessor.class);
private final BaseDataAccessor<ZNRecord> _baseDataAccessor;
- private final String _clusterName;
+ protected final String _clusterName;
private final Builder _propertyKeyBuilder;
private final GroupCommit _groupCommit = new GroupCommit();
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/api/OperationInterface.java b/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/api/OperationInterface.java
index 2062ce9..7be805f 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/api/OperationInterface.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/api/OperationInterface.java
@@ -19,14 +19,22 @@ package org.apache.helix.rest.clusterMaintenanceService.api;
* under the License.
*/
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
+import java.util.List;
import java.util.Map;
+import org.apache.helix.PropertyType;
import org.apache.helix.rest.clusterMaintenanceService.MaintenanceManagementInstanceInfo;
import org.apache.helix.rest.common.datamodel.RestSnapShot;
public interface OperationInterface {
+
+ List<PropertyType> PROPERTY_TYPE_LIST = new ArrayList<>(Arrays
+ .asList(PropertyType.IDEALSTATES, PropertyType.EXTERNALVIEW, PropertyType.STATEMODELDEFS));
+
// operation check
MaintenanceManagementInstanceInfo operationCheckForTakeSingleInstance(String instanceName,
Map<String, String> operationConfig, RestSnapShot sn);
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/common/HelixDataAccessorWrapper.java b/helix-rest/src/main/java/org/apache/helix/rest/common/HelixDataAccessorWrapper.java
index cd3cbf4..e47fd03 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/common/HelixDataAccessorWrapper.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/common/HelixDataAccessorWrapper.java
@@ -37,10 +37,13 @@ import com.codahale.metrics.SharedMetricRegistries;
import com.codahale.metrics.Timer;
import org.apache.helix.HelixProperty;
import org.apache.helix.PropertyKey;
+import org.apache.helix.PropertyType;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
+import org.apache.helix.model.IdealState;
import org.apache.helix.model.RESTConfig;
import org.apache.helix.rest.client.CustomRestClient;
import org.apache.helix.rest.client.CustomRestClientFactory;
+import org.apache.helix.rest.common.datamodel.RestSnapShot;
import org.apache.helix.rest.server.service.InstanceService;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
import org.slf4j.Logger;
@@ -65,11 +68,11 @@ public class HelixDataAccessorWrapper extends ZKHelixDataAccessor {
private static final String CUSTOM_PARTITION_CHECK_HTTP_REQUESTS_DURATION =
MetricRegistry.name(InstanceService.class, "custom_partition_check_http_requests_duration");
- private final Map<PropertyKey, HelixProperty> _propertyCache = new HashMap<>();
- private final Map<PropertyKey, List<String>> _batchNameCache = new HashMap<>();
protected String _namespace;
protected CustomRestClient _restClient;
+ private RestSnapShotSimpleImpl _restSnapShot;
+
/**
* @deprecated Because a namespace is required, please use the other constructors.
*
@@ -91,6 +94,7 @@ public class HelixDataAccessorWrapper extends ZKHelixDataAccessor {
super(dataAccessor);
_restClient = customRestClient;
_namespace = namespace;
+ _restSnapShot = new RestSnapShotSimpleImpl(_clusterName);
}
public Map<String, Map<String, Boolean>> getAllPartitionsHealthOnLiveInstance(
@@ -103,8 +107,9 @@ public class HelixDataAccessorWrapper extends ZKHelixDataAccessor {
* and customized REST API call.
*
* @param restConfig restConfig for the cluster contains customize REST API endpoint
- * @param customPayLoads user passed in customized payloads
- * @param skipZKRead skip the ZK read if this flag is true
+ * @param customPayLoads User passed in customized payloads
+ * @param skipZKRead Query the participant end point directly rather than fetch for
+ * partition health from ZK if this flag is true.
* @return A map of instance -> partition -> healthy or not (boolean).
*/
public Map<String, Map<String, Boolean>> getAllPartitionsHealthOnLiveInstance(
@@ -153,10 +158,10 @@ public class HelixDataAccessorWrapper extends ZKHelixDataAccessor {
* API will return nothing.
*
* @param instance instance to query
- * @param partitionHealthRecord retrieved partition health data from ZK. Could be emptry if we skip reading from ZK.
+ * @param partitionHealthRecord Retrieved partition health data from ZK. Could be emptry if we skip reading from ZK.
* @param restConfig restConfig for the cluster contains custom API endpoint
- * @param customPayLoads user passed in customized payloads
- * @param requireFullRead get all the partition status from custom API endpoint if it is true. It should skip
+ * @param customPayLoads User passed in customized payloads
+ * @param requireFullRead Get all the partition status from custom API endpoint if it is true. It should skip
* the payload of "PARTITION : list of partition need to be fetch" in REST call.
* @return A map of instance -> partition -> healthy or not (boolean).
*/
@@ -205,25 +210,62 @@ public class HelixDataAccessorWrapper extends ZKHelixDataAccessor {
}
}
+ public RestSnapShot getRestSnapShot() {
+ return _restSnapShot;
+ }
+
@Override
public <T extends HelixProperty> T getProperty(PropertyKey key) {
- if (_propertyCache.containsKey(key)) {
- return (T) _propertyCache.get(key);
+ T property = _restSnapShot.getProperty(key);
+ if (property == null) {
+ property = super.getProperty(key);
+ _restSnapShot.updateValue(key, property);
}
- T property = super.getProperty(key);
- _propertyCache.put(key, property);
+
return property;
}
@Override
public List<String> getChildNames(PropertyKey key) {
- if (_batchNameCache.containsKey(key)) {
- return _batchNameCache.get(key);
+ List<String> names = _restSnapShot.getChildNames(key);
+ if (names == null) {
+ names = super.getChildNames(key);
+ _restSnapShot.updateChildNames(key, names);
}
+ return names;
+ }
- List<String> names = super.getChildNames(key);
- _batchNameCache.put(key, names);
+ public void fetchIdealStatesExternalViewStateModel() {
+ PropertyKey.Builder propertyKeyBuilder = this.keyBuilder();
+ List<String> resources = getChildNames(propertyKeyBuilder.idealStates());
- return names;
+ for (String resourceName : resources) {
+ getProperty(propertyKeyBuilder.idealStates(resourceName));
+ IdealState externalView = getProperty(propertyKeyBuilder.externalView(resourceName));
+ if (externalView != null) {
+ String stateModeDef = externalView.getStateModelDefRef();
+ getProperty(propertyKeyBuilder.stateModelDef(stateModeDef));
+ }
+ }
+ _restSnapShot.addPropertyType(PropertyType.IDEALSTATES);
+ _restSnapShot.addPropertyType(PropertyType.EXTERNALVIEW);
+ _restSnapShot.addPropertyType(PropertyType.STATEMODELDEFS);
+ }
+
+ public void populateCache(List<PropertyType> propertyTypes) {
+ for (PropertyType propertyType : propertyTypes) {
+ switch (propertyType) {
+ case IDEALSTATES:
+ case EXTERNALVIEW:
+ case STATEMODELDEFS: {
+ if (!_restSnapShot.containsProperty(propertyType)) {
+ fetchIdealStatesExternalViewStateModel();
+ }
+ break;
+ }
+ default:
+ throw new UnsupportedOperationException("type selection is not supported yet!");
+ }
+ }
}
}
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/common/RestSnapShotSimpleImpl.java b/helix-rest/src/main/java/org/apache/helix/rest/common/RestSnapShotSimpleImpl.java
new file mode 100644
index 0000000..6ba828d
--- /dev/null
+++ b/helix-rest/src/main/java/org/apache/helix/rest/common/RestSnapShotSimpleImpl.java
@@ -0,0 +1,29 @@
+package org.apache.helix.rest.common;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.helix.PropertyKey;
+import org.apache.helix.rest.common.datamodel.RestSnapShot;
+
+
+public class RestSnapShotSimpleImpl extends RestSnapShot {
+ private final Map<PropertyKey, List<String>> _childNodesCache;
+
+ public RestSnapShotSimpleImpl(String clusterName) {
+ super(clusterName);
+ _childNodesCache = new HashMap<>();
+ }
+
+ public List<String> getChildNames(PropertyKey key) {
+ if (_childNodesCache.containsKey(key)) {
+ return _childNodesCache.get(key);
+ }
+ return null;
+ }
+
+ public void updateChildNames(PropertyKey key, List<String> children) {
+ _childNodesCache.put(key, children);
+ }
+}
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/common/datamodel/RestSnapShot.java b/helix-rest/src/main/java/org/apache/helix/rest/common/datamodel/RestSnapShot.java
index d4b8dbc..c979c7b 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/common/datamodel/RestSnapShot.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/common/datamodel/RestSnapShot.java
@@ -1,5 +1,4 @@
package org.apache.helix.rest.common.datamodel;
-
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -19,18 +18,48 @@ package org.apache.helix.rest.common.datamodel;
* under the License.
*/
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.helix.HelixProperty;
+import org.apache.helix.PropertyKey;
+import org.apache.helix.PropertyType;
+import org.apache.helix.datamodel.Snapshot;
+
/* This Snapshot can extend Snapshot from common/core module
* once there is more generic snapshot.
+ * An Snapshot object should contain all the Helix related info that an implementation of
+ * OperationAbstractClass would need.
*/
-public class RestSnapShot {
- /* An Snapshot object should contain all the Helix related info that an implementation of
- * OperationAbstractClass would need.
- */
+// TODO: Future: Support hierarchical Snapshot type for other services besides cluster MaintenanceService.
+
+public class RestSnapShot extends Snapshot<PropertyKey, HelixProperty> {
+
+ private Set<PropertyType> _propertyTypes;
+ private String _clusterName;
+
+ public RestSnapShot(String clusterName) {
+ _propertyTypes = new HashSet<>();
+ _clusterName = clusterName;
+ }
+
+ public void addPropertyType(PropertyType propertyType) {
+ _propertyTypes.add(propertyType);
+ }
+
+ public boolean containsProperty(PropertyType propertyType) {
+ return _propertyTypes.contains(propertyType);
+ }
- // TODO: Next: Add a KV map and get function for the first version in next change.
- // TODO: Define a Enum class for all Helix info types like ExternalView, InstanceConfig etc. An
- // implementation of OperationAbstractClass will need to define what are the types needed.
+ public <T extends HelixProperty> T getProperty(PropertyKey key) {
+ if (containsKey(key)) {
+ return (T) getValue(key);
+ }
+ return null;
+ }
- // TODO: Future: Support hierarchical Snapshot type for other services besides cluster MaintenanceService.
+ public String getClusterName() {
+ return _clusterName;
+ }
}
\ No newline at end of file