You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2018/11/29 20:47:52 UTC
[incubator-pinot] 01/01: commit
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch resource_allocation
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit e79b208f946b9e1cff78893e284e855f949935d5
Author: Xiaotian (Jackie) Jiang <xa...@linkedin.com>
AuthorDate: Thu Nov 29 12:47:28 2018 -0800
commit
---
.../helix/core/instance/InstanceAttributes.java | 68 ++++++++++++++++++++++
.../helix/core/instance/InstancePoolUtils.java | 34 +++++++++++
.../helix/core/instance/InstanceRole.java | 5 ++
.../instance/assignment/AssignmentStrategy.java | 4 ++
.../instances/AllInstancesConstraint.java | 22 +++++++
.../FixedNumInstancesPerReplicaConstraint.java | 42 +++++++++++++
.../instances/NumInstancesConstraint.java | 17 ++++++
.../replicagroup/CustomReplicaGroupConstraint.java | 12 ++++
.../DefaultReplicaGroupConstraint.java | 43 ++++++++++++++
.../replicagroup/NoReplicaGroupConstraint.java | 13 +++++
.../replicagroup/ReplicaGroupConstraint.java | 16 +++++
.../instance/selection/result/SelectionResult.java | 32 ++++++++++
.../selection/result/SelectionResultUtils.java | 20 +++++++
.../selection/strategy/SelectionStrategy.java | 12 ++++
14 files changed, 340 insertions(+)
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/InstanceAttributes.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/InstanceAttributes.java
new file mode 100644
index 0000000..434bc96
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/InstanceAttributes.java
@@ -0,0 +1,68 @@
+package com.linkedin.pinot.controller.helix.core.instance;
+
+import com.google.common.base.Preconditions;
+import javax.annotation.Nullable;
+
+
+public class InstanceAttributes {
+ private static final String TAG_DELIMITER = "_";
+ private static final String UNTAGGED = "untagged";
+
+ private final InstanceRole _role;
+ private final String _poolName;
+ private final int _groupId;
+
+ public InstanceAttributes(InstanceRole role, String poolName, int groupId) {
+ _role = role;
+ _poolName = poolName;
+ _groupId = groupId;
+ }
+
+ @Nullable
+ public static InstanceAttributes fromTag(String tag) {
+ String[] parts = tag.split(TAG_DELIMITER);
+ Preconditions.checkArgument(parts.length == 2 || parts.length == 3);
+
+ // For backward-compatible
+ if (parts.length == 2) {
+ // E.g. broker_untagged, server_untagged, minion_untagged
+ if (parts[1].equals(UNTAGGED)) {
+ return null;
+ }
+ // E.g. poolName_OFFLINE, poolName_REALTIME, poolName_BROKER
+ return new InstanceAttributes(InstanceRole.valueOf(parts[1]), parts[0], 0);
+ } else {
+ // E.g. poolName_0_OFFLINE, poolName_1_REALTIME
+ return new InstanceAttributes(InstanceRole.valueOf(parts[2]), parts[0], Integer.parseInt(parts[1]));
+ }
+ }
+
+ public InstanceRole getRole() {
+ return _role;
+ }
+
+ public String getPoolName() {
+ return _poolName;
+ }
+
+ public int getGroupId() {
+ return _groupId;
+ }
+
+ @Override
+ public int hashCode() {
+ return 37 * 37 * _role.hashCode() + 37 * _poolName.hashCode() + _groupId;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj instanceof InstanceAttributes) {
+ InstanceAttributes that = (InstanceAttributes) obj;
+ return _role == that._role && _poolName.equals(that._poolName) && _groupId == that._groupId;
+ }
+ return false;
+ }
+}
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/InstancePoolUtils.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/InstancePoolUtils.java
new file mode 100644
index 0000000..973bdd3
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/InstancePoolUtils.java
@@ -0,0 +1,34 @@
+package com.linkedin.pinot.controller.helix.core.instance;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.helix.model.InstanceConfig;
+
+
+public class InstancePoolUtils {
+ private InstancePoolUtils() {
+ }
+
+ public static Map<Integer, Set<String>> getGroupIdToInstancePoolMap(List<InstanceConfig> instanceConfigs,
+ InstanceRole role, String poolName) {
+ Map<Integer, Set<String>> groupIdToInstancePoolMap = new HashMap<>();
+ for (InstanceConfig instanceConfig : instanceConfigs) {
+ // Skip disabled instances
+ if (!instanceConfig.getInstanceEnabled()) {
+ continue;
+ }
+ for (String tag : instanceConfig.getTags()) {
+ InstanceAttributes instanceAttributes = InstanceAttributes.fromTag(tag);
+ if (instanceAttributes != null && instanceAttributes.getRole() == role && instanceAttributes.getPoolName()
+ .equals(poolName)) {
+ int groupId = instanceAttributes.getGroupId();
+ groupIdToInstancePoolMap.computeIfAbsent(groupId, k -> new HashSet<>()).add(instanceConfig.getInstanceName());
+ }
+ }
+ }
+ return groupIdToInstancePoolMap;
+ }
+}
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/InstanceRole.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/InstanceRole.java
new file mode 100644
index 0000000..9fa958a
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/InstanceRole.java
@@ -0,0 +1,5 @@
+package com.linkedin.pinot.controller.helix.core.instance;
+
+public enum InstanceRole {
+ CONTROLLER, BROKER, OFFLINE, REALTIME, MINION
+}
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/assignment/AssignmentStrategy.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/assignment/AssignmentStrategy.java
new file mode 100644
index 0000000..126b467
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/assignment/AssignmentStrategy.java
@@ -0,0 +1,4 @@
+package com.linkedin.pinot.controller.helix.core.instance.assignment;
+
+public interface AssignmentStrategy {
+}
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/instances/AllInstancesConstraint.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/instances/AllInstancesConstraint.java
new file mode 100644
index 0000000..8ec83aa
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/instances/AllInstancesConstraint.java
@@ -0,0 +1,22 @@
+package com.linkedin.pinot.controller.helix.core.instance.selection.constraint.instances;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+
+public class AllInstancesConstraint implements NumInstancesConstraint {
+
+ @Override
+ public Map<Integer, List<String>> apply(Map<Integer, Set<String>> replicaIdToCandidateInstancesMap) {
+ Map<Integer, List<String>> replicaIdToSelectedInstancesMap = new HashMap<>();
+ for (Map.Entry<Integer, Set<String>> entry : replicaIdToCandidateInstancesMap.entrySet()) {
+ List<String> allInstances = new ArrayList<>(entry.getValue());
+ allInstances.sort(null);
+ replicaIdToSelectedInstancesMap.put(entry.getKey(), allInstances);
+ }
+ return replicaIdToSelectedInstancesMap;
+ }
+}
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/instances/FixedNumInstancesPerReplicaConstraint.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/instances/FixedNumInstancesPerReplicaConstraint.java
new file mode 100644
index 0000000..642737f
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/instances/FixedNumInstancesPerReplicaConstraint.java
@@ -0,0 +1,42 @@
+package com.linkedin.pinot.controller.helix.core.instance.selection.constraint.instances;
+
+import com.google.common.base.Preconditions;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+
+public class FixedNumInstancesPerReplicaConstraint implements NumInstancesConstraint {
+ private final int _numInstancesPerReplica;
+
+ public FixedNumInstancesPerReplicaConstraint(int numInstancesPerReplica) {
+ _numInstancesPerReplica = numInstancesPerReplica;
+ }
+
+ @Override
+ public Map<Integer, List<String>> apply(Map<Integer, Set<String>> replicaIdToCandidateInstancesMap) {
+ Map<Integer, List<String>> replicaIdToSelectedInstancesMap = new HashMap<>();
+ for (Map.Entry<Integer, Set<String>> entry : replicaIdToCandidateInstancesMap.entrySet()) {
+ List<String> allInstances = new ArrayList<>(entry.getValue());
+ int numInstances = allInstances.size();
+ Preconditions.checkState(numInstances >= _numInstancesPerReplica);
+
+ List<String> selectedInstances;
+ if (numInstances == _numInstancesPerReplica) {
+ selectedInstances = allInstances;
+ } else {
+ Collections.shuffle(allInstances);
+ selectedInstances = new ArrayList<>(_numInstancesPerReplica);
+ for (int i = 0; i < _numInstancesPerReplica; i++) {
+ selectedInstances.add(allInstances.get(i));
+ }
+ }
+ selectedInstances.sort(null);
+ replicaIdToSelectedInstancesMap.put(entry.getKey(), selectedInstances);
+ }
+ return replicaIdToSelectedInstancesMap;
+ }
+}
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/instances/NumInstancesConstraint.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/instances/NumInstancesConstraint.java
new file mode 100644
index 0000000..0d05820
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/instances/NumInstancesConstraint.java
@@ -0,0 +1,17 @@
+package com.linkedin.pinot.controller.helix.core.instance.selection.constraint.instances;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+
+public interface NumInstancesConstraint {
+
+ /**
+ * Applies the number of instances constraint, and returns a map from replica Id to selected instances.
+ *
+ * @param replicaIdToCandidateInstancesMap Map from replica Id to candidate instances
+ * @return Map from replica Id to selected instances
+ */
+ Map<Integer, List<String>> apply(Map<Integer, Set<String>> replicaIdToCandidateInstancesMap);
+}
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/replicagroup/CustomReplicaGroupConstraint.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/replicagroup/CustomReplicaGroupConstraint.java
new file mode 100644
index 0000000..bf754e2
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/replicagroup/CustomReplicaGroupConstraint.java
@@ -0,0 +1,12 @@
+package com.linkedin.pinot.controller.helix.core.instance.selection.constraint.replicagroup;
+
+import java.util.Map;
+import java.util.Set;
+
+
+public class CustomReplicaGroupConstraint implements ReplicaGroupConstraint {
+ @Override
+ public Map<Integer, Set<String>> apply(Map<Integer, Set<String>> groupIdToInstancePoolMap) {
+ return null;
+ }
+}
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/replicagroup/DefaultReplicaGroupConstraint.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/replicagroup/DefaultReplicaGroupConstraint.java
new file mode 100644
index 0000000..07bebd2
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/replicagroup/DefaultReplicaGroupConstraint.java
@@ -0,0 +1,43 @@
+package com.linkedin.pinot.controller.helix.core.instance.selection.constraint.replicagroup;
+
+import com.google.common.base.Preconditions;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+
+public class DefaultReplicaGroupConstraint implements ReplicaGroupConstraint {
+ private final int _numReplicas;
+
+ public DefaultReplicaGroupConstraint(int numReplicas) {
+ _numReplicas = numReplicas;
+ }
+
+ @Override
+ public Map<Integer, Set<String>> apply(Map<Integer, Set<String>> groupIdToInstancePoolMap) {
+ List<Integer> allGroups = new ArrayList<>(groupIdToInstancePoolMap.keySet());
+ int numGroups = allGroups.size();
+ Preconditions.checkState(numGroups >= _numReplicas);
+
+ List<Integer> selectedGroups;
+ if (numGroups == _numReplicas) {
+ selectedGroups = allGroups;
+ } else {
+ Collections.shuffle(allGroups);
+ selectedGroups = new ArrayList<>(_numReplicas);
+ for (int i = 0; i < _numReplicas; i++) {
+ selectedGroups.add(allGroups.get(i));
+ }
+ }
+ selectedGroups.sort(null);
+
+ Map<Integer, Set<String>> replicaIdToCandidateInstancesMap = new HashMap<>();
+ for (int i = 0; i < _numReplicas; i++) {
+ replicaIdToCandidateInstancesMap.put(i, groupIdToInstancePoolMap.get(selectedGroups.get(i)));
+ }
+ return replicaIdToCandidateInstancesMap;
+ }
+}
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/replicagroup/NoReplicaGroupConstraint.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/replicagroup/NoReplicaGroupConstraint.java
new file mode 100644
index 0000000..dd1d879
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/replicagroup/NoReplicaGroupConstraint.java
@@ -0,0 +1,13 @@
+package com.linkedin.pinot.controller.helix.core.instance.selection.constraint.replicagroup;
+
+import java.util.Map;
+import java.util.Set;
+
+
+public class NoReplicaGroupConstraint implements ReplicaGroupConstraint {
+
+ @Override
+ public Map<Integer, Set<String>> apply(Map<Integer, Set<String>> groupIdToInstancePoolMap) {
+ return null;
+ }
+}
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/replicagroup/ReplicaGroupConstraint.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/replicagroup/ReplicaGroupConstraint.java
new file mode 100644
index 0000000..86b8f3f
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/replicagroup/ReplicaGroupConstraint.java
@@ -0,0 +1,16 @@
+package com.linkedin.pinot.controller.helix.core.instance.selection.constraint.replicagroup;
+
+import java.util.Map;
+import java.util.Set;
+
+
+public interface ReplicaGroupConstraint {
+
+ /**
+ * Applies the replica group constraint, and returns a map from replica Id to candidate instances.
+ *
+ * @param groupIdToInstancePoolMap Map from group Id to instance pool
+ * @return Map from replica Id to candidate instances
+ */
+ Map<Integer, Set<String>> apply(Map<Integer, Set<String>> groupIdToInstancePoolMap);
+}
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/result/SelectionResult.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/result/SelectionResult.java
new file mode 100644
index 0000000..4fc664b
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/result/SelectionResult.java
@@ -0,0 +1,32 @@
+package com.linkedin.pinot.controller.helix.core.instance.selection.result;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+
+public class SelectionResult {
+ private final Map<Integer, List<String>> _replicaIdToInstancesMap;
+
+ public SelectionResult() {
+ _replicaIdToInstancesMap = new HashMap<>();
+ }
+
+ public SelectionResult(Map<Integer, List<String>> replicaIdToInstancesMap) {
+ _replicaIdToInstancesMap = replicaIdToInstancesMap;
+ }
+
+ public int getNumReplicas() {
+ return _replicaIdToInstancesMap.size();
+ }
+
+ public List<String> getInstances(int replicaId) {
+ return _replicaIdToInstancesMap.get(replicaId);
+ }
+
+ public void setInstances(int replicaId, List<String> instances) {
+ _replicaIdToInstancesMap.set(replicaId, instances);
+ }
+}
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/result/SelectionResultUtils.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/result/SelectionResultUtils.java
new file mode 100644
index 0000000..dcddd92
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/result/SelectionResultUtils.java
@@ -0,0 +1,20 @@
+package com.linkedin.pinot.controller.helix.core.instance.selection.result;
+
+import java.util.Map;
+import org.apache.helix.ZNRecord;
+import org.apache.helix.store.zk.ZkHelixPropertyStore;
+
+
+public class SelectionResultUtils {
+ private SelectionResultUtils() {
+ }
+
+ public static Map<String, SelectionResult> readFromZK(ZkHelixPropertyStore<ZNRecord> propertyStore, String resource) {
+ throw new UnsupportedOperationException();
+ }
+
+ public static boolean writeToZK(ZkHelixPropertyStore<ZNRecord> propertyStore, String resource,
+ Map<String, SelectionResult> selectionResults) {
+ throw new UnsupportedOperationException();
+ }
+}
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/strategy/SelectionStrategy.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/strategy/SelectionStrategy.java
new file mode 100644
index 0000000..c462627
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/strategy/SelectionStrategy.java
@@ -0,0 +1,12 @@
+package com.linkedin.pinot.controller.helix.core.instance.selection.strategy;
+
+import com.linkedin.pinot.controller.helix.core.instance.InstanceRole;
+import com.linkedin.pinot.controller.helix.core.instance.selection.result.SelectionResult;
+import java.util.Map;
+import java.util.Set;
+
+
+public interface SelectionStrategy {
+
+ SelectionResult selectInstances(Map<Integer, Set<String>> groupIdToInstancePoolMap, )
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org