You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2018/11/29 20:47:51 UTC
[incubator-pinot] branch resource_allocation created (now e79b208)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a change to branch resource_allocation
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
at e79b208 commit
This branch includes the following new commits:
new e79b208 commit
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: commit
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch resource_allocation
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit e79b208f946b9e1cff78893e284e855f949935d5
Author: Xiaotian (Jackie) Jiang <xa...@linkedin.com>
AuthorDate: Thu Nov 29 12:47:28 2018 -0800
commit
---
.../helix/core/instance/InstanceAttributes.java | 68 ++++++++++++++++++++++
.../helix/core/instance/InstancePoolUtils.java | 34 +++++++++++
.../helix/core/instance/InstanceRole.java | 5 ++
.../instance/assignment/AssignmentStrategy.java | 4 ++
.../instances/AllInstancesConstraint.java | 22 +++++++
.../FixedNumInstancesPerReplicaConstraint.java | 42 +++++++++++++
.../instances/NumInstancesConstraint.java | 17 ++++++
.../replicagroup/CustomReplicaGroupConstraint.java | 12 ++++
.../DefaultReplicaGroupConstraint.java | 43 ++++++++++++++
.../replicagroup/NoReplicaGroupConstraint.java | 13 +++++
.../replicagroup/ReplicaGroupConstraint.java | 16 +++++
.../instance/selection/result/SelectionResult.java | 32 ++++++++++
.../selection/result/SelectionResultUtils.java | 20 +++++++
.../selection/strategy/SelectionStrategy.java | 12 ++++
14 files changed, 340 insertions(+)
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/InstanceAttributes.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/InstanceAttributes.java
new file mode 100644
index 0000000..434bc96
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/InstanceAttributes.java
@@ -0,0 +1,68 @@
+package com.linkedin.pinot.controller.helix.core.instance;
+
+import com.google.common.base.Preconditions;
+import javax.annotation.Nullable;
+
+
+public class InstanceAttributes {
+ private static final String TAG_DELIMITER = "_";
+ private static final String UNTAGGED = "untagged";
+
+ private final InstanceRole _role;
+ private final String _poolName;
+ private final int _groupId;
+
+ public InstanceAttributes(InstanceRole role, String poolName, int groupId) {
+ _role = role;
+ _poolName = poolName;
+ _groupId = groupId;
+ }
+
+ @Nullable
+ public static InstanceAttributes fromTag(String tag) {
+ String[] parts = tag.split(TAG_DELIMITER);
+ Preconditions.checkArgument(parts.length == 2 || parts.length == 3);
+
+ // For backward-compatible
+ if (parts.length == 2) {
+ // E.g. broker_untagged, server_untagged, minion_untagged
+ if (parts[1].equals(UNTAGGED)) {
+ return null;
+ }
+ // E.g. poolName_OFFLINE, poolName_REALTIME, poolName_BROKER
+ return new InstanceAttributes(InstanceRole.valueOf(parts[1]), parts[0], 0);
+ } else {
+ // E.g. poolName_0_OFFLINE, poolName_1_REALTIME
+ return new InstanceAttributes(InstanceRole.valueOf(parts[2]), parts[0], Integer.parseInt(parts[1]));
+ }
+ }
+
+ public InstanceRole getRole() {
+ return _role;
+ }
+
+ public String getPoolName() {
+ return _poolName;
+ }
+
+ public int getGroupId() {
+ return _groupId;
+ }
+
+ @Override
+ public int hashCode() {
+ return 37 * 37 * _role.hashCode() + 37 * _poolName.hashCode() + _groupId;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj instanceof InstanceAttributes) {
+ InstanceAttributes that = (InstanceAttributes) obj;
+ return _role == that._role && _poolName.equals(that._poolName) && _groupId == that._groupId;
+ }
+ return false;
+ }
+}
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/InstancePoolUtils.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/InstancePoolUtils.java
new file mode 100644
index 0000000..973bdd3
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/InstancePoolUtils.java
@@ -0,0 +1,34 @@
+package com.linkedin.pinot.controller.helix.core.instance;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.helix.model.InstanceConfig;
+
+
+public class InstancePoolUtils {
+ private InstancePoolUtils() {
+ }
+
+ public static Map<Integer, Set<String>> getGroupIdToInstancePoolMap(List<InstanceConfig> instanceConfigs,
+ InstanceRole role, String poolName) {
+ Map<Integer, Set<String>> groupIdToInstancePoolMap = new HashMap<>();
+ for (InstanceConfig instanceConfig : instanceConfigs) {
+ // Skip disabled instances
+ if (!instanceConfig.getInstanceEnabled()) {
+ continue;
+ }
+ for (String tag : instanceConfig.getTags()) {
+ InstanceAttributes instanceAttributes = InstanceAttributes.fromTag(tag);
+ if (instanceAttributes != null && instanceAttributes.getRole() == role && instanceAttributes.getPoolName()
+ .equals(poolName)) {
+ int groupId = instanceAttributes.getGroupId();
+ groupIdToInstancePoolMap.computeIfAbsent(groupId, k -> new HashSet<>()).add(instanceConfig.getInstanceName());
+ }
+ }
+ }
+ return groupIdToInstancePoolMap;
+ }
+}
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/InstanceRole.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/InstanceRole.java
new file mode 100644
index 0000000..9fa958a
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/InstanceRole.java
@@ -0,0 +1,5 @@
+package com.linkedin.pinot.controller.helix.core.instance;
+
+public enum InstanceRole {
+ CONTROLLER, BROKER, OFFLINE, REALTIME, MINION
+}
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/assignment/AssignmentStrategy.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/assignment/AssignmentStrategy.java
new file mode 100644
index 0000000..126b467
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/assignment/AssignmentStrategy.java
@@ -0,0 +1,4 @@
+package com.linkedin.pinot.controller.helix.core.instance.assignment;
+
+public interface AssignmentStrategy {
+}
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/instances/AllInstancesConstraint.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/instances/AllInstancesConstraint.java
new file mode 100644
index 0000000..8ec83aa
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/instances/AllInstancesConstraint.java
@@ -0,0 +1,22 @@
+package com.linkedin.pinot.controller.helix.core.instance.selection.constraint.instances;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+
+public class AllInstancesConstraint implements NumInstancesConstraint {
+
+ @Override
+ public Map<Integer, List<String>> apply(Map<Integer, Set<String>> replicaIdToCandidateInstancesMap) {
+ Map<Integer, List<String>> replicaIdToSelectedInstancesMap = new HashMap<>();
+ for (Map.Entry<Integer, Set<String>> entry : replicaIdToCandidateInstancesMap.entrySet()) {
+ List<String> allInstances = new ArrayList<>(entry.getValue());
+ allInstances.sort(null);
+ replicaIdToSelectedInstancesMap.put(entry.getKey(), allInstances);
+ }
+ return replicaIdToSelectedInstancesMap;
+ }
+}
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/instances/FixedNumInstancesPerReplicaConstraint.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/instances/FixedNumInstancesPerReplicaConstraint.java
new file mode 100644
index 0000000..642737f
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/instances/FixedNumInstancesPerReplicaConstraint.java
@@ -0,0 +1,42 @@
+package com.linkedin.pinot.controller.helix.core.instance.selection.constraint.instances;
+
+import com.google.common.base.Preconditions;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+
+public class FixedNumInstancesPerReplicaConstraint implements NumInstancesConstraint {
+ private final int _numInstancesPerReplica;
+
+ public FixedNumInstancesPerReplicaConstraint(int numInstancesPerReplica) {
+ _numInstancesPerReplica = numInstancesPerReplica;
+ }
+
+ @Override
+ public Map<Integer, List<String>> apply(Map<Integer, Set<String>> replicaIdToCandidateInstancesMap) {
+ Map<Integer, List<String>> replicaIdToSelectedInstancesMap = new HashMap<>();
+ for (Map.Entry<Integer, Set<String>> entry : replicaIdToCandidateInstancesMap.entrySet()) {
+ List<String> allInstances = new ArrayList<>(entry.getValue());
+ int numInstances = allInstances.size();
+ Preconditions.checkState(numInstances >= _numInstancesPerReplica);
+
+ List<String> selectedInstances;
+ if (numInstances == _numInstancesPerReplica) {
+ selectedInstances = allInstances;
+ } else {
+ Collections.shuffle(allInstances);
+ selectedInstances = new ArrayList<>(_numInstancesPerReplica);
+ for (int i = 0; i < _numInstancesPerReplica; i++) {
+ selectedInstances.add(allInstances.get(i));
+ }
+ }
+ selectedInstances.sort(null);
+ replicaIdToSelectedInstancesMap.put(entry.getKey(), selectedInstances);
+ }
+ return replicaIdToSelectedInstancesMap;
+ }
+}
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/instances/NumInstancesConstraint.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/instances/NumInstancesConstraint.java
new file mode 100644
index 0000000..0d05820
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/instances/NumInstancesConstraint.java
@@ -0,0 +1,17 @@
+package com.linkedin.pinot.controller.helix.core.instance.selection.constraint.instances;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+
+public interface NumInstancesConstraint {
+
+ /**
+ * Applies the number of instances constraint, and returns a map from replica Id to selected instances.
+ *
+ * @param replicaIdToCandidateInstancesMap Map from replica Id to candidate instances
+ * @return Map from replica Id to selected instances
+ */
+ Map<Integer, List<String>> apply(Map<Integer, Set<String>> replicaIdToCandidateInstancesMap);
+}
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/replicagroup/CustomReplicaGroupConstraint.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/replicagroup/CustomReplicaGroupConstraint.java
new file mode 100644
index 0000000..bf754e2
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/replicagroup/CustomReplicaGroupConstraint.java
@@ -0,0 +1,12 @@
+package com.linkedin.pinot.controller.helix.core.instance.selection.constraint.replicagroup;
+
+import java.util.Map;
+import java.util.Set;
+
+
+public class CustomReplicaGroupConstraint implements ReplicaGroupConstraint {
+ @Override
+ public Map<Integer, Set<String>> apply(Map<Integer, Set<String>> groupIdToInstancePoolMap) {
+ return null;
+ }
+}
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/replicagroup/DefaultReplicaGroupConstraint.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/replicagroup/DefaultReplicaGroupConstraint.java
new file mode 100644
index 0000000..07bebd2
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/replicagroup/DefaultReplicaGroupConstraint.java
@@ -0,0 +1,43 @@
+package com.linkedin.pinot.controller.helix.core.instance.selection.constraint.replicagroup;
+
+import com.google.common.base.Preconditions;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+
+public class DefaultReplicaGroupConstraint implements ReplicaGroupConstraint {
+ private final int _numReplicas;
+
+ public DefaultReplicaGroupConstraint(int numReplicas) {
+ _numReplicas = numReplicas;
+ }
+
+ @Override
+ public Map<Integer, Set<String>> apply(Map<Integer, Set<String>> groupIdToInstancePoolMap) {
+ List<Integer> allGroups = new ArrayList<>(groupIdToInstancePoolMap.keySet());
+ int numGroups = allGroups.size();
+ Preconditions.checkState(numGroups >= _numReplicas);
+
+ List<Integer> selectedGroups;
+ if (numGroups == _numReplicas) {
+ selectedGroups = allGroups;
+ } else {
+ Collections.shuffle(allGroups);
+ selectedGroups = new ArrayList<>(_numReplicas);
+ for (int i = 0; i < _numReplicas; i++) {
+ selectedGroups.add(allGroups.get(i));
+ }
+ }
+ selectedGroups.sort(null);
+
+ Map<Integer, Set<String>> replicaIdToCandidateInstancesMap = new HashMap<>();
+ for (int i = 0; i < _numReplicas; i++) {
+ replicaIdToCandidateInstancesMap.put(i, groupIdToInstancePoolMap.get(selectedGroups.get(i)));
+ }
+ return replicaIdToCandidateInstancesMap;
+ }
+}
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/replicagroup/NoReplicaGroupConstraint.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/replicagroup/NoReplicaGroupConstraint.java
new file mode 100644
index 0000000..dd1d879
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/replicagroup/NoReplicaGroupConstraint.java
@@ -0,0 +1,13 @@
+package com.linkedin.pinot.controller.helix.core.instance.selection.constraint.replicagroup;
+
+import java.util.Map;
+import java.util.Set;
+
+
+public class NoReplicaGroupConstraint implements ReplicaGroupConstraint {
+
+ @Override
+ public Map<Integer, Set<String>> apply(Map<Integer, Set<String>> groupIdToInstancePoolMap) {
+ return null;
+ }
+}
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/replicagroup/ReplicaGroupConstraint.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/replicagroup/ReplicaGroupConstraint.java
new file mode 100644
index 0000000..86b8f3f
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/constraint/replicagroup/ReplicaGroupConstraint.java
@@ -0,0 +1,16 @@
+package com.linkedin.pinot.controller.helix.core.instance.selection.constraint.replicagroup;
+
+import java.util.Map;
+import java.util.Set;
+
+
+public interface ReplicaGroupConstraint {
+
+ /**
+ * Applies the replica group constraint, and returns a map from replica Id to candidate instances.
+ *
+ * @param groupIdToInstancePoolMap Map from group Id to instance pool
+ * @return Map from replica Id to candidate instances
+ */
+ Map<Integer, Set<String>> apply(Map<Integer, Set<String>> groupIdToInstancePoolMap);
+}
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/result/SelectionResult.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/result/SelectionResult.java
new file mode 100644
index 0000000..4fc664b
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/result/SelectionResult.java
@@ -0,0 +1,32 @@
+package com.linkedin.pinot.controller.helix.core.instance.selection.result;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+
+public class SelectionResult {
+ private final Map<Integer, List<String>> _replicaIdToInstancesMap;
+
+ public SelectionResult() {
+ _replicaIdToInstancesMap = new HashMap<>();
+ }
+
+ public SelectionResult(Map<Integer, List<String>> replicaIdToInstancesMap) {
+ _replicaIdToInstancesMap = replicaIdToInstancesMap;
+ }
+
+ public int getNumReplicas() {
+ return _replicaIdToInstancesMap.size();
+ }
+
+ public List<String> getInstances(int replicaId) {
+ return _replicaIdToInstancesMap.get(replicaId);
+ }
+
+ public void setInstances(int replicaId, List<String> instances) {
+ _replicaIdToInstancesMap.set(replicaId, instances);
+ }
+}
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/result/SelectionResultUtils.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/result/SelectionResultUtils.java
new file mode 100644
index 0000000..dcddd92
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/result/SelectionResultUtils.java
@@ -0,0 +1,20 @@
+package com.linkedin.pinot.controller.helix.core.instance.selection.result;
+
+import java.util.Map;
+import org.apache.helix.ZNRecord;
+import org.apache.helix.store.zk.ZkHelixPropertyStore;
+
+
+public class SelectionResultUtils {
+ private SelectionResultUtils() {
+ }
+
+ public static Map<String, SelectionResult> readFromZK(ZkHelixPropertyStore<ZNRecord> propertyStore, String resource) {
+ throw new UnsupportedOperationException();
+ }
+
+ public static boolean writeToZK(ZkHelixPropertyStore<ZNRecord> propertyStore, String resource,
+ Map<String, SelectionResult> selectionResults) {
+ throw new UnsupportedOperationException();
+ }
+}
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/strategy/SelectionStrategy.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/strategy/SelectionStrategy.java
new file mode 100644
index 0000000..c462627
--- /dev/null
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/instance/selection/strategy/SelectionStrategy.java
@@ -0,0 +1,12 @@
+package com.linkedin.pinot.controller.helix.core.instance.selection.strategy;
+
+import com.linkedin.pinot.controller.helix.core.instance.InstanceRole;
+import com.linkedin.pinot.controller.helix.core.instance.selection.result.SelectionResult;
+import java.util.Map;
+import java.util.Set;
+
+
+public interface SelectionStrategy {
+
+ SelectionResult selectInstances(Map<Integer, Set<String>> groupIdToInstancePoolMap, )
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org