You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by jx...@apache.org on 2024/03/19 22:50:01 UTC

(helix) branch master updated: [apache/helix] -- Fix PreferenceList Ordering Changes during Maintenance Mode (#2778)

This is an automated email from the ASF dual-hosted git repository.

jxue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git


The following commit(s) were added to refs/heads/master by this push:
     new ce33735dd [apache/helix] -- Fix PreferenceList Ordering Changes during Maintenance Mode (#2778)
ce33735dd is described below

commit ce33735dda1dd8614cb8d84094119de1a4b30484
Author: Himanshu Kandwal <hi...@gmail.com>
AuthorDate: Tue Mar 19 15:49:55 2024 -0700

    [apache/helix] -- Fix PreferenceList Ordering Changes during Maintenance Mode (#2778)
    
    Fixed PreferenceList Ordering Changes (indeterminism) during Maintenance Mode.
---
 .../rebalancer/MaintenanceRebalancer.java          | 31 ++++++-
 .../rebalancer/TestMaintenanceRebalancer.java      | 60 +++++++++++++
 ...MaintenanceRebalancer.ComputeNewIdealState.json | 98 ++++++++++++++++++++++
 3 files changed, 187 insertions(+), 2 deletions(-)

diff --git a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/MaintenanceRebalancer.java b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/MaintenanceRebalancer.java
index 9f04de535..9b5b4f929 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/MaintenanceRebalancer.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/MaintenanceRebalancer.java
@@ -28,6 +28,7 @@ import org.apache.helix.controller.dataproviders.ResourceControllerDataProvider;
 import org.apache.helix.controller.stages.CurrentStateOutput;
 import org.apache.helix.model.IdealState;
 import org.apache.helix.model.Partition;
+import org.apache.helix.model.StateModelDefinition;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -55,12 +56,38 @@ public class MaintenanceRebalancer extends SemiAutoRebalancer<ResourceController
 
     // One principal is to prohibit DROP -> OFFLINE and OFFLINE -> DROP state transitions.
     // Derived preference list from current state with state priority
+    StateModelDefinition stateModelDef = clusterData.getStateModelDef(currentIdealState.getStateModelDefRef());
+
     for (Partition partition : currentStateMap.keySet()) {
       Map<String, String> stateMap = currentStateMap.get(partition);
       List<String> preferenceList = new ArrayList<>(stateMap.keySet());
+
+      /**
+       * This sorting preserves the ordering of current state hosts in the order of current IS pref list
+       * Example:
+       * ideal state pref-list: [A, B, C]
+       * current-state: {
+       *     A: FOLLOWER,
+       *     B: LEADER,
+       *     C: FOLLOWER
+       * }
+       * Lets say newPrefList = new ArrayList<>(current-state.keySet()) => [C, B, A]
+       *
+       * Sort 1: Sort based on preference-list order:
+       * --------------------------------------------------------
+       * newPrefList = [C, B, A] => [A, B, C]
+       */
       Collections.sort(preferenceList, new PreferenceListNodeComparator(stateMap,
-          clusterData.getStateModelDef(currentIdealState.getStateModelDefRef()),
-          Collections.<String>emptyList()));
+          stateModelDef, currentIdealState.getPreferenceList(partition.getPartitionName())));
+
+      /**
+       * Sort 2: Sort based on state-priority order:
+       * --------------------------------------------------------
+       * newPrefList = [A, B, C] => [B, A, C]
+       * Here, A will be 2nd and C will be third always as both have same priority so original (pref-list) order will be maintained.
+       */
+      preferenceList.sort(new StatePriorityComparator(stateMap, stateModelDef));
+
       currentIdealState.setPreferenceList(partition.getPartitionName(), preferenceList);
     }
     LOG.info(String
diff --git a/helix-core/src/test/java/org/apache/helix/controller/rebalancer/TestMaintenanceRebalancer.java b/helix-core/src/test/java/org/apache/helix/controller/rebalancer/TestMaintenanceRebalancer.java
new file mode 100644
index 000000000..50cdd418f
--- /dev/null
+++ b/helix-core/src/test/java/org/apache/helix/controller/rebalancer/TestMaintenanceRebalancer.java
@@ -0,0 +1,60 @@
+package org.apache.helix.controller.rebalancer;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.helix.controller.dataproviders.ResourceControllerDataProvider;
+import org.apache.helix.controller.stages.CurrentStateOutput;
+import org.apache.helix.model.IdealState;
+import org.apache.helix.model.MasterSlaveSMD;
+import org.apache.helix.model.Partition;
+import org.apache.helix.util.TestInputLoader;
+import org.testng.Assert;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TestMaintenanceRebalancer {
+
+  private static final String RESOURCE_NAME = "testResource";
+  private static final String PARTITION_NAME = "testResourcePartition";
+
+  @Test(dataProvider = "TestComputeIdealStateInput")
+  public void testComputeIdealState(String comment, String stateModelName, List<String> liveInstances,
+      List<String> preferenceList, Map<String, String> currentStateMap, List<String> expectedPrefList) {
+    System.out.println("Test case comment: " + comment);
+    MaintenanceRebalancer rebalancer = new MaintenanceRebalancer();
+
+    Partition partition = new Partition(PARTITION_NAME);
+    CurrentStateOutput currentStateOutput = new CurrentStateOutput();
+    for (String instance : currentStateMap.keySet()) {
+      currentStateOutput.setCurrentState(RESOURCE_NAME, partition, instance, currentStateMap.get(instance));
+    }
+
+    IdealState currentIdealState = new IdealState(RESOURCE_NAME);
+    currentIdealState.setRebalanceMode(IdealState.RebalanceMode.FULL_AUTO);
+    currentIdealState.setRebalancerClassName("org.apache.helix.controller.rebalancer.waged.WagedRebalancer");
+    currentIdealState.setStateModelDefRef(stateModelName);
+    currentIdealState.setPreferenceList(PARTITION_NAME, preferenceList);
+
+    ResourceControllerDataProvider dataCache = mock(ResourceControllerDataProvider.class);
+    when(dataCache.getStateModelDef("MasterSlave")).thenReturn(MasterSlaveSMD.build());
+
+    IdealState updatedIdealState = rebalancer
+        .computeNewIdealState(RESOURCE_NAME, currentIdealState, currentStateOutput, dataCache);
+
+    List<String> partitionPrefList = updatedIdealState.getPreferenceList(PARTITION_NAME);
+    Assert.assertTrue(partitionPrefList.equals(expectedPrefList));
+  }
+
+  @DataProvider(name = "TestComputeIdealStateInput")
+  public Object[][] loadTestComputeIdealStateInput() {
+    final String[] params = {
+        "comment", "stateModel", "liveInstances", "preferenceList", "currentStateMap", "expectedPreferenceList"
+    };
+    return TestInputLoader.loadTestInputs("MaintenanceRebalancer.ComputeNewIdealState.json", params);
+  }
+
+}
diff --git a/helix-core/src/test/resources/MaintenanceRebalancer.ComputeNewIdealState.json b/helix-core/src/test/resources/MaintenanceRebalancer.ComputeNewIdealState.json
new file mode 100644
index 000000000..9212b90bd
--- /dev/null
+++ b/helix-core/src/test/resources/MaintenanceRebalancer.ComputeNewIdealState.json
@@ -0,0 +1,98 @@
+[
+  {
+    "comment": "[Case 1] Same state - No (A, B, C) -> (A, C, B) for different hash order",
+    "stateModel": "MasterSlave",
+    "liveInstances": [],
+    "preferenceList": [
+      "lKmBBEPisM", "8PNpMU7EgT", "HOs3rOFCad"
+    ],
+    "currentStateMap": {
+      "lKmBBEPisM" : "MASTER",
+      "8PNpMU7EgT" : "SLAVE",
+      "HOs3rOFCad" : "SLAVE"
+    },
+    "expectedPreferenceList": [
+      "lKmBBEPisM", "8PNpMU7EgT", "HOs3rOFCad"
+    ]
+  },
+  {
+    "comment": "[Case 2] Same state - No (A, B, C) -> (A, C, B) for different hash order",
+    "stateModel": "MasterSlave",
+    "liveInstances": [],
+    "preferenceList": [
+      "Q3ZZuCeBpu", "QY9U91XBYo", "GMYBW20gmC"
+    ],
+    "currentStateMap": {
+      "Q3ZZuCeBpu" : "MASTER",
+      "QY9U91XBYo" : "SLAVE",
+      "GMYBW20gmC" : "SLAVE"
+    },
+    "expectedPreferenceList": [
+      "Q3ZZuCeBpu", "QY9U91XBYo", "GMYBW20gmC"
+    ]
+  },
+  {
+    "comment": "[Case 3] second becomes leader, (A, B, C) -> (B, A, C) for different hash order",
+    "stateModel": "MasterSlave",
+    "liveInstances": [],
+    "preferenceList": [
+      "lKmBBEPisM", "8PNpMU7EgT", "HOs3rOFCad"
+    ],
+    "currentStateMap": {
+      "lKmBBEPisM" : "SLAVE",
+      "8PNpMU7EgT" : "MASTER",
+      "HOs3rOFCad" : "SLAVE"
+    },
+    "expectedPreferenceList": [
+      "8PNpMU7EgT", "lKmBBEPisM", "HOs3rOFCad"
+    ]
+  },
+  {
+    "comment": "[Case 4] second becomes leader, (A, B, C) -> (B, A, C) for different hash order",
+    "stateModel": "MasterSlave",
+    "liveInstances": [],
+    "preferenceList": [
+      "Q3ZZuCeBpu", "QY9U91XBYo", "GMYBW20gmC"
+    ],
+    "currentStateMap": {
+      "Q3ZZuCeBpu" : "SLAVE",
+      "QY9U91XBYo" : "MASTER",
+      "GMYBW20gmC" : "SLAVE"
+    },
+    "expectedPreferenceList": [
+      "QY9U91XBYo", "Q3ZZuCeBpu", "GMYBW20gmC"
+    ]
+  },
+  {
+    "comment": "[Case 5] leader becomes offline, (A, B, C) -> (B, C, A) for different hash order",
+    "stateModel": "MasterSlave",
+    "liveInstances": [],
+    "preferenceList": [
+      "lKmBBEPisM", "8PNpMU7EgT", "HOs3rOFCad"
+    ],
+    "currentStateMap": {
+      "lKmBBEPisM" : "OFFLINE",
+      "8PNpMU7EgT" : "MASTER",
+      "HOs3rOFCad" : "SLAVE"
+    },
+    "expectedPreferenceList": [
+      "8PNpMU7EgT", "HOs3rOFCad", "lKmBBEPisM"
+    ]
+  },
+  {
+    "comment": "[Case 6] leader becomes offline, (A, B, C) -> (B, C, A) for different hash order",
+    "stateModel": "MasterSlave",
+    "liveInstances": [],
+    "preferenceList": [
+      "Q3ZZuCeBpu", "QY9U91XBYo", "GMYBW20gmC"
+    ],
+    "currentStateMap": {
+      "Q3ZZuCeBpu" : "OFFLINE",
+      "QY9U91XBYo" : "MASTER",
+      "GMYBW20gmC" : "SLAVE"
+    },
+    "expectedPreferenceList": [
+      "QY9U91XBYo", "GMYBW20gmC", "Q3ZZuCeBpu"
+    ]
+  }
+]