You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by lx...@apache.org on 2018/03/23 18:58:10 UTC
[1/6] helix git commit: Persist preferenceLists into
TargetExternalView.
Repository: helix
Updated Branches:
refs/heads/master a5de29c2a -> 623330e3a
Persist preferenceLists into TargetExternalView.
Project: http://git-wip-us.apache.org/repos/asf/helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/helix/commit/16f807af
Tree: http://git-wip-us.apache.org/repos/asf/helix/tree/16f807af
Diff: http://git-wip-us.apache.org/repos/asf/helix/diff/16f807af
Branch: refs/heads/master
Commit: 16f807afb3f1ca6cd48bc408d256c92ab2616321
Parents: 986e79c
Author: Lei Xia <lx...@linkedin.com>
Authored: Thu Mar 8 10:19:23 2018 -0800
Committer: Lei Xia <lx...@linkedin.com>
Committed: Fri Mar 23 11:57:06 2018 -0700
----------------------------------------------------------------------
.../stages/TargetExteralViewCalcStage.java | 36 +++++++++++++++-----
.../controller/TestTargetExternalView.java | 22 ++++++++----
2 files changed, 44 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/helix/blob/16f807af/helix-core/src/main/java/org/apache/helix/controller/stages/TargetExteralViewCalcStage.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/TargetExteralViewCalcStage.java b/helix-core/src/main/java/org/apache/helix/controller/stages/TargetExteralViewCalcStage.java
index eb08e4e..b7e4ebd 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/stages/TargetExteralViewCalcStage.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/stages/TargetExteralViewCalcStage.java
@@ -62,33 +62,53 @@ public class TargetExteralViewCalcStage extends AbstractBaseStage {
AccessOption.PERSISTENT);
}
- IntermediateStateOutput intermediateAssignment =
+ BestPossibleStateOutput bestPossibleAssignments =
+ event.getAttribute(AttributeName.BEST_POSSIBLE_STATE.name());
+
+ IntermediateStateOutput intermediateAssignments =
event.getAttribute(AttributeName.INTERMEDIATE_STATE.name());
Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES.name());
List<PropertyKey> keys = new ArrayList<>();
- List<HelixProperty> targetExternalViews = new ArrayList<>();
+ List<ExternalView> targetExternalViews = new ArrayList<>();
- for (String resourceName : intermediateAssignment.resourceSet()) {
+ for (String resourceName : bestPossibleAssignments.resourceSet()) {
if (cache.getIdealState(resourceName) == null || cache.getIdealState(resourceName).isExternalViewDisabled()) {
continue;
}
Resource resource = resourceMap.get(resourceName);
if (resource != null) {
PartitionStateMap partitionStateMap =
- intermediateAssignment.getPartitionStateMap(resourceName);
- Map<String, Map<String, String>> assignmentToPersist = convertToMapFields(partitionStateMap.getStateMap());
+ intermediateAssignments.getPartitionStateMap(resourceName);
+ Map<String, Map<String, String>> intermediateAssignment = convertToMapFields(
+ partitionStateMap.getStateMap());
+
+ Map<String, List<String>> preferenceLists =
+ bestPossibleAssignments.getPreferenceLists(resourceName);
+ boolean needPersist = false;
ExternalView targetExternalView = cache.getTargetExternalView(resourceName);
if (targetExternalView == null) {
targetExternalView = new ExternalView(resourceName);
targetExternalView.getRecord()
.getSimpleFields()
.putAll(cache.getIdealState(resourceName).getRecord().getSimpleFields());
+ needPersist = true;
+ }
+
+ if (preferenceLists != null && !targetExternalView.getRecord().getListFields()
+ .equals(preferenceLists)) {
+ targetExternalView.getRecord().setListFields(preferenceLists);
+ needPersist = true;
}
- if (assignmentToPersist != null && !targetExternalView.getRecord().getMapFields()
- .equals(assignmentToPersist)) {
- targetExternalView.getRecord().setMapFields(assignmentToPersist);
+
+ if (intermediateAssignment != null && !targetExternalView.getRecord().getMapFields()
+ .equals(intermediateAssignment)) {
+ targetExternalView.getRecord().setMapFields(intermediateAssignment);
+ needPersist = true;
+ }
+
+ if (needPersist) {
keys.add(accessor.keyBuilder().targetExternalView(resourceName));
targetExternalViews.add(targetExternalView);
cache.updateTargetExternalView(resourceName, targetExternalView);
http://git-wip-us.apache.org/repos/asf/helix/blob/16f807af/helix-core/src/test/java/org/apache/helix/integration/controller/TestTargetExternalView.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/integration/controller/TestTargetExternalView.java b/helix-core/src/test/java/org/apache/helix/integration/controller/TestTargetExternalView.java
index 81321df..9103db1 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/controller/TestTargetExternalView.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/controller/TestTargetExternalView.java
@@ -27,11 +27,13 @@ import org.apache.helix.integration.task.TaskTestBase;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.IdealState;
+import org.apache.helix.tools.ClusterVerifiers.BestPossibleExternalViewVerifier;
+import org.apache.helix.tools.ClusterVerifiers.HelixClusterVerifier;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
-public class TestTargetExternalView extends TaskTestBase{
+public class TestTargetExternalView extends TaskTestBase {
private ConfigAccessor _configAccessor;
private HelixDataAccessor _accessor;
@@ -57,7 +59,9 @@ public class TestTargetExternalView extends TaskTestBase{
clusterConfig.setPersistIntermediateAssignment(true);
_configAccessor.setClusterConfig(CLUSTER_NAME, clusterConfig);
_gSetupTool.getClusterManagementTool().rebalance(CLUSTER_NAME, _testDbs.get(0), 3);
- Thread.sleep(2000L);
+
+ HelixClusterVerifier verifier = new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR).build();
+ Assert.assertTrue(verifier.verify());
Assert
.assertEquals(_accessor.getChildNames(_accessor.keyBuilder().targetExternalViews()).size(),
@@ -68,19 +72,25 @@ public class TestTargetExternalView extends TaskTestBase{
List<IdealState> idealStates = _accessor.getChildValues(_accessor.keyBuilder().idealStates());
for (int i = 0; i < idealStates.size(); i++) {
- Assert.assertTrue(targetExternalViews.get(i).getRecord().getMapFields()
- .equals(idealStates.get(i).getRecord().getMapFields()));
+ Assert.assertEquals(targetExternalViews.get(i).getRecord().getMapFields(),
+ idealStates.get(i).getRecord().getMapFields());
+ Assert.assertEquals(targetExternalViews.get(i).getRecord().getListFields(),
+ idealStates.get(i).getRecord().getListFields());
}
// Disable one instance to see whether the target external views changes.
_gSetupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME, _participants[0].getInstanceName(), false);
+ Assert.assertTrue(verifier.verify());
+
targetExternalViews = _accessor.getChildValues(_accessor.keyBuilder().externalViews());
idealStates = _accessor.getChildValues(_accessor.keyBuilder().idealStates());
for (int i = 0; i < idealStates.size(); i++) {
- Assert.assertTrue(
- targetExternalViews.get(i).getRecord().getMapFields().equals(idealStates.get(i).getRecord().getMapFields()));
+ Assert.assertEquals(targetExternalViews.get(i).getRecord().getMapFields(),
+ idealStates.get(i).getRecord().getMapFields());
+ Assert.assertEquals(targetExternalViews.get(i).getRecord().getListFields(),
+ idealStates.get(i).getRecord().getListFields());
}
}
}
[2/6] helix git commit: Including version number in Participant and
Controller history, and add additional logs.
Posted by lx...@apache.org.
Including version number in Participant and Controller history, and add additional logs.
Project: http://git-wip-us.apache.org/repos/asf/helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/helix/commit/fb0b6747
Tree: http://git-wip-us.apache.org/repos/asf/helix/tree/fb0b6747
Diff: http://git-wip-us.apache.org/repos/asf/helix/diff/fb0b6747
Branch: refs/heads/master
Commit: fb0b67471112e44b9fd416dca51a185359369308
Parents: a5de29c
Author: Lei Xia <lx...@linkedin.com>
Authored: Fri Feb 16 23:05:49 2018 -0800
Committer: Lei Xia <lx...@linkedin.com>
Committed: Fri Mar 23 11:57:06 2018 -0700
----------------------------------------------------------------------
.../apache/helix/common/caches/CurrentStateCache.java | 14 ++++++--------
.../helix/controller/stages/ClusterDataCache.java | 3 +--
.../org/apache/helix/manager/zk/CallbackHandler.java | 11 ++++-------
.../helix/manager/zk/DistributedLeaderElection.java | 2 +-
.../apache/helix/manager/zk/ParticipantManager.java | 4 +++-
.../org/apache/helix/manager/zk/ZKHelixManager.java | 5 ++++-
.../apache/helix/manager/zk/ZkBaseDataAccessor.java | 11 ++++++-----
.../java/org/apache/helix/model/LeaderHistory.java | 4 +++-
.../org/apache/helix/model/ParticipantHistory.java | 8 +++++---
9 files changed, 33 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/helix/blob/fb0b6747/helix-core/src/main/java/org/apache/helix/common/caches/CurrentStateCache.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/common/caches/CurrentStateCache.java b/helix-core/src/main/java/org/apache/helix/common/caches/CurrentStateCache.java
index d921512..5e69c1d 100644
--- a/helix-core/src/main/java/org/apache/helix/common/caches/CurrentStateCache.java
+++ b/helix-core/src/main/java/org/apache/helix/common/caches/CurrentStateCache.java
@@ -138,7 +138,7 @@ public class CurrentStateCache {
reloadKeys.add(key);
}
} else {
- LOG.debug("stat is null for key: " + key);
+ LOG.warn("stat is null for key: " + key);
reloadKeys.add(key);
}
}
@@ -150,18 +150,16 @@ public class CurrentStateCache {
if (currentState != null) {
currentStatesMap.put(key, currentState);
} else {
- LOG.debug("CurrentState null for key: " + key);
+ LOG.warn("CurrentState null for key: " + key);
}
}
_currentStateCache = Collections.unmodifiableMap(currentStatesMap);
- if (LOG.isDebugEnabled()) {
- LOG.debug("# of CurrentState paths read from ZooKeeper " + reloadKeys.size());
- LOG.debug("# of CurrentState paths skipped reading from ZK: " + (currentStateKeys.size()
- - reloadKeys.size()));
- }
- LOG.info("Takes " + (System.currentTimeMillis() - start) + " ms to reload new current states!");
+ LOG.info("# of CurrentStates reload: " + reloadKeys.size() + ", skipped:" + (currentStateKeys.size()
+ - reloadKeys.size()));
+ LOG.info("Takes " + (System.currentTimeMillis() - start) + " ms to reload new current states for cluster: "
+ + _clusterName);
}
/**
http://git-wip-us.apache.org/repos/asf/helix/blob/fb0b6747/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterDataCache.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterDataCache.java b/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterDataCache.java
index 4fa0c8c..e946a9d 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterDataCache.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterDataCache.java
@@ -19,7 +19,6 @@ package org.apache.helix.controller.stages;
* under the License.
*/
-import com.google.common.collect.Maps;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -192,7 +191,7 @@ public class ClusterDataCache {
_instanceMessagesCache
.refresh(accessor, _liveInstanceMap);
_currentStateCache.refresh(accessor, _liveInstanceMap);
-
+
// current state must be refreshed before refreshing relay messages
// because we need to use current state to validate all relay messages.
_instanceMessagesCache
http://git-wip-us.apache.org/repos/asf/helix/blob/fb0b6747/helix-core/src/main/java/org/apache/helix/manager/zk/CallbackHandler.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/CallbackHandler.java b/helix-core/src/main/java/org/apache/helix/manager/zk/CallbackHandler.java
index 42adade..c5a035f 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/CallbackHandler.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/CallbackHandler.java
@@ -530,7 +530,7 @@ public class CallbackHandler implements IZkChildListener, IZkDataListener {
@Override
public void handleDataChange(String dataPath, Object data) {
- logger.debug("Data change callback: paths changed: " + dataPath);
+ logger.info("Data change callback: paths changed: " + dataPath);
try {
updateNotificationTime(System.nanoTime());
@@ -548,10 +548,7 @@ public class CallbackHandler implements IZkChildListener, IZkDataListener {
}
@Override public void handleDataDeleted(String dataPath) {
- logger.debug("Data change callback: path deleted: " + dataPath);
- if (_changeType == IDEAL_STATE || _changeType == LIVE_INSTANCE) {
- logger.info("Data deleted callback, deleted path: " + dataPath);
- }
+ logger.info("Data change callback: path deleted: " + dataPath);
try {
updateNotificationTime(System.nanoTime());
@@ -576,8 +573,8 @@ public class CallbackHandler implements IZkChildListener, IZkDataListener {
@Override
public void handleChildChange(String parentPath, List<String> currentChilds) {
- logger.debug("Data change callback: child changed, path: " + parentPath + ", current childs: "
- + currentChilds);
+ logger.info("Data change callback: child changed, path: " + parentPath + ", current child count: "
+ + currentChilds.size());
try {
updateNotificationTime(System.nanoTime());
http://git-wip-us.apache.org/repos/asf/helix/blob/fb0b6747/helix-core/src/main/java/org/apache/helix/manager/zk/DistributedLeaderElection.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/DistributedLeaderElection.java b/helix-core/src/main/java/org/apache/helix/manager/zk/DistributedLeaderElection.java
index 6022edd..c676925 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/DistributedLeaderElection.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/DistributedLeaderElection.java
@@ -155,7 +155,7 @@ public class DistributedLeaderElection implements ControllerChangeListener {
if (history == null) {
history = new LeaderHistory(PropertyType.HISTORY.toString());
}
- history.updateHistory(manager.getClusterName(), manager.getInstanceName());
+ history.updateHistory(manager.getClusterName(), manager.getInstanceName(), manager.getVersion());
if(!accessor.setProperty(keyBuilder.controllerLeaderHistory(), history)) {
LOG.error("Failed to persist leader history to ZK!");
}
http://git-wip-us.apache.org/repos/asf/helix/blob/fb0b6747/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
index 880607a..db236b8 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
@@ -181,6 +181,7 @@ public class ParticipantManager {
retry = false;
try {
_zkclient.createEphemeral(liveInstancePath, liveInstance.getRecord());
+ LOG.info("LiveInstance created, path: " + liveInstancePath + ", sessionId: " + liveInstance.getSessionId());
} catch (ZkNodeExistsException e) {
LOG.warn("found another instance with same instanceName: " + _instanceName + " in cluster "
+ _clusterName);
@@ -238,6 +239,7 @@ public class ParticipantManager {
if (retry) {
try {
_zkclient.createEphemeral(liveInstancePath, liveInstance.getRecord());
+ LOG.info("LiveInstance created, path: " + liveInstancePath + ", sessionId: " + liveInstance.getSessionId());
} catch (Exception e) {
String errorMessage =
"instance: " + _instanceName + " already has a live-instance in cluster "
@@ -248,7 +250,7 @@ public class ParticipantManager {
}
ParticipantHistory history = getHistory();
- history.reportOnline(_sessionId);
+ history.reportOnline(_sessionId, _manager.getVersion());
persistHistory(history);
}
http://git-wip-us.apache.org/repos/asf/helix/blob/fb0b6747/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
index dee9adf..d156c88 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
@@ -938,7 +938,8 @@ public class ZKHelixManager implements HelixManager, IZkStateListener {
switch (state) {
case SyncConnected:
ZkConnection zkConnection = (ZkConnection) _zkclient.getConnection();
- LOG.info("KeeperState: " + state + ", zookeeper:" + zkConnection.getZookeeper());
+ LOG.info("KeeperState: " + state + ", instance: "
+ + _instanceName + ", type: " + _instanceType + ", zookeeper:" + zkConnection.getZookeeper());
break;
case Disconnected:
LOG.info("KeeperState:" + state + ", disconnectedSessionId: " + _sessionId + ", instance: "
@@ -978,6 +979,8 @@ public class ZKHelixManager implements HelixManager, IZkStateListener {
@Override
public void handleNewSession() throws Exception {
+ LOG.info(
+ "Handle new session, sessionId: " + _sessionId + ", instance: " + _instanceName + ", type: " + _instanceType);
waitUntilConnected();
/**
http://git-wip-us.apache.org/repos/asf/helix/blob/fb0b6747/helix-core/src/main/java/org/apache/helix/manager/zk/ZkBaseDataAccessor.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ZkBaseDataAccessor.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ZkBaseDataAccessor.java
index f8678e5..9525103 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZkBaseDataAccessor.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZkBaseDataAccessor.java
@@ -22,8 +22,10 @@ package org.apache.helix.manager.zk;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import org.I0Itec.zkclient.DataUpdater;
import org.I0Itec.zkclient.IZkChildListener;
import org.I0Itec.zkclient.IZkDataListener;
@@ -373,7 +375,7 @@ public class ZkBaseDataAccessor<T> implements BaseDataAccessor<T> {
// construct return results
List<T> records = new ArrayList<T>(Collections.<T> nCopies(paths.size(), null));
- StringBuilder nodeFailToRead = new StringBuilder();
+ Map<String, Integer> pathFailToRead = new HashMap<>();
for (int i = 0; i < paths.size(); i++) {
if (!needRead[i])
continue;
@@ -389,12 +391,11 @@ public class ZkBaseDataAccessor<T> implements BaseDataAccessor<T> {
} else if (Code.get(cb.getRc()) != Code.NONODE && throwException) {
throw new HelixException(String.format("Failed to read node %s", paths.get(i)));
} else {
- nodeFailToRead.append(paths + ",");
+ pathFailToRead.put(paths.get(i), cb.getRc());
}
}
- if (nodeFailToRead.length() > 0) {
- LOG.warn(String.format("Fail to read nodes for paths : %s",
- nodeFailToRead.toString().substring(nodeFailToRead.length() - 1)));
+ if (pathFailToRead.size() > 0) {
+ LOG.warn("Fail to read record for paths: " + pathFailToRead);
}
return records;
} finally {
http://git-wip-us.apache.org/repos/asf/helix/blob/fb0b6747/helix-core/src/main/java/org/apache/helix/model/LeaderHistory.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/model/LeaderHistory.java b/helix-core/src/main/java/org/apache/helix/model/LeaderHistory.java
index 5e7bdcd..31d125f 100644
--- a/helix-core/src/main/java/org/apache/helix/model/LeaderHistory.java
+++ b/helix-core/src/main/java/org/apache/helix/model/LeaderHistory.java
@@ -41,6 +41,7 @@ public class LeaderHistory extends HelixProperty {
HISTORY,
TIME,
DATE,
+ VERSION,
CONTROLLER
}
@@ -57,7 +58,7 @@ public class LeaderHistory extends HelixProperty {
* @param clusterName the cluster the instance leads
* @param instanceName the name of the leader instance
*/
- public void updateHistory(String clusterName, String instanceName) {
+ public void updateHistory(String clusterName, String instanceName, String version) {
/* keep this for back-compatible */
// TODO: remove this in future when we confirmed no one consumes it
List<String> list = _record.getListField(clusterName);
@@ -92,6 +93,7 @@ public class LeaderHistory extends HelixProperty {
historyEntry.put(ConfigProperty.CONTROLLER.name(), instanceName);
historyEntry.put(ConfigProperty.TIME.name(), String.valueOf(currentTime));
historyEntry.put(ConfigProperty.DATE.name(), dateTime);
+ historyEntry.put(ConfigProperty.VERSION.name(), version);
historyList.add(historyEntry.toString());
}
http://git-wip-us.apache.org/repos/asf/helix/blob/fb0b6747/helix-core/src/main/java/org/apache/helix/model/ParticipantHistory.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/model/ParticipantHistory.java b/helix-core/src/main/java/org/apache/helix/model/ParticipantHistory.java
index 2955ba5..e56c3e3 100644
--- a/helix-core/src/main/java/org/apache/helix/model/ParticipantHistory.java
+++ b/helix-core/src/main/java/org/apache/helix/model/ParticipantHistory.java
@@ -46,6 +46,7 @@ public class ParticipantHistory extends HelixProperty {
SESSION,
HISTORY,
OFFLINE,
+ VERSION,
LAST_OFFLINE_TIME
}
@@ -74,8 +75,8 @@ public class ParticipantHistory extends HelixProperty {
*
* @return
*/
- public void reportOnline(String sessionId) {
- updateSessionHistory(sessionId);
+ public void reportOnline(String sessionId, String version) {
+ updateSessionHistory(sessionId, version);
_record.setSimpleField(ConfigProperty.LAST_OFFLINE_TIME.name(), String.valueOf(ONLINE));
}
@@ -102,7 +103,7 @@ public class ParticipantHistory extends HelixProperty {
/**
* Add record to session online history list
*/
- private void updateSessionHistory(String sessionId) {
+ private void updateSessionHistory(String sessionId, String version) {
List<String> list = _record.getListField(ConfigProperty.HISTORY.name());
if (list == null) {
list = new ArrayList<>();
@@ -124,6 +125,7 @@ public class ParticipantHistory extends HelixProperty {
df.setTimeZone(TimeZone.getTimeZone("UTC"));
String dateTime = df.format(new Date(timeMillis));
sessionEntry.put(ConfigProperty.DATE.name(), dateTime);
+ sessionEntry.put(ConfigProperty.VERSION.name(), version);
list.add(sessionEntry.toString());
}
[4/6] helix git commit: Fix a bug in AutoRebalancer that it fails to
compute ideal mapping if "ANY_LIVEINSTANCE" is specified as the replica.
Posted by lx...@apache.org.
Fix a bug in AutoRebalancer that it fails to compute ideal mapping if "ANY_LIVEINSTANCE" is specified as the replica.
Project: http://git-wip-us.apache.org/repos/asf/helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/helix/commit/42557199
Tree: http://git-wip-us.apache.org/repos/asf/helix/tree/42557199
Diff: http://git-wip-us.apache.org/repos/asf/helix/diff/42557199
Branch: refs/heads/master
Commit: 42557199a99feab1a0157569d49925f60a0955b1
Parents: 16f807a
Author: Lei Xia <lx...@linkedin.com>
Authored: Wed Mar 14 17:29:53 2018 -0700
Committer: Lei Xia <lx...@linkedin.com>
Committed: Fri Mar 23 11:57:07 2018 -0700
----------------------------------------------------------------------
.../org/apache/helix/controller/rebalancer/AutoRebalancer.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/helix/blob/42557199/helix-core/src/main/java/org/apache/helix/controller/rebalancer/AutoRebalancer.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/AutoRebalancer.java b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/AutoRebalancer.java
index 1c385dd..1af159d 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/AutoRebalancer.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/AutoRebalancer.java
@@ -72,10 +72,10 @@ public class AutoRebalancer extends AbstractRebalancer {
throw new HelixException("State Model Definition null for resource: " + resourceName);
}
Map<String, LiveInstance> liveInstance = clusterData.getLiveInstances();
- String replicas = currentIdealState.getReplicas();
+ int replicas = currentIdealState.getReplicaCount(liveInstance.size());
LinkedHashMap<String, Integer> stateCountMap = stateModelDef
- .getStateCountMap(liveInstance.size(), Integer.parseInt(replicas));
+ .getStateCountMap(liveInstance.size(), replicas);
List<String> liveNodes = new ArrayList<String>(liveInstance.keySet());
List<String> allNodes = new ArrayList<String>(clusterData.getInstanceConfigMap().keySet());
allNodes.removeAll(clusterData.getDisabledInstances());
[6/6] helix git commit: Fix a bug when controller handles relay
message timeout,
and print out log when controller ignores some relay messages.
Posted by lx...@apache.org.
Fix a bug when controller handles relay message timeout, and print out log when controller ignores some relay messages.
Project: http://git-wip-us.apache.org/repos/asf/helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/helix/commit/623330e3
Tree: http://git-wip-us.apache.org/repos/asf/helix/tree/623330e3
Diff: http://git-wip-us.apache.org/repos/asf/helix/diff/623330e3
Branch: refs/heads/master
Commit: 623330e3ab2e834daae6de1aae91d7ba17e951e0
Parents: d65ef66
Author: Lei Xia <lx...@linkedin.com>
Authored: Mon Mar 12 14:05:52 2018 -0700
Committer: Lei Xia <lx...@linkedin.com>
Committed: Fri Mar 23 11:57:07 2018 -0700
----------------------------------------------------------------------
.../common/caches/InstanceMessagesCache.java | 18 ++++++++++++++----
1 file changed, 14 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/helix/blob/623330e3/helix-core/src/main/java/org/apache/helix/common/caches/InstanceMessagesCache.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/common/caches/InstanceMessagesCache.java b/helix-core/src/main/java/org/apache/helix/common/caches/InstanceMessagesCache.java
index 9ac40c3..438e3a6 100644
--- a/helix-core/src/main/java/org/apache/helix/common/caches/InstanceMessagesCache.java
+++ b/helix-core/src/main/java/org/apache/helix/common/caches/InstanceMessagesCache.java
@@ -23,6 +23,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
@@ -148,23 +149,32 @@ public class InstanceMessagesCache {
String instanceSessionId = liveInstanceMap.get(instance).getSessionId();
if (!instanceSessionId.equals(sessionId)) {
+ LOG.info("Instance SessionId does not match, ignore relay messages attached to message "
+ + message.getId());
continue;
}
Map<String, CurrentState> sessionCurrentStateMap = instanceCurrentStateMap.get(sessionId);
if (sessionCurrentStateMap == null) {
+ LOG.info("No sessionCurrentStateMap found, ignore relay messages attached to message "
+ + message.getId());
continue;
}
CurrentState currentState = sessionCurrentStateMap.get(resourceName);
if (currentState == null || !targetState.equals(currentState.getState(partitionName))) {
+ LOG.info("CurrentState " + currentState
+ + " do not match the target state of the message, ignore relay messages attached to message "
+ + message.getId());
continue;
}
long transitionCompleteTime = currentState.getEndTime(partitionName);
- for (Message msg : message.getRelayMessages().values()) {
- msg.setRelayTime(transitionCompleteTime);
- if (!message.isExpired()) {
- relayMessages.add(msg);
+ for (Message relayMsg : message.getRelayMessages().values()) {
+ relayMsg.setRelayTime(transitionCompleteTime);
+ if (!relayMsg.isExpired()) {
+ relayMessages.add(relayMsg);
+ } else {
+ LOG.info("Relay message " + relayMsg.getId() + " already expired, ignore it!");
}
}
}
[5/6] helix git commit: Include resource name in the log.
Posted by lx...@apache.org.
Include resource name in the log.
Project: http://git-wip-us.apache.org/repos/asf/helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/helix/commit/d65ef66c
Tree: http://git-wip-us.apache.org/repos/asf/helix/tree/d65ef66c
Diff: http://git-wip-us.apache.org/repos/asf/helix/diff/d65ef66c
Branch: refs/heads/master
Commit: d65ef66cd5698d8d9a2acb4aef8d2372ecf5e5b8
Parents: 4255719
Author: Lei Xia <lx...@linkedin.com>
Authored: Fri Dec 1 17:58:06 2017 -0800
Committer: Lei Xia <lx...@linkedin.com>
Committed: Fri Mar 23 11:57:07 2018 -0700
----------------------------------------------------------------------
.../helix/controller/stages/IntermediateStateCalcStage.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/helix/blob/d65ef66c/helix-core/src/main/java/org/apache/helix/controller/stages/IntermediateStateCalcStage.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/IntermediateStateCalcStage.java b/helix-core/src/main/java/org/apache/helix/controller/stages/IntermediateStateCalcStage.java
index 785ed7b..e4324e4 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/stages/IntermediateStateCalcStage.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/stages/IntermediateStateCalcStage.java
@@ -381,8 +381,8 @@ public class IntermediateStateCalcStage extends AbstractBaseStage {
}
logger.info(String
- .format("needRecovery: %d, recoverybalanceThrottled: %d", partitionsNeedRecovery.size(),
- partitionRecoveryBalanceThrottled.size()));
+ .format("%s: needRecovery: %d, recoverybalanceThrottled: %d", resource.getResourceName(),
+ partitionsNeedRecovery.size(), partitionRecoveryBalanceThrottled.size()));
return partitionRecoveryBalanceThrottled;
}
@@ -418,7 +418,7 @@ public class IntermediateStateCalcStage extends AbstractBaseStage {
}
logger.info(String
- .format("loadbalanceNeeded: %d, loadbalanceThrottled: %d", partitionsNeedLoadbalance.size(),
+ .format("%s: loadbalanceNeeded: %d, loadbalanceThrottled: %d", resource.getResourceName(), partitionsNeedLoadbalance.size(),
partitionsLoadbalanceThrottled.size()));
if (logger.isDebugEnabled()) {
[3/6] helix git commit: Add cluster level metrics to show whether
cluster is in maintenance mode or paused.
Posted by lx...@apache.org.
Add cluster level metrics to show whether cluster is in maintenance mode or paused.
Project: http://git-wip-us.apache.org/repos/asf/helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/helix/commit/986e79c9
Tree: http://git-wip-us.apache.org/repos/asf/helix/tree/986e79c9
Diff: http://git-wip-us.apache.org/repos/asf/helix/diff/986e79c9
Branch: refs/heads/master
Commit: 986e79c9f1b7d40242411c384028d0cd36436cb5
Parents: fb0b674
Author: Lei Xia <lx...@linkedin.com>
Authored: Wed Feb 21 12:25:52 2018 -0800
Committer: Lei Xia <lx...@linkedin.com>
Committed: Fri Mar 23 11:57:06 2018 -0700
----------------------------------------------------------------------
.../controller/GenericHelixController.java | 2 ++
.../monitoring/mbeans/ClusterStatusMonitor.java | 25 +++++++++++++++++++-
.../mbeans/ClusterStatusMonitorMBean.java | 13 ++++++++++
3 files changed, 39 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/helix/blob/986e79c9/helix-core/src/main/java/org/apache/helix/controller/GenericHelixController.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/GenericHelixController.java b/helix-core/src/main/java/org/apache/helix/controller/GenericHelixController.java
index f2709a6..df5c86b 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/GenericHelixController.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/GenericHelixController.java
@@ -606,6 +606,8 @@ public class GenericHelixController implements IdealStateChangeListener,
updateControllerState(changeContext, maintenanceSignal, _inMaintenanceMode);
enableClusterStatusMonitor(true);
_clusterStatusMonitor.setEnabled(!_paused);
+ _clusterStatusMonitor.setPaused(_paused);
+ _clusterStatusMonitor.setMaintenance(_inMaintenanceMode);
} else {
enableClusterStatusMonitor(false);
}
http://git-wip-us.apache.org/repos/asf/helix/blob/986e79c9/helix-core/src/main/java/org/apache/helix/monitoring/mbeans/ClusterStatusMonitor.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/monitoring/mbeans/ClusterStatusMonitor.java b/helix-core/src/main/java/org/apache/helix/monitoring/mbeans/ClusterStatusMonitor.java
index fe682ac..40801b1 100644
--- a/helix-core/src/main/java/org/apache/helix/monitoring/mbeans/ClusterStatusMonitor.java
+++ b/helix-core/src/main/java/org/apache/helix/monitoring/mbeans/ClusterStatusMonitor.java
@@ -56,6 +56,9 @@ public class ClusterStatusMonitor implements ClusterStatusMonitorMBean {
private final MBeanServer _beanServer;
private boolean _enabled = true;
+ private boolean _inMaintenance = false;
+ private boolean _paused = false;
+
private Set<String> _liveInstances = Collections.emptySet();
private Set<String> _instances = Collections.emptySet();
private Set<String> _disabledInstances = Collections.emptySet();
@@ -659,7 +662,8 @@ public class ClusterStatusMonitor implements ClusterStatusMonitorMBean {
private synchronized void unregisterPerInstanceResources(Collection<PerInstanceResourceMonitor.BeanName> beanNames)
throws MalformedObjectNameException {
for (PerInstanceResourceMonitor.BeanName beanName : beanNames) {
- unregister(getObjectName(getPerInstanceResourceBeanName(beanName.instanceName(), beanName.resourceName())));
+ unregister(getObjectName(
+ getPerInstanceResourceBeanName(beanName.instanceName(), beanName.resourceName())));
}
_perInstanceResourceMap.keySet().removeAll(beanNames);
}
@@ -759,6 +763,25 @@ public class ClusterStatusMonitor implements ClusterStatusMonitorMBean {
return _enabled ? 1 : 0;
}
+ @Override
+ public long getMaintenance() {
+ return _inMaintenance ? 1 : 0;
+ }
+
+ public void setMaintenance(boolean inMaintenance) {
+ _inMaintenance = inMaintenance;
+ }
+
+
+ @Override
+ public long getPaused() {
+ return _paused ? 1 : 0;
+ }
+
+ public void setPaused(boolean paused) {
+ _paused = paused;
+ }
+
public void setEnabled(boolean enabled) {
this._enabled = enabled;
}
http://git-wip-us.apache.org/repos/asf/helix/blob/986e79c9/helix-core/src/main/java/org/apache/helix/monitoring/mbeans/ClusterStatusMonitorMBean.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/monitoring/mbeans/ClusterStatusMonitorMBean.java b/helix-core/src/main/java/org/apache/helix/monitoring/mbeans/ClusterStatusMonitorMBean.java
index 483e4d8..4cdd357 100644
--- a/helix-core/src/main/java/org/apache/helix/monitoring/mbeans/ClusterStatusMonitorMBean.java
+++ b/helix-core/src/main/java/org/apache/helix/monitoring/mbeans/ClusterStatusMonitorMBean.java
@@ -48,4 +48,17 @@ public interface ClusterStatusMonitorMBean extends SensorNameProvider {
* @return 1 if cluster is enabled, otherwise 0
*/
public long getEnabled();
+
+ /**
+ *
+ * @return 1 if cluster is in maintenance mode, otherwise 0
+ */
+ public long getMaintenance();
+
+
+ /**
+ *
+ * @return 1 if cluster is paused, otherwise 0
+ */
+ public long getPaused();
}