You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by jx...@apache.org on 2019/02/12 22:28:21 UTC
[helix] branch master updated: HELIX: Fix null tgtSessionId message
bug
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
The following commit(s) were added to refs/heads/master by this push:
new b0c3bfd HELIX: Fix null tgtSessionId message bug
b0c3bfd is described below
commit b0c3bfd693b9e8352ffa55ce3c429a46057d3512
Author: narendly <na...@gmail.com>
AuthorDate: Mon Feb 11 16:22:32 2019 -0800
HELIX: Fix null tgtSessionId message bug
It has been reported that the Controller sometimes sends a message with a null tgtSessionId, which was causing an NPE on the Participant.
Changelist:
1. Controller side fix
2. Participant side fix with a try-catch
3. Modify isValid for messages
---
.../controller/stages/MessageGenerationPhase.java | 9 ++
.../messaging/handling/HelixTaskExecutor.java | 132 +++++++++++----------
.../main/java/org/apache/helix/model/Message.java | 13 +-
3 files changed, 85 insertions(+), 69 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/MessageGenerationPhase.java b/helix-core/src/main/java/org/apache/helix/controller/stages/MessageGenerationPhase.java
index c829082..4c693a6 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/stages/MessageGenerationPhase.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/stages/MessageGenerationPhase.java
@@ -267,6 +267,15 @@ public abstract class MessageGenerationPhase extends AbstractBaseStage {
for (String state : statesPriorityList) {
if (messageMap.containsKey(state)) {
for (Message message : messageMap.get(state)) {
+ // This is for a bug where a message's target session id is null
+ if (!message.isValid()) {
+ LogUtil.logError(logger, _eventId, String.format(
+ "An invalid message was generated! Discarding this message. sessionIdMap: %s, CurrentStateMap: %s, InstanceStateMap: %s, AllInstances: %s, LiveInstances: %s, Message: %s",
+ sessionIdMap, currentStateOutput.getCurrentStateMap(resourceName, partition),
+ instanceStateMap, cache.getAllInstances(), cache.getLiveInstances().keySet(),
+ message));
+ continue; // Do not add this message
+ }
output.addMessage(resourceName, partition, message);
}
}
diff --git a/helix-core/src/main/java/org/apache/helix/messaging/handling/HelixTaskExecutor.java b/helix-core/src/main/java/org/apache/helix/messaging/handling/HelixTaskExecutor.java
index 3ae90d3..a0481e3 100644
--- a/helix-core/src/main/java/org/apache/helix/messaging/handling/HelixTaskExecutor.java
+++ b/helix-core/src/main/java/org/apache/helix/messaging/handling/HelixTaskExecutor.java
@@ -779,78 +779,86 @@ public class HelixTaskExecutor implements MessageListener, TaskExecutor {
Set<String> createCurStateNames = new HashSet<>();
for (Message message : messages) {
- // nop messages are simply removed. It is used to trigger onMessage() in
- // situations such as register a new message handler factory
- if (message.getMsgType().equalsIgnoreCase(MessageType.NO_OP.toString())) {
- LOG.info(
- "Dropping NO-OP message. mid: " + message.getId() + ", from: " + message.getMsgSrc());
- reportAndRemoveMessage(message, accessor, instanceName, ProcessedMessageState.DISCARDED);
- continue;
- }
+ try {
+ // nop messages are simply removed. It is used to trigger onMessage() in
+ // situations such as register a new message handler factory
+ if (message.getMsgType().equalsIgnoreCase(MessageType.NO_OP.toString())) {
+ LOG.info(
+ "Dropping NO-OP message. mid: " + message.getId() + ", from: " + message.getMsgSrc());
+ reportAndRemoveMessage(message, accessor, instanceName, ProcessedMessageState.DISCARDED);
+ continue;
+ }
- String tgtSessionId = message.getTgtSessionId();
- // sessionId mismatch normally means message comes from expired session, just remove it
- if (!sessionId.equals(tgtSessionId) && !tgtSessionId.equals("*")) {
- String warningMessage =
- "SessionId does NOT match. expected sessionId: " + sessionId
- + ", tgtSessionId in message: " + tgtSessionId + ", messageId: "
- + message.getMsgId();
- LOG.warn(warningMessage);
- reportAndRemoveMessage(message, accessor, instanceName, ProcessedMessageState.DISCARDED);
- _statusUpdateUtil.logWarning(message, HelixStateMachineEngine.class, warningMessage, manager);
-
- // Proactively send a session sync message from participant to controller
- // upon session mismatch after a new session is established
- if (manager.getInstanceType() == InstanceType.PARTICIPANT
- || manager.getInstanceType() == InstanceType.CONTROLLER_PARTICIPANT) {
- if (message.getCreateTimeStamp() > manager.getSessionStartTime()) {
- syncSessionToController(manager);
+ String tgtSessionId = message.getTgtSessionId();
+ // sessionId mismatch normally means message comes from expired session, just remove it
+ if (!sessionId.equals(tgtSessionId) && !tgtSessionId.equals("*")) {
+ String warningMessage =
+ "SessionId does NOT match. expected sessionId: " + sessionId
+ + ", tgtSessionId in message: " + tgtSessionId + ", messageId: "
+ + message.getMsgId();
+ LOG.warn(warningMessage);
+ reportAndRemoveMessage(message, accessor, instanceName, ProcessedMessageState.DISCARDED);
+ _statusUpdateUtil.logWarning(message, HelixStateMachineEngine.class, warningMessage, manager);
+
+ // Proactively send a session sync message from participant to controller
+ // upon session mismatch after a new session is established
+ if (manager.getInstanceType() == InstanceType.PARTICIPANT
+ || manager.getInstanceType() == InstanceType.CONTROLLER_PARTICIPANT) {
+ if (message.getCreateTimeStamp() > manager.getSessionStartTime()) {
+ syncSessionToController(manager);
+ }
}
+ continue;
}
- continue;
- }
-
- if ((manager.getInstanceType() == InstanceType.CONTROLLER
- || manager.getInstanceType() == InstanceType.CONTROLLER_PARTICIPANT)
- && MessageType.PARTICIPANT_SESSION_CHANGE.name().equals(message.getMsgType())) {
- LOG.info(String.format("Controller received PARTICIPANT_SESSION_CHANGE msg from src: %s",
- message.getMsgSrc()));
- PropertyKey key = new Builder(manager.getClusterName()).liveInstances();
- List<LiveInstance> liveInstances = manager.getHelixDataAccessor().getChildValues(key);
- _controller.onLiveInstanceChange(liveInstances, changeContext);
- reportAndRemoveMessage(message, accessor, instanceName, ProcessedMessageState.COMPLETED);
- continue;
- }
- // don't process message that is of READ or UNPROCESSABLE state
- if (MessageState.NEW != message.getMsgState()) {
- // It happens because we don't delete message right after
- // read. Instead we keep it until the current state is updated.
- // We will read the message again if there is a new message but we
- // check for the status and ignore if its already read
- if (LOG.isTraceEnabled()) {
- LOG.trace("Message already read. msgId: " + message.getMsgId());
+ if ((manager.getInstanceType() == InstanceType.CONTROLLER
+ || manager.getInstanceType() == InstanceType.CONTROLLER_PARTICIPANT)
+ && MessageType.PARTICIPANT_SESSION_CHANGE.name().equals(message.getMsgType())) {
+ LOG.info(String.format("Controller received PARTICIPANT_SESSION_CHANGE msg from src: %s",
+ message.getMsgSrc()));
+ PropertyKey key = new Builder(manager.getClusterName()).liveInstances();
+ List<LiveInstance> liveInstances = manager.getHelixDataAccessor().getChildValues(key);
+ _controller.onLiveInstanceChange(liveInstances, changeContext);
+ reportAndRemoveMessage(message, accessor, instanceName, ProcessedMessageState.COMPLETED);
+ continue;
}
- continue;
- }
- if (message.isExpired()) {
- LOG.info(
- "Dropping expired message. mid: " + message.getId() + ", from: " + message.getMsgSrc()
- + " relayed from: " + message.getRelaySrcHost());
- reportAndRemoveMessage(message, accessor, instanceName, ProcessedMessageState.DISCARDED);
- continue;
- }
+ // don't process message that is of READ or UNPROCESSABLE state
+ if (MessageState.NEW != message.getMsgState()) {
+ // It happens because we don't delete message right after
+ // read. Instead we keep it until the current state is updated.
+ // We will read the message again if there is a new message but we
+ // check for the status and ignore if its already read
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Message already read. msgId: " + message.getMsgId());
+ }
+ continue;
+ }
- // State Transition Cancellation
- if (message.getMsgType().equals(MessageType.STATE_TRANSITION_CANCELLATION.name())) {
- boolean success = cancelNotStartedStateTransition(message, stateTransitionHandlers, accessor, instanceName);
- if (success) {
+ if (message.isExpired()) {
+ LOG.info(
+ "Dropping expired message. mid: " + message.getId() + ", from: " + message.getMsgSrc()
+ + " relayed from: " + message.getRelaySrcHost());
+ reportAndRemoveMessage(message, accessor, instanceName, ProcessedMessageState.DISCARDED);
continue;
}
- }
- _monitor.reportReceivedMessage(message);
+ // State Transition Cancellation
+ if (message.getMsgType().equals(MessageType.STATE_TRANSITION_CANCELLATION.name())) {
+ boolean success = cancelNotStartedStateTransition(message, stateTransitionHandlers, accessor, instanceName);
+ if (success) {
+ continue;
+ }
+ }
+
+ _monitor.reportReceivedMessage(message);
+ } catch (Exception e) {
+ LOG.error("Failed to process the message {}. Deleting the message from ZK. Exception: {}",
+ message, e);
+ removeMessageFromTaskAndFutureMap(message);
+ removeMessageFromZK(accessor, message, instanceName);
+ continue;
+ }
// create message handlers, if handlers not found, leave its state as NEW
NotificationContext msgWorkingContext = changeContext.clone();
diff --git a/helix-core/src/main/java/org/apache/helix/model/Message.java b/helix-core/src/main/java/org/apache/helix/model/Message.java
index e619b2b..4177b44 100644
--- a/helix-core/src/main/java/org/apache/helix/model/Message.java
+++ b/helix-core/src/main/java/org/apache/helix/model/Message.java
@@ -905,16 +905,15 @@ public class Message extends HelixProperty {
// TODO: refactor message to state transition message and task-message and
// implement this function separately
- if (getMsgType().equals(MessageType.STATE_TRANSITION.name())) {
- boolean isNotValid =
- isNullOrEmpty(getTgtName()) || isNullOrEmpty(getPartitionName())
- || isNullOrEmpty(getResourceName()) || isNullOrEmpty(getStateModelDef())
- || isNullOrEmpty(getToState()) || isNullOrEmpty(getStateModelFactoryName())
- || isNullOrEmpty(getFromState());
+ if (getMsgType().equals(MessageType.STATE_TRANSITION.name())
+ || getMsgType().equals(MessageType.STATE_TRANSITION_CANCELLATION.name())) {
+ boolean isNotValid = isNullOrEmpty(getTgtName()) || isNullOrEmpty(getPartitionName())
+ || isNullOrEmpty(getResourceName()) || isNullOrEmpty(getStateModelDef())
+ || isNullOrEmpty(getToState()) || isNullOrEmpty(getFromState())
+ || isNullOrEmpty(getTgtSessionId());
return !isNotValid;
}
-
return true;
}
}