You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by jx...@apache.org on 2019/02/12 22:28:21 UTC

[helix] branch master updated: HELIX: Fix null tgtSessionId message bug

This is an automated email from the ASF dual-hosted git repository.

jxue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git


The following commit(s) were added to refs/heads/master by this push:
     new b0c3bfd  HELIX: Fix null tgtSessionId message bug
b0c3bfd is described below

commit b0c3bfd693b9e8352ffa55ce3c429a46057d3512
Author: narendly <na...@gmail.com>
AuthorDate: Mon Feb 11 16:22:32 2019 -0800

    HELIX: Fix null tgtSessionId message bug
    
    It has been reported that the Controller sometimes sends a message with a null tgtSessionId, which was causing an NPE on the Participant.
    Changelist:
        1. Controller side fix
        2. Participant side fix with a try-catch
        3. Modify isValid for messages
---
 .../controller/stages/MessageGenerationPhase.java  |   9 ++
 .../messaging/handling/HelixTaskExecutor.java      | 132 +++++++++++----------
 .../main/java/org/apache/helix/model/Message.java  |  13 +-
 3 files changed, 85 insertions(+), 69 deletions(-)

diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/MessageGenerationPhase.java b/helix-core/src/main/java/org/apache/helix/controller/stages/MessageGenerationPhase.java
index c829082..4c693a6 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/stages/MessageGenerationPhase.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/stages/MessageGenerationPhase.java
@@ -267,6 +267,15 @@ public abstract class MessageGenerationPhase extends AbstractBaseStage {
       for (String state : statesPriorityList) {
         if (messageMap.containsKey(state)) {
           for (Message message : messageMap.get(state)) {
+            // This is for a bug where a message's target session id is null
+            if (!message.isValid()) {
+              LogUtil.logError(logger, _eventId, String.format(
+                  "An invalid message was generated! Discarding this message. sessionIdMap: %s, CurrentStateMap: %s, InstanceStateMap: %s, AllInstances: %s, LiveInstances: %s, Message: %s",
+                  sessionIdMap, currentStateOutput.getCurrentStateMap(resourceName, partition),
+                  instanceStateMap, cache.getAllInstances(), cache.getLiveInstances().keySet(),
+                  message));
+              continue; // Do not add this message
+            }
             output.addMessage(resourceName, partition, message);
           }
         }
diff --git a/helix-core/src/main/java/org/apache/helix/messaging/handling/HelixTaskExecutor.java b/helix-core/src/main/java/org/apache/helix/messaging/handling/HelixTaskExecutor.java
index 3ae90d3..a0481e3 100644
--- a/helix-core/src/main/java/org/apache/helix/messaging/handling/HelixTaskExecutor.java
+++ b/helix-core/src/main/java/org/apache/helix/messaging/handling/HelixTaskExecutor.java
@@ -779,78 +779,86 @@ public class HelixTaskExecutor implements MessageListener, TaskExecutor {
     Set<String> createCurStateNames = new HashSet<>();
 
     for (Message message : messages) {
-      // nop messages are simply removed. It is used to trigger onMessage() in
-      // situations such as register a new message handler factory
-      if (message.getMsgType().equalsIgnoreCase(MessageType.NO_OP.toString())) {
-        LOG.info(
-            "Dropping NO-OP message. mid: " + message.getId() + ", from: " + message.getMsgSrc());
-        reportAndRemoveMessage(message, accessor, instanceName, ProcessedMessageState.DISCARDED);
-        continue;
-      }
+      try {
+        // nop messages are simply removed. It is used to trigger onMessage() in
+        // situations such as register a new message handler factory
+        if (message.getMsgType().equalsIgnoreCase(MessageType.NO_OP.toString())) {
+          LOG.info(
+              "Dropping NO-OP message. mid: " + message.getId() + ", from: " + message.getMsgSrc());
+          reportAndRemoveMessage(message, accessor, instanceName, ProcessedMessageState.DISCARDED);
+          continue;
+        }
 
-      String tgtSessionId = message.getTgtSessionId();
-      // sessionId mismatch normally means message comes from expired session, just remove it
-      if (!sessionId.equals(tgtSessionId) && !tgtSessionId.equals("*")) {
-        String warningMessage =
-            "SessionId does NOT match. expected sessionId: " + sessionId
-                + ", tgtSessionId in message: " + tgtSessionId + ", messageId: "
-                + message.getMsgId();
-        LOG.warn(warningMessage);
-        reportAndRemoveMessage(message, accessor, instanceName, ProcessedMessageState.DISCARDED);
-        _statusUpdateUtil.logWarning(message, HelixStateMachineEngine.class, warningMessage, manager);
-
-        // Proactively send a session sync message from participant to controller
-        // upon session mismatch after a new session is established
-        if (manager.getInstanceType() == InstanceType.PARTICIPANT
-            || manager.getInstanceType() == InstanceType.CONTROLLER_PARTICIPANT) {
-          if (message.getCreateTimeStamp() > manager.getSessionStartTime()) {
-            syncSessionToController(manager);
+        String tgtSessionId = message.getTgtSessionId();
+        // sessionId mismatch normally means message comes from expired session, just remove it
+        if (!sessionId.equals(tgtSessionId) && !tgtSessionId.equals("*")) {
+          String warningMessage =
+              "SessionId does NOT match. expected sessionId: " + sessionId
+                  + ", tgtSessionId in message: " + tgtSessionId + ", messageId: "
+                  + message.getMsgId();
+          LOG.warn(warningMessage);
+          reportAndRemoveMessage(message, accessor, instanceName, ProcessedMessageState.DISCARDED);
+          _statusUpdateUtil.logWarning(message, HelixStateMachineEngine.class, warningMessage, manager);
+
+          // Proactively send a session sync message from participant to controller
+          // upon session mismatch after a new session is established
+          if (manager.getInstanceType() == InstanceType.PARTICIPANT
+              || manager.getInstanceType() == InstanceType.CONTROLLER_PARTICIPANT) {
+            if (message.getCreateTimeStamp() > manager.getSessionStartTime()) {
+              syncSessionToController(manager);
+            }
           }
+          continue;
         }
-        continue;
-      }
-
-      if ((manager.getInstanceType() == InstanceType.CONTROLLER
-          || manager.getInstanceType() == InstanceType.CONTROLLER_PARTICIPANT)
-          && MessageType.PARTICIPANT_SESSION_CHANGE.name().equals(message.getMsgType())) {
-        LOG.info(String.format("Controller received PARTICIPANT_SESSION_CHANGE msg from src: %s",
-            message.getMsgSrc()));
-        PropertyKey key = new Builder(manager.getClusterName()).liveInstances();
-        List<LiveInstance> liveInstances = manager.getHelixDataAccessor().getChildValues(key);
-        _controller.onLiveInstanceChange(liveInstances, changeContext);
-        reportAndRemoveMessage(message, accessor, instanceName, ProcessedMessageState.COMPLETED);
-        continue;
-      }
 
-      // don't process message that is of READ or UNPROCESSABLE state
-      if (MessageState.NEW != message.getMsgState()) {
-        // It happens because we don't delete message right after
-        // read. Instead we keep it until the current state is updated.
-        // We will read the message again if there is a new message but we
-        // check for the status and ignore if its already read
-        if (LOG.isTraceEnabled()) {
-          LOG.trace("Message already read. msgId: " + message.getMsgId());
+        if ((manager.getInstanceType() == InstanceType.CONTROLLER
+            || manager.getInstanceType() == InstanceType.CONTROLLER_PARTICIPANT)
+            && MessageType.PARTICIPANT_SESSION_CHANGE.name().equals(message.getMsgType())) {
+          LOG.info(String.format("Controller received PARTICIPANT_SESSION_CHANGE msg from src: %s",
+              message.getMsgSrc()));
+          PropertyKey key = new Builder(manager.getClusterName()).liveInstances();
+          List<LiveInstance> liveInstances = manager.getHelixDataAccessor().getChildValues(key);
+          _controller.onLiveInstanceChange(liveInstances, changeContext);
+          reportAndRemoveMessage(message, accessor, instanceName, ProcessedMessageState.COMPLETED);
+          continue;
         }
-        continue;
-      }
 
-      if (message.isExpired()) {
-        LOG.info(
-            "Dropping expired message. mid: " + message.getId() + ", from: " + message.getMsgSrc()
-                + " relayed from: " + message.getRelaySrcHost());
-        reportAndRemoveMessage(message, accessor, instanceName, ProcessedMessageState.DISCARDED);
-        continue;
-      }
+        // don't process message that is of READ or UNPROCESSABLE state
+        if (MessageState.NEW != message.getMsgState()) {
+          // It happens because we don't delete message right after
+          // read. Instead we keep it until the current state is updated.
+          // We will read the message again if there is a new message but we
+          // check for the status and ignore if its already read
+          if (LOG.isTraceEnabled()) {
+            LOG.trace("Message already read. msgId: " + message.getMsgId());
+          }
+          continue;
+        }
 
-      // State Transition Cancellation
-      if (message.getMsgType().equals(MessageType.STATE_TRANSITION_CANCELLATION.name())) {
-        boolean success = cancelNotStartedStateTransition(message, stateTransitionHandlers, accessor, instanceName);
-        if (success) {
+        if (message.isExpired()) {
+          LOG.info(
+              "Dropping expired message. mid: " + message.getId() + ", from: " + message.getMsgSrc()
+                  + " relayed from: " + message.getRelaySrcHost());
+          reportAndRemoveMessage(message, accessor, instanceName, ProcessedMessageState.DISCARDED);
           continue;
         }
-      }
 
-      _monitor.reportReceivedMessage(message);
+        // State Transition Cancellation
+        if (message.getMsgType().equals(MessageType.STATE_TRANSITION_CANCELLATION.name())) {
+          boolean success = cancelNotStartedStateTransition(message, stateTransitionHandlers, accessor, instanceName);
+          if (success) {
+            continue;
+          }
+        }
+
+        _monitor.reportReceivedMessage(message);
+      } catch (Exception e) {
+        LOG.error("Failed to process the message {}. Deleting the message from ZK. Exception: {}",
+            message, e);
+        removeMessageFromTaskAndFutureMap(message);
+        removeMessageFromZK(accessor, message, instanceName);
+        continue;
+      }
 
       // create message handlers, if handlers not found, leave its state as NEW
       NotificationContext msgWorkingContext = changeContext.clone();
diff --git a/helix-core/src/main/java/org/apache/helix/model/Message.java b/helix-core/src/main/java/org/apache/helix/model/Message.java
index e619b2b..4177b44 100644
--- a/helix-core/src/main/java/org/apache/helix/model/Message.java
+++ b/helix-core/src/main/java/org/apache/helix/model/Message.java
@@ -905,16 +905,15 @@ public class Message extends HelixProperty {
     // TODO: refactor message to state transition message and task-message and
     // implement this function separately
 
-    if (getMsgType().equals(MessageType.STATE_TRANSITION.name())) {
-      boolean isNotValid =
-          isNullOrEmpty(getTgtName()) || isNullOrEmpty(getPartitionName())
-              || isNullOrEmpty(getResourceName()) || isNullOrEmpty(getStateModelDef())
-              || isNullOrEmpty(getToState()) || isNullOrEmpty(getStateModelFactoryName())
-              || isNullOrEmpty(getFromState());
+    if (getMsgType().equals(MessageType.STATE_TRANSITION.name())
+        || getMsgType().equals(MessageType.STATE_TRANSITION_CANCELLATION.name())) {
+      boolean isNotValid = isNullOrEmpty(getTgtName()) || isNullOrEmpty(getPartitionName())
+          || isNullOrEmpty(getResourceName()) || isNullOrEmpty(getStateModelDef())
+          || isNullOrEmpty(getToState()) || isNullOrEmpty(getFromState())
+          || isNullOrEmpty(getTgtSessionId());
 
       return !isNotValid;
     }
-
     return true;
   }
 }