You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pekko.apache.org by he...@apache.org on 2026/06/19 21:23:49 UTC
(pekko) branch fix/remote-send-consistency-flaky created (now 658433a168)
This is an automated email from the ASF dual-hosted git repository.
He-Pin pushed a change to branch fix/remote-send-consistency-flaky
in repository https://gitbox.apache.org/repos/asf/pekko.git
at 658433a168 fix(artery): distribute ActorSelection messages across lanes by target path
This branch includes the following new commits:
new 658433a168 fix(artery): distribute ActorSelection messages across lanes by target path
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pekko.apache.org
For additional commands, e-mail: commits-help@pekko.apache.org
(pekko) 01/01: fix(artery): distribute ActorSelection messages across lanes by target path
Posted by he...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
He-Pin pushed a commit to branch fix/remote-send-consistency-flaky
in repository https://gitbox.apache.org/repos/asf/pekko.git
commit 658433a1684b35481f46d904cd5820870fa69e95
Author: 虎鸣 <he...@alibaba-inc.com>
AuthorDate: Sat Jun 20 05:21:16 2026 +0800
fix(artery): distribute ActorSelection messages across lanes by target path
Motivation:
The "must be able to send messages with actorSelection concurrently
preserving order" test flakes on CI: 4 sender actors each drive 1000
round-trips via ActorSelection (4004 messages total). With multi-lane
artery config (outbound-lanes > 1), all ActorSelection messages are
routed to the same outbound queue because selectQueue uses the anchor's
UID as the distribution key. The anchor for ActorSelection is the root
guardian (RootActorPath), whose UID is always 0 (ActorCell.undefinedUid).
So math.abs(0 % N) = 0 for any N — all ActorSelection traffic
concentrates on lane 0 while other lanes sit idle.
Similarly, inbound lane partitioning uses the wire recipient's UID
(root guardian = 0), concentrating all inbound ActorSelection processing
on a single inbound lane.
Modification:
Outbound: Add a dedicated case for ActorSelectionMessage in
Association.send that computes the queue index from the selection's
target path elements hash instead of the anchor's UID. PriorityMessage
ActorSelection (cluster heartbeats) continues going through the control
queue.
Inbound: Update ArteryTransport.inboundLanePartitioner to parse the
ActorSelectionMessage's target path from the envelope byte buffer and
use it as the destination hash key, distributing inbound ActorSelection
processing across all inbound lanes.
Result:
ActorSelection messages are distributed across all outbound and inbound
lanes based on target path, eliminating the single-lane throughput
bottleneck while preserving per-path message ordering.
Tests:
sbt -Dpekko.test.timefactor=2 "remote / Test / testOnly
*SendConsistency*" — 24/24 passed (all 6 spec variants)
sbt "remote / mimaReportBinaryIssues" — clean
References:
Fixes #3089
---
.../pekko/remote/artery/ArteryTransport.scala | 47 +++++++++++++++++++---
.../apache/pekko/remote/artery/Association.scala | 22 +++++++++-
2 files changed, 61 insertions(+), 8 deletions(-)
diff --git a/remote/src/main/scala/org/apache/pekko/remote/artery/ArteryTransport.scala b/remote/src/main/scala/org/apache/pekko/remote/artery/ArteryTransport.scala
index ba3f8e650b..6c55fff6aa 100644
--- a/remote/src/main/scala/org/apache/pekko/remote/artery/ArteryTransport.scala
+++ b/remote/src/main/scala/org/apache/pekko/remote/artery/ArteryTransport.scala
@@ -13,6 +13,7 @@
package org.apache.pekko.remote.artery
+import java.nio.ByteBuffer
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.atomic.AtomicLong
@@ -25,7 +26,7 @@ import scala.concurrent.Future
import scala.concurrent.Promise
import scala.concurrent.duration._
import scala.util.Try
-import scala.util.control.NoStackTrace
+import scala.util.control.{ NoStackTrace, NonFatal }
import org.apache.pekko
import pekko.Done
@@ -36,6 +37,7 @@ import pekko.dispatch.Dispatchers
import pekko.event.Logging
import pekko.event.MarkerLoggingAdapter
import pekko.remote.AddressUidExtension
+import pekko.remote.ContainerFormats
import pekko.remote.RemoteActorRef
import pekko.remote.RemoteActorRefProvider
import pekko.remote.RemoteTransport
@@ -54,6 +56,7 @@ import pekko.stream._
import pekko.stream.scaladsl.Flow
import pekko.stream.scaladsl.Keep
import pekko.stream.scaladsl.Sink
+import pekko.serialization.SerializationExtension
import pekko.util.OptionVal
import pekko.util.WildcardIndex
@@ -333,6 +336,9 @@ private[remote] abstract class ArteryTransport(_system: ExtendedActorSystem, _pr
protected val inboundLanes = settings.Advanced.InboundLanes
+ private lazy val actorSelectionMessageSerializerId =
+ SerializationExtension(system).serializerFor(classOf[pekko.actor.ActorSelectionMessage]).identifier
+
val largeMessageChannelEnabled: Boolean =
!settings.LargeMessageDestinations.wildcardTree.isEmpty ||
!settings.LargeMessageDestinations.doubleWildcardTree.isEmpty
@@ -460,14 +466,16 @@ private[remote] abstract class ArteryTransport(_system: ExtendedActorSystem, _pr
// Select inbound lane based on destination to preserve message order,
// Also include the uid of the sending system in the hash to spread
- // "hot" destinations, e.g. ActorSelection anchor.
+ // "hot" destinations. For ActorSelection, the recipient is typically the
+ // root guardian, so use the selected target path as the destination key.
protected val inboundLanePartitioner: InboundEnvelope => Int = env => {
env.recipient match {
case OptionVal.Some(r) =>
- val a = r.path.uid
- val b = env.originUid
- val hashA = 23 + a
- val hash: Int = 23 * hashA + java.lang.Long.hashCode(b)
+ val destinationHash =
+ if (env.serializer == actorSelectionMessageSerializerId && (env.envelopeBuffer ne null))
+ ArteryTransport.actorSelectionMessagePathHash(env.envelopeBuffer.byteBuffer).getOrElse(r.path.uid)
+ else r.path.uid
+ val hash = ArteryTransport.inboundLanePartitionHash(destinationHash, env.originUid)
math.abs(hash % inboundLanes)
case _ =>
// the lane is set by the DuplicateHandshakeReq stage, otherwise 0
@@ -986,4 +994,31 @@ private[remote] object ArteryTransport {
case _ => "message"
}
+ def inboundLanePartitionHash(destinationHash: Int, originUid: Long): Int = {
+ val hashA = 23 + destinationHash
+ 23 * hashA + java.lang.Long.hashCode(originUid)
+ }
+
+ def actorSelectionMessagePathHash(byteBuffer: ByteBuffer): Option[Int] = {
+ val copy = byteBuffer.asReadOnlyBuffer()
+ try Some(actorSelectionMessagePathHash(ContainerFormats.SelectionEnvelope.parseFrom(copy)))
+ catch {
+ case NonFatal(_) => None
+ }
+ }
+
+ def actorSelectionMessagePathHash(selectionEnvelope: ContainerFormats.SelectionEnvelope): Int = {
+ var hash = 23
+ val patterns = selectionEnvelope.getPatternList
+ var i = 0
+ while (i < patterns.size) {
+ val pattern = patterns.get(i)
+ hash = 23 * hash + pattern.getType.getNumber
+ if (pattern.hasMatcher)
+ hash = 23 * hash + pattern.getMatcher.hashCode
+ i += 1
+ }
+ hash
+ }
+
}
diff --git a/remote/src/main/scala/org/apache/pekko/remote/artery/Association.scala b/remote/src/main/scala/org/apache/pekko/remote/artery/Association.scala
index eb9bf1176a..f0fc935036 100644
--- a/remote/src/main/scala/org/apache/pekko/remote/artery/Association.scala
+++ b/remote/src/main/scala/org/apache/pekko/remote/artery/Association.scala
@@ -424,8 +424,26 @@ private[remote] class Association(
}(materializer.executionContext)
case _: SystemMessage =>
sendSystemMessage(outboundEnvelope)
- case ActorSelectionMessage(_: PriorityMessage, _, _) | _: ControlMessage | _: ClearSystemMessageDelivery =>
- // ActorSelectionMessage with PriorityMessage is used by cluster and remote failure detector heartbeating
+ case sel: ActorSelectionMessage =>
+ sel.msg match {
+ case _: PriorityMessage =>
+ // ActorSelectionMessage with PriorityMessage is used by cluster and remote failure detector heartbeating
+ if (!controlQueue.offer(outboundEnvelope)) {
+ dropped(ControlQueueIndex, controlQueueSize, outboundEnvelope)
+ }
+ case _ =>
+ // Distribute ActorSelection messages across lanes based on the target
+ // path hash rather than the anchor's (typically root guardian, UID=0).
+ // Without this, all ActorSelection messages concentrate on a single
+ // outbound lane, creating a throughput bottleneck.
+ val queueIndex =
+ if (outboundLanes == 1) OrdinaryQueueIndex
+ else OrdinaryQueueIndex + ((sel.elements.hashCode() & Int.MaxValue) % outboundLanes)
+ val queue = queues(queueIndex)
+ if (!queue.offer(outboundEnvelope))
+ dropped(queueIndex, queueSize, outboundEnvelope)
+ }
+ case _: ControlMessage | _: ClearSystemMessageDelivery =>
if (!controlQueue.offer(outboundEnvelope)) {
dropped(ControlQueueIndex, controlQueueSize, outboundEnvelope)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pekko.apache.org
For additional commands, e-mail: commits-help@pekko.apache.org