You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@texera.apache.org by li...@apache.org on 2026/04/21 08:20:28 UTC

(texera) branch xinyuan-scheduler-jump created (now 555e55df54)

This is an automated email from the ASF dual-hosted git repository.

aglinxinyuan pushed a change to branch xinyuan-scheduler-jump
in repository https://gitbox.apache.org/repos/asf/texera.git


      at 555e55df54 Add scheduler jump-to-operator support

This branch includes the following new commits:

     new 555e55df54 Add scheduler jump-to-operator support

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



(texera) 01/01: Add scheduler jump-to-operator support

Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

aglinxinyuan pushed a commit to branch xinyuan-scheduler-jump
in repository https://gitbox.apache.org/repos/asf/texera.git

commit 555e55df5483a103e4282b503e33ee3b0dcc1bb2
Author: Xinyuan Lin <xi...@uci.edu>
AuthorDate: Tue Apr 21 01:20:16 2026 -0700

    Add scheduler jump-to-operator support
---
 .../engine/architecture/rpc/controlcommands.proto  |  7 +++-
 .../architecture/rpc/controllerservice.proto       |  3 +-
 .../amber/engine/architecture/rpc/__init__.py      | 43 ++++++++++++++++++++++
 .../ControllerAsyncRPCHandlerInitializer.scala     |  1 +
 .../controller/ControllerProcessor.scala           |  2 +-
 .../controller/WorkflowScheduler.scala             |  3 ++
 .../promisehandlers/JumpToOperatorHandler.scala}   | 27 +++++++++-----
 .../engine/architecture/scheduling/Schedule.scala  | 11 ++++++
 .../scheduling/WorkflowExecutionCoordinator.scala  | 11 ++++--
 9 files changed, 92 insertions(+), 16 deletions(-)

diff --git a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto
index d714f64a15..d8d8b512d0 100644
--- a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto
+++ b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlcommands.proto
@@ -46,6 +46,7 @@ message ControlRequest {
     PortCompletedRequest portCompletedRequest = 9;
     WorkerStateUpdatedRequest workerStateUpdatedRequest = 10;
     LinkWorkersRequest linkWorkersRequest = 11;
+    JumpToOperatorRequest jumpToOperatorRequest = 12;
 
     // request for worker
     AddInputChannelRequest addInputChannelRequest = 50;
@@ -278,4 +279,8 @@ enum StatisticsUpdateTarget {
 message QueryStatisticsRequest{
   repeated core.ActorVirtualIdentity filterByWorkers = 1;
   StatisticsUpdateTarget updateTarget = 2;
-}
\ No newline at end of file
+}
+
+message JumpToOperatorRequest{
+  core.OperatorIdentity targetOperatorId = 1 [(scalapb.field).no_box = true];
+}
diff --git a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controllerservice.proto b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controllerservice.proto
index 70d189a341..25c90e3e93 100644
--- a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controllerservice.proto
+++ b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controllerservice.proto
@@ -42,7 +42,8 @@ service ControllerService {
   rpc PauseWorkflow(EmptyRequest) returns (EmptyReturn);
   rpc WorkerStateUpdated(WorkerStateUpdatedRequest) returns (EmptyReturn);
   rpc WorkerExecutionCompleted(EmptyRequest) returns (EmptyReturn);
+  rpc JumpToOperator(JumpToOperatorRequest) returns (EmptyReturn);
   rpc LinkWorkers(LinkWorkersRequest) returns (EmptyReturn);
   rpc ControllerInitiateQueryStatistics(QueryStatisticsRequest) returns (EmptyReturn);
   rpc RetryWorkflow(RetryWorkflowRequest) returns (EmptyReturn);
-}
\ No newline at end of file
+}
diff --git a/amber/src/main/python/proto/org/apache/texera/amber/engine/architecture/rpc/__init__.py b/amber/src/main/python/proto/org/apache/texera/amber/engine/architecture/rpc/__init__.py
index b7522a696a..f946e7123e 100644
--- a/amber/src/main/python/proto/org/apache/texera/amber/engine/architecture/rpc/__init__.py
+++ b/amber/src/main/python/proto/org/apache/texera/amber/engine/architecture/rpc/__init__.py
@@ -102,6 +102,9 @@ class ControlRequest(betterproto.Message):
     link_workers_request: "LinkWorkersRequest" = betterproto.message_field(
         11, group="sealed_value"
     )
+    jump_to_operator_request: "JumpToOperatorRequest" = betterproto.message_field(
+        12, group="sealed_value"
+    )
     add_input_channel_request: "AddInputChannelRequest" = betterproto.message_field(
         50, group="sealed_value"
     )
@@ -394,6 +397,11 @@ class QueryStatisticsRequest(betterproto.Message):
     update_target: "StatisticsUpdateTarget" = betterproto.enum_field(2)
 
 
+@dataclass(eq=False, repr=False)
+class JumpToOperatorRequest(betterproto.Message):
+    target_operator_id: "___core__.OperatorIdentity" = betterproto.message_field(1)
+
+
 @dataclass(eq=False, repr=False)
 class ControlReturn(betterproto.Message):
     """The generic return message"""
@@ -1243,6 +1251,23 @@ class ControllerServiceStub(betterproto.ServiceStub):
             metadata=metadata,
         )
 
+    async def jump_to_operator(
+        self,
+        jump_to_operator_request: "JumpToOperatorRequest",
+        *,
+        timeout: Optional[float] = None,
+        deadline: Optional["Deadline"] = None,
+        metadata: Optional["MetadataLike"] = None
+    ) -> "EmptyReturn":
+        return await self._unary_unary(
+            "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/JumpToOperator",
+            jump_to_operator_request,
+            EmptyReturn,
+            timeout=timeout,
+            deadline=deadline,
+            metadata=metadata,
+        )
+
     async def link_workers(
         self,
         link_workers_request: "LinkWorkersRequest",
@@ -1880,6 +1905,11 @@ class ControllerServiceBase(ServiceBase):
     ) -> "EmptyReturn":
         raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED)
 
+    async def jump_to_operator(
+        self, jump_to_operator_request: "JumpToOperatorRequest"
+    ) -> "EmptyReturn":
+        raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED)
+
     async def link_workers(
         self, link_workers_request: "LinkWorkersRequest"
     ) -> "EmptyReturn":
@@ -1984,6 +2014,13 @@ class ControllerServiceBase(ServiceBase):
         response = await self.worker_execution_completed(request)
         await stream.send_message(response)
 
+    async def __rpc_jump_to_operator(
+        self, stream: "grpclib.server.Stream[JumpToOperatorRequest, EmptyReturn]"
+    ) -> None:
+        request = await stream.recv_message()
+        response = await self.jump_to_operator(request)
+        await stream.send_message(response)
+
     async def __rpc_link_workers(
         self, stream: "grpclib.server.Stream[LinkWorkersRequest, EmptyReturn]"
     ) -> None:
@@ -2079,6 +2116,12 @@ class ControllerServiceBase(ServiceBase):
                 EmptyRequest,
                 EmptyReturn,
             ),
+            "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/JumpToOperator": grpclib.const.Handler(
+                self.__rpc_jump_to_operator,
+                grpclib.const.Cardinality.UNARY_UNARY,
+                JumpToOperatorRequest,
+                EmptyReturn,
+            ),
             "/org.apache.texera.amber.engine.architecture.rpc.ControllerService/LinkWorkers": grpclib.const.Handler(
                 self.__rpc_link_workers,
                 grpclib.const.Cardinality.UNARY_UNARY,
diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ControllerAsyncRPCHandlerInitializer.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ControllerAsyncRPCHandlerInitializer.scala
index 4d9a36bab4..03a4d47968 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ControllerAsyncRPCHandlerInitializer.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ControllerAsyncRPCHandlerInitializer.scala
@@ -34,6 +34,7 @@ class ControllerAsyncRPCHandlerInitializer(
     with AmberLogging
     with LinkWorkersHandler
     with WorkerExecutionCompletedHandler
+    with JumpToOperatorHandler
     with WorkerStateUpdatedHandler
     with PauseHandler
     with QueryWorkerStatisticsHandler
diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ControllerProcessor.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ControllerProcessor.scala
index 7a8e94cf3a..3461619cb3 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ControllerProcessor.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/ControllerProcessor.scala
@@ -44,7 +44,7 @@ class ControllerProcessor(
   val workflowScheduler: WorkflowScheduler =
     new WorkflowScheduler(workflowContext, actorId)
   val workflowExecutionCoordinator: WorkflowExecutionCoordinator = new WorkflowExecutionCoordinator(
-    () => this.workflowScheduler.getNextRegions,
+    workflowScheduler,
     workflowExecution,
     controllerConfig,
     asyncRPCClient
diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/WorkflowScheduler.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/WorkflowScheduler.scala
index 9dcf3ad4bf..c8a107e045 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/WorkflowScheduler.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/WorkflowScheduler.scala
@@ -20,6 +20,7 @@
 package org.apache.texera.amber.engine.architecture.controller
 
 import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity
+import org.apache.texera.amber.core.virtualidentity.OperatorIdentity
 import org.apache.texera.amber.core.workflow.{PhysicalPlan, WorkflowContext}
 import org.apache.texera.amber.engine.architecture.scheduling.{
   CostBasedScheduleGenerator,
@@ -54,4 +55,6 @@ class WorkflowScheduler(
 
   def getNextRegions: Set[Region] = if (!schedule.hasNext) Set() else schedule.next()
 
+  def jumpToOperator(opId: OperatorIdentity): Unit = schedule.jumpToOperator(opId)
+
 }
diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/JumpToOperatorHandler.scala
similarity index 50%
copy from amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala
copy to amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/JumpToOperatorHandler.scala
index 6f34c9ed1e..aad72f08e9 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/JumpToOperatorHandler.scala
@@ -17,18 +17,25 @@
  * under the License.
  */
 
-package org.apache.texera.amber.engine.architecture.scheduling
+package org.apache.texera.amber.engine.architecture.controller.promisehandlers
 
-case class Schedule(private val levelSets: Map[Int, Set[Region]]) extends Iterator[Set[Region]] {
-  private var currentLevel = levelSets.keys.minOption.getOrElse(0)
-
-  def getRegions: List[Region] = levelSets.values.flatten.toList
+import com.twitter.util.Future
+import org.apache.texera.amber.engine.architecture.controller.ControllerAsyncRPCHandlerInitializer
+import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{
+  AsyncRPCContext,
+  JumpToOperatorRequest
+}
+import org.apache.texera.amber.engine.architecture.rpc.controlreturns.EmptyReturn
 
-  override def hasNext: Boolean = levelSets.isDefinedAt(currentLevel)
+/** Requests the scheduler to continue from the region containing the target operator. */
+trait JumpToOperatorHandler {
+  this: ControllerAsyncRPCHandlerInitializer =>
 
-  override def next(): Set[Region] = {
-    val regions = levelSets(currentLevel)
-    currentLevel += 1
-    regions
+  override def jumpToOperator(
+      msg: JumpToOperatorRequest,
+      ctx: AsyncRPCContext
+  ): Future[EmptyReturn] = {
+    cp.workflowExecutionCoordinator.jumpToOperator(msg.targetOperatorId)
+    EmptyReturn()
   }
 }
diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala
index 6f34c9ed1e..fa8121cad5 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/Schedule.scala
@@ -19,6 +19,8 @@
 
 package org.apache.texera.amber.engine.architecture.scheduling
 
+import org.apache.texera.amber.core.virtualidentity.OperatorIdentity
+
 case class Schedule(private val levelSets: Map[Int, Set[Region]]) extends Iterator[Set[Region]] {
   private var currentLevel = levelSets.keys.minOption.getOrElse(0)
 
@@ -31,4 +33,13 @@ case class Schedule(private val levelSets: Map[Int, Set[Region]]) extends Iterat
     currentLevel += 1
     regions
   }
+
+  def jumpToOperator(opId: OperatorIdentity): Unit =
+    levelSets
+      .collectFirst {
+        case (level, regions)
+            if regions.exists(_.getOperators.exists(_.id.logicalOpId == opId)) =>
+          level
+      }
+      .foreach(currentLevel = _)
 }
diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/WorkflowExecutionCoordinator.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/WorkflowExecutionCoordinator.scala
index 05585f88d8..2b8e3ce145 100644
--- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/WorkflowExecutionCoordinator.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/WorkflowExecutionCoordinator.scala
@@ -21,19 +21,20 @@ package org.apache.texera.amber.engine.architecture.scheduling
 
 import com.twitter.util.Future
 import com.typesafe.scalalogging.LazyLogging
+import org.apache.texera.amber.core.virtualidentity.OperatorIdentity
 import org.apache.texera.amber.core.workflow.{GlobalPortIdentity, PhysicalLink}
 import org.apache.texera.amber.engine.architecture.common.{
   AkkaActorRefMappingService,
   AkkaActorService
 }
-import org.apache.texera.amber.engine.architecture.controller.ControllerConfig
+import org.apache.texera.amber.engine.architecture.controller.{ControllerConfig, WorkflowScheduler}
 import org.apache.texera.amber.engine.architecture.controller.execution.WorkflowExecution
 import org.apache.texera.amber.engine.common.rpc.AsyncRPCClient
 
 import scala.collection.mutable
 
 class WorkflowExecutionCoordinator(
-    getNextRegions: () => Set[Region],
+    workflowScheduler: WorkflowScheduler,
     workflowExecution: WorkflowExecution,
     controllerConfig: ControllerConfig,
     asyncRPCClient: AsyncRPCClient
@@ -81,7 +82,7 @@ class WorkflowExecutionCoordinator(
     // All existing regions are completed. Start the next region (if any).
     Future
       .collect({
-        val nextRegions = getNextRegions()
+        val nextRegions = workflowScheduler.getNextRegions
         executedRegions.append(nextRegions)
         nextRegions
           .map(region => {
@@ -116,4 +117,8 @@ class WorkflowExecutionCoordinator(
       .toSet
   }
 
+  def jumpToOperator(opId: OperatorIdentity): Unit = {
+    workflowScheduler.jumpToOperator(opId)
+  }
+
 }