You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@texera.apache.org by xi...@apache.org on 2026/06/02 04:32:06 UTC
(texera) 37/47: Merge branch 'refs/heads/main' into xiaozhen-caching-prototype
This is an automated email from the ASF dual-hosted git repository.
Xiao-zhen-Liu pushed a commit to branch xiaozhen-caching-prototype
in repository https://gitbox.apache.org/repos/asf/texera.git
commit 76c03feae6a671c69bf0da27ef7e8046fb4959e9
Merge: 57a492f24d 95496cce27
Author: Xiaozhen Liu <xi...@uci.edu>
AuthorDate: Tue Feb 24 14:22:11 2026 -0800
Merge branch 'refs/heads/main' into xiaozhen-caching-prototype
# Conflicts:
# amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/QueryWorkerStatisticsHandler.scala
# amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/WorkerExecutionCompletedHandler.scala
# amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala
.github/release/vote-email-template.md | 24 +-
.github/workflows/build-and-push-images.yml | 72 +++--
.github/workflows/create-release-candidate.yml | 313 +++++++++++++++------
.github/workflows/github-action-build.yml | 19 +-
.../engine/architecture/rpc/controlcommands.proto | 7 +
.../input_port_materialization_reader_runnable.py | 3 +-
.../architecture/controller/ClientEvent.scala | 3 +
.../architecture/controller/Controller.scala | 3 +
.../controller/ControllerTimerService.scala | 54 +++-
.../controller/promisehandlers/PauseHandler.scala | 14 +-
.../promisehandlers/PortCompletedHandler.scala | 11 +-
.../QueryWorkerStatisticsHandler.scala | 58 +++-
.../controller/promisehandlers/ResumeHandler.scala | 15 +-
.../promisehandlers/StartWorkflowHandler.scala | 1 +
.../WorkerExecutionCompletedHandler.scala | 5 +-
.../WorkerStateUpdatedHandler.scala | 11 +-
.../scheduling/RegionExecutionCoordinator.scala | 15 +-
.../texera/web/ServletAwareConfigurator.scala | 2 +
.../apache/texera/web/auth/GuestAuthFilter.scala | 2 +-
.../apache/texera/web/auth/UserAuthenticator.scala | 1 +
.../texera/web/resource/auth/AuthResource.scala | 2 +
.../dashboard/admin/user/AdminUserResource.scala | 6 +-
.../web/resource/dashboard/user/UserResource.scala | 61 ++--
.../texera/web/service/ExecutionStatsService.scala | 17 +-
bin/k8s/README.md | 26 +-
.../access-control-service-deployment.yaml | 2 +-
bin/k8s/templates/config-service-deployment.yaml | 2 +-
bin/k8s/templates/example-data-loader-job.yaml | 2 +-
bin/k8s/templates/file-service-deployment.yaml | 2 +-
bin/k8s/templates/webserver-deployment.yaml | 2 +-
.../workflow-compiling-service-deployment.yaml | 2 +-
...workflow-computing-unit-manager-deployment.yaml | 4 +-
...ow-computing-unit-master-prepull-daemonset.yaml | 2 +-
bin/k8s/values.yaml | 20 +-
bin/single-node/.env | 5 +
bin/single-node/docker-compose.yml | 18 +-
.../scala/org/apache/texera/auth/JwtParser.scala | 3 +-
common/config/src/main/resources/application.conf | 3 +
.../texera/amber/config/ApplicationConfig.scala | 2 +
common/workflow-core/build.sbt | 2 +-
.../apache/texera/amber/operator/LogicalOp.scala | 4 +-
.../sklearn/testing/SklearnTestingOpDesc.scala | 115 ++++++++
file-service/build.sbt | 2 +-
.../texera/service/resource/DatasetResource.scala | 25 +-
.../service/resource/DatasetResourceSpec.scala | 4 +
frontend/.yarnrc.yml | 1 +
frontend/src/app/app.module.ts | 4 +
.../src/app/common/service/gmail/gmail.service.ts | 2 +-
.../src/app/common/service/user/auth.service.ts | 104 ++++++-
.../registration-request-modal.component.html | 54 ++++
.../registration-request-modal.component.scss} | 18 +-
.../registration-request-modal.component.ts | 51 ++++
.../app/common/service/user/stub-user.service.ts | 2 +
.../src/app/common/service/user/user.service.ts | 32 ---
frontend/src/app/common/type/user.ts | 1 +
.../component/admin/user/admin-user.component.html | 6 +
.../component/admin/user/admin-user.component.ts | 5 +
.../dashboard/component/dashboard.component.html | 32 ---
.../app/dashboard/component/dashboard.component.ts | 67 +----
.../user/list-item/list-item.component.html | 15 +-
.../user/list-item/list-item.component.scss | 1 +
.../user/list-item/list-item.component.ts | 49 +++-
.../markdown-description.component.html | 97 +++++++
.../markdown-description.component.scss | 95 +++++++
.../markdown-description.component.ts | 143 ++++++++++
.../dataset-detail.component.html | 8 +-
.../dataset-detail.component.ts | 21 ++
.../detail/hub-workflow-detail.component.html | 2 +-
.../coeditor-user-icon.component.ts | 10 +-
.../workspace/component/menu/menu.component.html | 8 +
.../app/workspace/component/menu/menu.component.ts | 18 ++
.../workflow-editor/workflow-editor.component.scss | 4 +
.../workflow-editor/workflow-editor.component.ts | 49 +++-
.../workspace/service/joint-ui/joint-ui.service.ts | 19 ++
.../src/assets/operator_images/SklearnTesting.png | Bin 0 -> 843070 bytes
sql/texera_ddl.sql | 5 +-
.../updates/21.sql | 19 +-
.../updates/22.sql | 23 +-
78 files changed, 1465 insertions(+), 466 deletions(-)
diff --cc amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/PortCompletedHandler.scala
index abb8b1357a,810c098c41..b64e11eeb0
--- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/PortCompletedHandler.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/PortCompletedHandler.scala
@@@ -30,13 -29,12 +30,14 @@@ import org.apache.texera.amber.engine.a
import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{
AsyncRPCContext,
PortCompletedRequest,
- QueryStatisticsRequest
+ QueryStatisticsRequest,
+ StatisticsUpdateTarget
}
import org.apache.texera.amber.engine.architecture.rpc.controlreturns.EmptyReturn
+import org.apache.texera.amber.engine.architecture.scheduling.config.OutputPortConfig
import org.apache.texera.amber.engine.common.virtualidentity.util.CONTROLLER
import org.apache.texera.amber.util.VirtualIdentityUtils
+import org.apache.texera.web.resource.dashboard.user.workflow.WorkflowExecutionsResource
/** Notify the completion of a port:
* - For input port, it means the worker has finished consuming and processing all the data
diff --cc amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala
index fe34536c47,e490cde3d9..cb5d3606f3
--- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala
+++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala
@@@ -37,8 -37,8 +37,9 @@@ import org.apache.texera.amber.engine.a
}
import org.apache.texera.amber.engine.architecture.controller.{
ControllerConfig,
+ ExecutionStateUpdate,
ExecutionStatsUpdate,
+ RuntimeStatisticsPersist,
WorkerAssignmentUpdate
}
import org.apache.texera.amber.engine.architecture.rpc.controlcommands._
@@@ -598,14 -490,9 +599,12 @@@ class RegionExecutionCoordinator
region: Region,
isDependeePhase: Boolean
): Future[Seq[Unit]] = {
+ if (region.cached) {
+ return Future.value(Seq.empty)
+ }
- asyncRPCClient.sendToClient(
- ExecutionStatsUpdate(
- workflowExecution.getAllRegionExecutionsStats
- )
- )
+ val stats = workflowExecution.getAllRegionExecutionsStats
+ asyncRPCClient.sendToClient(ExecutionStatsUpdate(stats))
+ asyncRPCClient.sendToClient(RuntimeStatisticsPersist(stats))
val allStarterOperators = region.getStarterOperators
val starterOpsForThisPhase =
if (isDependeePhase) allStarterOperators.filter(_.dependeeInputs.nonEmpty)
diff --cc frontend/src/app/workspace/service/joint-ui/joint-ui.service.ts
index eda67e5973,40a4129071..ba016ff700
--- a/frontend/src/app/workspace/service/joint-ui/joint-ui.service.ts
+++ b/frontend/src/app/workspace/service/joint-ui/joint-ui.service.ts
@@@ -344,13 -322,15 +346,18 @@@ export class JointUIService
const inputMetrics = statistics.inputPortMetrics;
const outputMetrics = statistics.outputPortMetrics;
+ // Cached operators show "-" for inputs and non-materialized outputs, and label workers as "from cache".
+ const isSkippedFromCache = statistics.operatorState === OperatorState.CompletedFromCache;
const workerCount = statistics.numWorkers ?? 1;
- element.attr(`.${operatorWorkerCountClass}/text`, "#workers: " + String(workerCount));
+ const workerCountLabel = isSkippedFromCache ? "from cache" : "#workers: " + String(workerCount);
+ element.attr(`.${operatorWorkerCountClass}/text`, workerCountLabel);
+ element.attr(
+ `.${operatorStatusTextClass}/text`,
+ "status: " + JointUIService.getStatusDisplayText(statistics.operatorState)
+ );
+
inPorts.forEach(portDef => {
const portId = portDef.id;
if (portId != null) {