You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@texera.apache.org by gi...@apache.org on 2026/07/04 05:32:49 UTC
(texera) 01/01: perf(local-dev): skip scaladoc and honor --skip in sbt build (#6087)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-6087-28bd6e09f22fdbc0c0439892c97c1bdd2992240c
in repository https://gitbox.apache.org/repos/asf/texera.git
commit 5786a10a738dc3bfac26b030475a7251575b7c95
Author: Yicong Huang <17...@users.noreply.github.com>
AuthorDate: Fri Jul 3 22:32:30 2026 -0700
perf(local-dev): skip scaladoc and honor --skip in sbt build (#6087)
### What changes were proposed in this PR?
Change `build_all()` in `bin/local-dev/main.sh` so its sbt invocations
(a) pass CLI-only flags that empty `Compile / doc / sources`, disable
`packageDoc / publishArtifact`, enable `-Dsbt.pipelining=true`, and
raise the sbt heap to `-J-Xmx4g -J-XX:+UseG1GC`, and (b) translate
`--skip=<svc>` into an explicit per-project `<Proj>/dist` target list
instead of the root aggregate. `computing-unit-master` has no own sbt
project (it rides amber's dist), so `WorkflowExecutionService/dist` is
force-added when master is kept but `texera-web` is skipped. The
pre-bounce loop and the unzip loop now honor `--skip` too, so skipped
services keep their prior jars. `build.sbt` is intentionally untouched —
the knobs live only in the local-dev entrypoint so CI, standalone `sbt`
runs, and IDE builds keep the current behavior.
### Any related issues, documentation, discussions?
Closes #6086.
### How was this PR tested?
Added static regression tests in
`bin/local-dev/tests/test_local_dev_sh.sh` covering the CLI flags and
the `is_skipped` guards; cold-start `sbt clean dist` wall-clock measured
locally, n=3 per configuration:
| Configuration | Mean | Individual runs |
|---|---|---|
| Baseline: `sbt -no-colors clean dist` | 91s | 91 / 87 / 94 |
| Flags applied, all 7 project dists | 58s (−36%) | 58 / 57 / 61 |
| Flags applied,
`--skip=notebook-migration-service,texera-web,computing-unit-master` (5
project dists) | 38s (−58%) | 34 / 39 / 40 |
### Was this PR authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 4.7 (1M context)
---
bin/local-dev/main.sh | 43 +++++++++++++++++++++++++++++---
bin/local-dev/tests/test_local_dev_sh.sh | 35 ++++++++++++++++++++++++++
2 files changed, 75 insertions(+), 3 deletions(-)
diff --git a/bin/local-dev/main.sh b/bin/local-dev/main.sh
index 5d8ca22169..e5e83eed6c 100755
--- a/bin/local-dev/main.sh
+++ b/bin/local-dev/main.sh
@@ -1875,9 +1875,42 @@ svc_src_changed() {
build_all() {
BUILD_DID_RUN=false
local log="$LOG_DIR/sbt-dist.log"
+
+ # Build a whitelist of sbt project/dist targets from the non-skipped JVM
+ # services. Empty ⇒ nothing to build. computing-unit-master has no own
+ # sbt project (rides amber's dist); if it's kept but texera-web is
+ # skipped we force WorkflowExecutionService/dist back in.
+ local -a sbt_task=()
+ local svc="" proj=""
+ for svc in "${SERVICES[@]}"; do
+ [[ "$(amap_get SVC_TYPE "$svc")" == "jvm" ]] || continue
+ is_skipped "$svc" && continue
+ proj=$(amap_get SVC_SBT "$svc")
+ [[ -n "$proj" ]] && sbt_task+=("${proj}/dist")
+ done
+ if ! is_skipped computing-unit-master && is_skipped texera-web; then
+ sbt_task+=("WorkflowExecutionService/dist")
+ fi
+ if (( ${#sbt_task[@]} == 0 )); then
+ tui_skip "sbt dist: skipped (no JVM services selected)"
+ return 0
+ fi
+
+ # CLI-only build knobs applied to the local-dev entrypoint (build.sbt
+ # untouched): skip scaladoc (biggest single win on dist), pipeline
+ # signature-then-body compile across projects, raise heap + G1GC.
+ local -a sbt_opts=(
+ -no-colors
+ -J-Xmx4g
+ -J-XX:+UseG1GC
+ -Dsbt.pipelining=true
+ 'set every (Compile / doc / sources) := Seq.empty'
+ 'set every (Compile / packageDoc / publishArtifact) := false'
+ )
+
if [[ "${FRESH:-false}" == "true" ]]; then
if tui_run_with_spinner "$log" "sbt clean dist ${DIM}(log: $log)${RESET}" \
- sbt -no-colors clean dist; then
+ sbt "${sbt_opts[@]}" clean "${sbt_task[@]}"; then
tui_ok "sbt: clean dist done"
BUILD_DID_RUN=true
else
@@ -1889,7 +1922,7 @@ build_all() {
return 0
else
if tui_run_with_spinner "$log" "sbt dist ${DIM}(log: $log)${RESET}" \
- sbt -no-colors dist; then
+ sbt "${sbt_opts[@]}" "${sbt_task[@]}"; then
tui_ok "sbt: dist done"
BUILD_DID_RUN=true
else
@@ -1899,10 +1932,12 @@ build_all() {
fi
# Stop any running JVMs BEFORE unzip — overwriting jars under a live JVM
# corrupts its lazy class loads and the service silently dies later.
+ # --skip'd services are left alone; the user asked us not to touch them.
if [[ "$BUILD_DID_RUN" == "true" ]]; then
- local svc="" pid=""
+ local pid=""
for svc in "${SERVICES[@]}"; do
[[ "$(amap_get SVC_TYPE "$svc")" == "jvm" ]] || continue
+ is_skipped "$svc" && continue
pid=$(svc_running_pid "$svc")
[[ -z "$pid" ]] && continue
tui_step "$svc: pre-bouncing PID $pid (jars about to change)"
@@ -1914,6 +1949,7 @@ build_all() {
local still_up=0
for svc in "${SERVICES[@]}"; do
[[ "$(amap_get SVC_TYPE "$svc")" == "jvm" ]] || continue
+ is_skipped "$svc" && continue
[[ -n "$(svc_running_pid "$svc")" ]] && still_up=$((still_up+1))
done
(( still_up == 0 )) && break
@@ -1925,6 +1961,7 @@ build_all() {
local zip_glob="" unzip_dest=""
for svc in "${SERVICES[@]}"; do
[[ "$(amap_get SVC_TYPE "$svc")" == "jvm" ]] || continue
+ is_skipped "$svc" && continue
zip_glob=$(amap_get SVC_ZIP_GLOB "$svc")
unzip_dest=$(amap_get SVC_UNZIP_DEST "$svc")
# Sibling services (empty ZIP_GLOB) share another service's dist —
diff --git a/bin/local-dev/tests/test_local_dev_sh.sh b/bin/local-dev/tests/test_local_dev_sh.sh
index 4f0a8d61a5..df633d1417 100755
--- a/bin/local-dev/tests/test_local_dev_sh.sh
+++ b/bin/local-dev/tests/test_local_dev_sh.sh
@@ -413,5 +413,40 @@ for fn in cmd_up cmd_auto; do
fi
done
+# 24) build_all applies CLI-only sbt speedups: skip scaladoc + enable
+# pipelining. Keeping the knobs here (not in build.sbt) means CI and
+# standalone sbt invocations are unaffected.
+build_body=$(awk '/^build_all\(\)/{f=1} f{print} f&&/^}/{exit}' \
+ "$REPO_ROOT/bin/local-dev/main.sh")
+if [[ "$build_body" == *"-Dsbt.pipelining=true"* ]]; then
+ _pass "build_all: sbt invocation enables -Dsbt.pipelining=true"
+else
+ _fail "build_all: -Dsbt.pipelining=true flag missing"
+fi
+if [[ "$build_body" == *"Compile / doc / sources) := Seq.empty"* ]]; then
+ _pass "build_all: scaladoc sources emptied via 'set every'"
+else
+ _fail "build_all: doc-skip 'set every' setting missing"
+fi
+
+# 25) --skip=<svc> flows into the sbt build: skipped JVM services drop out
+# of the per-project dist target list, and their running jars are left
+# alone in both the pre-bounce and unzip loops.
+prebounce=$(printf '%s\n' "$build_body" | awk '
+ /Stop any running JVMs BEFORE unzip/ {f=1}
+ f {print}
+ f && /unzipping dist artifacts/ {exit}')
+if [[ "$prebounce" == *"is_skipped"* ]]; then
+ _pass "build_all: pre-bounce loop honors --skip"
+else
+ _fail "build_all: pre-bounce loop kills --skip'd services"
+fi
+unzip_section=$(printf '%s\n' "$build_body" | awk '/unzipping dist artifacts/{f=1} f{print}')
+if [[ "$unzip_section" == *"is_skipped"* ]]; then
+ _pass "build_all: unzip loop honors --skip"
+else
+ _fail "build_all: unzip loop touches --skip'd services"
+fi
+
printf "\n%d passed, %d failed\n" "$PASS" "$FAIL"
(( FAIL == 0 ))