You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@texera.apache.org by gi...@apache.org on 2026/07/04 05:32:49 UTC

(texera) 01/01: perf(local-dev): skip scaladoc and honor --skip in sbt build (#6087)

This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-6087-28bd6e09f22fdbc0c0439892c97c1bdd2992240c
in repository https://gitbox.apache.org/repos/asf/texera.git

commit 5786a10a738dc3bfac26b030475a7251575b7c95
Author: Yicong Huang <17...@users.noreply.github.com>
AuthorDate: Fri Jul 3 22:32:30 2026 -0700

    perf(local-dev): skip scaladoc and honor --skip in sbt build (#6087)
    
    ### What changes were proposed in this PR?
    
    Change `build_all()` in `bin/local-dev/main.sh` so its sbt invocations
    (a) pass CLI-only flags that empty `Compile / doc / sources`, disable
    `packageDoc / publishArtifact`, enable `-Dsbt.pipelining=true`, and
    raise the sbt heap to `-J-Xmx4g -J-XX:+UseG1GC`, and (b) translate
    `--skip=<svc>` into an explicit per-project `<Proj>/dist` target list
    instead of the root aggregate. `computing-unit-master` has no own sbt
    project (it rides amber's dist), so `WorkflowExecutionService/dist` is
    force-added when master is kept but `texera-web` is skipped. The
    pre-bounce loop and the unzip loop now honor `--skip` too, so skipped
    services keep their prior jars. `build.sbt` is intentionally untouched —
    the knobs live only in the local-dev entrypoint so CI, standalone `sbt`
    runs, and IDE builds keep the current behavior.
    
    ### Any related issues, documentation, discussions?
    
    Closes #6086.
    
    ### How was this PR tested?
    
    Added static regression tests in
    `bin/local-dev/tests/test_local_dev_sh.sh` covering the CLI flags and
    the `is_skipped` guards; cold-start `sbt clean dist` wall-clock measured
    locally, n=3 per configuration:
    
    | Configuration | Mean | Individual runs |
    |---|---|---|
    | Baseline: `sbt -no-colors clean dist` | 91s | 91 / 87 / 94 |
    | Flags applied, all 7 project dists | 58s (−36%) | 58 / 57 / 61 |
    | Flags applied,
    `--skip=notebook-migration-service,texera-web,computing-unit-master` (5
    project dists) | 38s (−58%) | 34 / 39 / 40 |
    
    ### Was this PR authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 4.7 (1M context)
---
 bin/local-dev/main.sh                    | 43 +++++++++++++++++++++++++++++---
 bin/local-dev/tests/test_local_dev_sh.sh | 35 ++++++++++++++++++++++++++
 2 files changed, 75 insertions(+), 3 deletions(-)

diff --git a/bin/local-dev/main.sh b/bin/local-dev/main.sh
index 5d8ca22169..e5e83eed6c 100755
--- a/bin/local-dev/main.sh
+++ b/bin/local-dev/main.sh
@@ -1875,9 +1875,42 @@ svc_src_changed() {
 build_all() {
     BUILD_DID_RUN=false
     local log="$LOG_DIR/sbt-dist.log"
+
+    # Build a whitelist of sbt project/dist targets from the non-skipped JVM
+    # services. Empty ⇒ nothing to build. computing-unit-master has no own
+    # sbt project (rides amber's dist); if it's kept but texera-web is
+    # skipped we force WorkflowExecutionService/dist back in.
+    local -a sbt_task=()
+    local svc="" proj=""
+    for svc in "${SERVICES[@]}"; do
+        [[ "$(amap_get SVC_TYPE "$svc")" == "jvm" ]] || continue
+        is_skipped "$svc" && continue
+        proj=$(amap_get SVC_SBT "$svc")
+        [[ -n "$proj" ]] && sbt_task+=("${proj}/dist")
+    done
+    if ! is_skipped computing-unit-master && is_skipped texera-web; then
+        sbt_task+=("WorkflowExecutionService/dist")
+    fi
+    if (( ${#sbt_task[@]} == 0 )); then
+        tui_skip "sbt dist: skipped (no JVM services selected)"
+        return 0
+    fi
+
+    # CLI-only build knobs applied to the local-dev entrypoint (build.sbt
+    # untouched): skip scaladoc (biggest single win on dist), pipeline
+    # signature-then-body compile across projects, raise heap + G1GC.
+    local -a sbt_opts=(
+        -no-colors
+        -J-Xmx4g
+        -J-XX:+UseG1GC
+        -Dsbt.pipelining=true
+        'set every (Compile / doc / sources) := Seq.empty'
+        'set every (Compile / packageDoc / publishArtifact) := false'
+    )
+
     if [[ "${FRESH:-false}" == "true" ]]; then
         if tui_run_with_spinner "$log" "sbt clean dist  ${DIM}(log: $log)${RESET}" \
-            sbt -no-colors clean dist; then
+            sbt "${sbt_opts[@]}" clean "${sbt_task[@]}"; then
             tui_ok "sbt: clean dist done"
             BUILD_DID_RUN=true
         else
@@ -1889,7 +1922,7 @@ build_all() {
         return 0
     else
         if tui_run_with_spinner "$log" "sbt dist  ${DIM}(log: $log)${RESET}" \
-            sbt -no-colors dist; then
+            sbt "${sbt_opts[@]}" "${sbt_task[@]}"; then
             tui_ok "sbt: dist done"
             BUILD_DID_RUN=true
         else
@@ -1899,10 +1932,12 @@ build_all() {
     fi
     # Stop any running JVMs BEFORE unzip — overwriting jars under a live JVM
     # corrupts its lazy class loads and the service silently dies later.
+    # --skip'd services are left alone; the user asked us not to touch them.
     if [[ "$BUILD_DID_RUN" == "true" ]]; then
-        local svc="" pid=""
+        local pid=""
         for svc in "${SERVICES[@]}"; do
             [[ "$(amap_get SVC_TYPE "$svc")" == "jvm" ]] || continue
+            is_skipped "$svc" && continue
             pid=$(svc_running_pid "$svc")
             [[ -z "$pid" ]] && continue
             tui_step "$svc: pre-bouncing PID $pid (jars about to change)"
@@ -1914,6 +1949,7 @@ build_all() {
             local still_up=0
             for svc in "${SERVICES[@]}"; do
                 [[ "$(amap_get SVC_TYPE "$svc")" == "jvm" ]] || continue
+                is_skipped "$svc" && continue
                 [[ -n "$(svc_running_pid "$svc")" ]] && still_up=$((still_up+1))
             done
             (( still_up == 0 )) && break
@@ -1925,6 +1961,7 @@ build_all() {
     local zip_glob="" unzip_dest=""
     for svc in "${SERVICES[@]}"; do
         [[ "$(amap_get SVC_TYPE "$svc")" == "jvm" ]] || continue
+        is_skipped "$svc" && continue
         zip_glob=$(amap_get SVC_ZIP_GLOB "$svc")
         unzip_dest=$(amap_get SVC_UNZIP_DEST "$svc")
         # Sibling services (empty ZIP_GLOB) share another service's dist —
diff --git a/bin/local-dev/tests/test_local_dev_sh.sh b/bin/local-dev/tests/test_local_dev_sh.sh
index 4f0a8d61a5..df633d1417 100755
--- a/bin/local-dev/tests/test_local_dev_sh.sh
+++ b/bin/local-dev/tests/test_local_dev_sh.sh
@@ -413,5 +413,40 @@ for fn in cmd_up cmd_auto; do
     fi
 done
 
+# 24) build_all applies CLI-only sbt speedups: skip scaladoc + enable
+#     pipelining. Keeping the knobs here (not in build.sbt) means CI and
+#     standalone sbt invocations are unaffected.
+build_body=$(awk '/^build_all\(\)/{f=1} f{print} f&&/^}/{exit}' \
+    "$REPO_ROOT/bin/local-dev/main.sh")
+if [[ "$build_body" == *"-Dsbt.pipelining=true"* ]]; then
+    _pass "build_all: sbt invocation enables -Dsbt.pipelining=true"
+else
+    _fail "build_all: -Dsbt.pipelining=true flag missing"
+fi
+if [[ "$build_body" == *"Compile / doc / sources) := Seq.empty"* ]]; then
+    _pass "build_all: scaladoc sources emptied via 'set every'"
+else
+    _fail "build_all: doc-skip 'set every' setting missing"
+fi
+
+# 25) --skip=<svc> flows into the sbt build: skipped JVM services drop out
+#     of the per-project dist target list, and their running jars are left
+#     alone in both the pre-bounce and unzip loops.
+prebounce=$(printf '%s\n' "$build_body" | awk '
+    /Stop any running JVMs BEFORE unzip/ {f=1}
+    f {print}
+    f && /unzipping dist artifacts/ {exit}')
+if [[ "$prebounce" == *"is_skipped"* ]]; then
+    _pass "build_all: pre-bounce loop honors --skip"
+else
+    _fail "build_all: pre-bounce loop kills --skip'd services"
+fi
+unzip_section=$(printf '%s\n' "$build_body" | awk '/unzipping dist artifacts/{f=1} f{print}')
+if [[ "$unzip_section" == *"is_skipped"* ]]; then
+    _pass "build_all: unzip loop honors --skip"
+else
+    _fail "build_all: unzip loop touches --skip'd services"
+fi
+
 printf "\n%d passed, %d failed\n" "$PASS" "$FAIL"
 (( FAIL == 0 ))