You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by lo...@apache.org on 2024/10/24 21:36:44 UTC

(beam) branch master updated: [#28187] Add a Java gradle task to run validates runner tests on Prism. (#32919)

This is an automated email from the ASF dual-hosted git repository.

lostluck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new 5a6b10a57b5 [#28187] Add a Java gradle task to run validates runner tests on Prism. (#32919)
5a6b10a57b5 is described below

commit 5a6b10a57b5d0142e3446e1a93f0af1bc0222d8b
Author: Robert Burke <lo...@users.noreply.github.com>
AuthorDate: Thu Oct 24 14:36:37 2024 -0700

    [#28187] Add a Java gradle task to run validates runner tests on Prism. (#32919)
---
 .../beam_PreCommit_Java_PVR_Prism_Loopback.yml     | 114 ++++++++++
 runners/prism/build.gradle                         |   3 +
 runners/prism/java/build.gradle                    | 241 +++++++++++++++++++++
 .../apache/beam/runners/prism/PrismExecutor.java   |   1 +
 .../apache/beam/runners/prism/PrismLocator.java    |   6 +
 .../beam/runners/prism/PrismPipelineOptions.java   |   6 +
 .../org/apache/beam/runners/prism/PrismRunner.java |   6 +-
 .../beam/runners/prism/PrismLocatorTest.java       |   5 +-
 sdks/go/cmd/prism/prism.go                         |  19 +-
 9 files changed, 393 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/beam_PreCommit_Java_PVR_Prism_Loopback.yml b/.github/workflows/beam_PreCommit_Java_PVR_Prism_Loopback.yml
new file mode 100644
index 00000000000..ea5cf9b5578
--- /dev/null
+++ b/.github/workflows/beam_PreCommit_Java_PVR_Prism_Loopback.yml
@@ -0,0 +1,114 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: PreCommit Java PVR Prism Loopback
+
+on:
+  push:
+    tags: ['v*']
+    branches: ['master', 'release-*']
+    paths:
+      - 'model/**'
+      - 'sdks/go/pkg/beam/runners/prism/**'
+      - 'sdks/go/cmd/prism/**'
+      - 'runners/prism/**'
+      - 'runners/java-fn-execution/**'
+      - 'sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/**'
+      - '.github/workflows/beam_PreCommit_Java_PVR_Prism_Loopback.yml'
+  pull_request_target:
+    branches: ['master', 'release-*']
+    paths:
+      - 'model/**'
+      - 'sdks/go/pkg/beam/runners/prism/**'
+      - 'sdks/go/cmd/prism/**'
+      - 'runners/prism/**'
+      - 'runners/java-fn-execution/**'
+      - 'sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/**'
+      - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_PVR_Prism_Loopback.json'
+  issue_comment:
+    types: [created]
+  schedule:
+    - cron: '22 2/6 * * *'
+  workflow_dispatch:
+
+# This allows a subsequently queued workflow run to interrupt previous runs
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}'
+  cancel-in-progress: true
+
+#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
+permissions:
+  actions: write
+  pull-requests: write
+  checks: write
+  contents: read
+  deployments: read
+  id-token: none
+  issues: write
+  discussions: read
+  packages: read
+  pages: read
+  repository-projects: read
+  security-events: read
+  statuses: read
+
+env:
+  DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+  GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
+  GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
+
+jobs:
+  beam_PreCommit_Java_PVR_Prism_Loopback:
+    name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
+    strategy:
+      matrix:
+        job_name: ["beam_PreCommit_Java_PVR_Prism_Loopback"]
+        job_phrase: ["Run Java_PVR_Prism_Loopback PreCommit"]
+    timeout-minutes: 240
+    runs-on: [self-hosted, ubuntu-20.04]
+    if: |
+      github.event_name == 'push' ||
+      github.event_name == 'pull_request_target' ||
+      (github.event_name == 'schedule' && github.repository == 'apache/beam') ||
+      github.event_name == 'workflow_dispatch' ||
+      github.event.comment.body == 'Run Java_PVR_Prism_Loopback PreCommit'
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup repository
+        uses: ./.github/actions/setup-action
+        with:
+          comment_phrase: ${{ matrix.job_phrase }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
+      - name: Setup environment
+        uses: ./.github/actions/setup-environment-action
+      - name: run prismLoopbackValidatesRunnerTests script
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :runners:prism:java:prismLoopbackValidatesRunnerTests
+      - name: Archive JUnit Test Results
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
+        with:
+          name: JUnit Test Results
+          path: "**/build/reports/tests/"
+      - name: Upload test report
+        uses: actions/upload-artifact@v4
+        with:
+          name: java-code-coverage-report
+          path: "**/build/test-results/**/*.xml"
diff --git a/runners/prism/build.gradle b/runners/prism/build.gradle
index 711a1aa2dd7..1009b9856e7 100644
--- a/runners/prism/build.gradle
+++ b/runners/prism/build.gradle
@@ -42,6 +42,9 @@ ext.set('buildTarget', buildTarget)
 def buildTask = tasks.named("build") {
     // goPrepare is a task registered in applyGoNature.
     dependsOn("goPrepare")
+    // Allow Go to manage the caching, not gradle.
+    outputs.cacheIf { false }
+    outputs.upToDateWhen { false }
     doLast {
         exec {
             workingDir = modDir
diff --git a/runners/prism/java/build.gradle b/runners/prism/java/build.gradle
index de9a30ad818..f6655900f62 100644
--- a/runners/prism/java/build.gradle
+++ b/runners/prism/java/build.gradle
@@ -16,6 +16,8 @@
  * limitations under the License.
  */
 
+import groovy.json.JsonOutput
+
 plugins { id 'org.apache.beam.module' }
 
 applyJavaNature(
@@ -43,3 +45,242 @@ tasks.test {
     var prismBuildTask = dependsOn(':runners:prism:build')
     systemProperty 'prism.buildTarget', prismBuildTask.project.property('buildTarget').toString()
 }
+
+// Below is configuration to support running the Java Validates Runner tests.
+
+configurations {
+  validatesRunner
+}
+
+dependencies {
+  implementation project(path: ":sdks:java:core", configuration: "shadow")
+  implementation library.java.hamcrest
+  permitUnusedDeclared library.java.hamcrest
+  implementation library.java.joda_time
+  implementation library.java.slf4j_api
+  implementation library.java.vendored_guava_32_1_2_jre
+
+  testImplementation library.java.hamcrest
+  testImplementation library.java.junit
+  testImplementation library.java.mockito_core
+  testImplementation library.java.slf4j_jdk14
+
+  validatesRunner project(path: ":sdks:java:core", configuration: "shadowTest")
+  validatesRunner project(path: ":runners:core-java", configuration: "testRuntimeMigration")
+  validatesRunner project(path: project.path, configuration: "testRuntimeMigration")
+}
+
+project.evaluationDependsOn(":sdks:java:core")
+project.evaluationDependsOn(":runners:core-java")
+
+def sickbayTests = [
+    // PortableMetrics doesn't implement "getCommitedOrNull" from Metrics
+    // Preventing Prism from passing these tests.
+    // In particular, it doesn't subclass MetricResult with an override, and
+    // it explicilty passes "false" to commited supported in create.
+    //
+    // There is not currently a category for excluding these _only_ in committed mode
+    'org.apache.beam.sdk.metrics.MetricsTest$CommittedMetricTests.testAllCommittedMetrics',
+    'org.apache.beam.sdk.metrics.MetricsTest$CommittedMetricTests.testCommittedCounterMetrics',
+    'org.apache.beam.sdk.metrics.MetricsTest$CommittedMetricTests.testCommittedDistributionMetrics',
+    'org.apache.beam.sdk.metrics.MetricsTest$CommittedMetricTests.testCommittedStringSetMetrics',
+    'org.apache.beam.sdk.metrics.MetricsTest$CommittedMetricTests.testCommittedGaugeMetrics',
+
+    // Triggers / Accumulation modes not yet implemented in prism.
+    // https://github.com/apache/beam/issues/31438
+    'org.apache.beam.sdk.transforms.CombineTest$WindowingTests.testGlobalCombineWithDefaultsAndTriggers',
+    'org.apache.beam.sdk.transforms.CombineTest$BasicTests.testHotKeyCombiningWithAccumulationMode',
+    'org.apache.beam.sdk.transforms.windowing.WindowTest.testNoWindowFnDoesNotReassignWindows',
+    'org.apache.beam.sdk.transforms.GroupByKeyTest$BasicTests.testAfterProcessingTimeContinuationTriggerUsingState',
+    'org.apache.beam.sdk.transforms.GroupByKeyTest$BasicTests.testCombiningAccumulatingProcessingTime',
+    'org.apache.beam.sdk.transforms.GroupByKeyTest$BasicTests.testAfterProcessingTimeContinuationTriggerEarly',
+    'org.apache.beam.sdk.transforms.ParDoTest$BundleInvariantsTests.testWatermarkUpdateMidBundle',
+    'org.apache.beam.sdk.transforms.ViewTest.testTriggeredLatestSingleton',
+    // Requires Allowed Lateness, among others.
+    'org.apache.beam.sdk.transforms.ParDoTest$TimerTests.testEventTimeTimerSetWithinAllowedLateness',
+    'org.apache.beam.sdk.testing.TestStreamTest.testFirstElementLate',
+    'org.apache.beam.sdk.testing.TestStreamTest.testDiscardingMode',
+    'org.apache.beam.sdk.testing.TestStreamTest.testEarlyPanesOfWindow',
+    'org.apache.beam.sdk.testing.TestStreamTest.testElementsAtAlmostPositiveInfinity',
+    'org.apache.beam.sdk.testing.TestStreamTest.testLateDataAccumulating',
+    'org.apache.beam.sdk.testing.TestStreamTest.testMultipleStreams',
+    'org.apache.beam.sdk.testing.TestStreamTest.testProcessingTimeTrigger',
+
+    // Coding error somehow: short write: reached end of stream after reading 5 bytes; 98 bytes expected
+    'org.apache.beam.sdk.testing.TestStreamTest.testMultiStage',
+
+    // Prism not firing sessions correctly (seems to be merging inapppropriately)
+    'org.apache.beam.sdk.transforms.CombineTest$WindowingTests.testSessionsCombine',
+    'org.apache.beam.sdk.transforms.CombineTest$WindowingTests.testSessionsCombineWithContext',
+
+    // Java side dying during execution.
+    // https://github.com/apache/beam/issues/32930
+    'org.apache.beam.sdk.transforms.FlattenTest.testFlattenMultipleCoders',
+    // Stream corruption error java side: failed:java.io.StreamCorruptedException: invalid stream header: 206E6F74
+    // Likely due to prism't coder changes.
+    'org.apache.beam.sdk.transforms.FlattenTest.testFlattenWithDifferentInputAndOutputCoders2',
+
+    // java.lang.IllegalStateException: Output with tag Tag<output> must have a schema in order to call getRowReceiver
+    // Ultimately because getRoeReceiver code path SDK side isn't friendly to LengthPrefix wrapping of row coders.
+    // https://github.com/apache/beam/issues/32931
+    'org.apache.beam.sdk.transforms.ParDoSchemaTest.testReadAndWrite',
+    'org.apache.beam.sdk.transforms.ParDoSchemaTest.testReadAndWriteMultiOutput',
+    'org.apache.beam.sdk.transforms.ParDoSchemaTest.testReadAndWriteWithSchemaRegistry',
+
+    // Technically these tests "succeed"
+    // the test is just complaining that an AssertionException isn't a RuntimeException
+    //
+    // java.lang.RuntimeException: test error in finalize
+    'org.apache.beam.sdk.transforms.ParDoTest$LifecycleTests.testParDoWithErrorInFinishBatch',
+    // java.lang.RuntimeException: test error in process
+    'org.apache.beam.sdk.transforms.ParDoTest$LifecycleTests.testParDoWithErrorInProcessElement',
+    // java.lang.RuntimeException: test error in initialize
+    'org.apache.beam.sdk.transforms.ParDoTest$LifecycleTests.testParDoWithErrorInStartBatch',
+
+    // Only known window fns supported, not general window merging
+    // Custom window fns not yet implemented in prism.
+    // https://github.com/apache/beam/issues/31921
+    'org.apache.beam.sdk.transforms.windowing.WindowTest.testMergingCustomWindows',
+    'org.apache.beam.sdk.transforms.windowing.WindowTest.testMergingCustomWindowsKeyedCollection',
+    'org.apache.beam.sdk.transforms.windowing.WindowTest.testMergingCustomWindowsWithoutCustomWindowTypes',
+    'org.apache.beam.sdk.transforms.windowing.WindowingTest.testMergingWindowing',
+    'org.apache.beam.sdk.transforms.windowing.WindowingTest.testNonPartitioningWindowing',
+    'org.apache.beam.sdk.transforms.GroupByKeyTest$WindowTests.testGroupByKeyMergingWindows',
+
+    // Possibly a different error being hidden behind the main error.
+    // org.apache.beam.sdk.util.WindowedValue$ValueInGlobalWindow cannot be cast to class java.lang.String
+    // TODO(https://github.com/apache/beam/issues/29973)
+    'org.apache.beam.sdk.transforms.ReshuffleTest.testReshufflePreservesMetadata',
+    // TODO(https://github.com/apache/beam/issues/31231)
+    'org.apache.beam.sdk.transforms.RedistributeTest.testRedistributePreservesMetadata',
+
+    // Prism isn't handling Java's side input views properly.
+    // https://github.com/apache/beam/issues/32932
+    // java.lang.IllegalArgumentException: PCollection with more than one element accessed as a singleton view.
+    // Consider using Combine.globally().asSingleton() to combine the PCollection into a single value
+    'org.apache.beam.sdk.transforms.ViewTest.testDiscardingNonSingletonSideInput',
+    // java.util.NoSuchElementException: Empty PCollection accessed as a singleton view.
+    'org.apache.beam.sdk.transforms.ViewTest.testDiscardingNonSingletonSideInput',
+    // ava.lang.IllegalArgumentException: Duplicate values for a
+    'org.apache.beam.sdk.transforms.ViewTest.testMapSideInputWithNullValuesCatchesDuplicates',
+    // java.lang.IllegalArgumentException: PCollection with more than one element accessed as a singleton view....
+    'org.apache.beam.sdk.transforms.ViewTest.testNonSingletonSideInput',
+    // java.util.NoSuchElementException: Empty PCollection accessed as a singleton view.
+    'org.apache.beam.sdk.transforms.ViewTest.testEmptySingletonSideInput',
+    // Prism side encoding error.
+    // java.lang.IllegalStateException: java.io.EOFException
+    'org.apache.beam.sdk.transforms.ViewTest.testSideInputWithNestedIterables',
+
+    // Requires Time Sorted Input
+    'org.apache.beam.sdk.transforms.ParDoTest$StateTests.testRequiresTimeSortedInput',
+    'org.apache.beam.sdk.transforms.ParDoTest$StateTests.testRequiresTimeSortedInputWithTestStream',
+    'org.apache.beam.sdk.transforms.ParDoTest$StateTests.testRequiresTimeSortedInputWithLateDataAndAllowedLateness',
+    'org.apache.beam.sdk.transforms.ParDoTest$StateTests.testTwoRequiresTimeSortedInputWithLateData',
+    'org.apache.beam.sdk.transforms.ParDoTest$StateTests.testRequiresTimeSortedInputWithLateData',
+
+    // Timer race condition/ordering issue in Prism.
+    'org.apache.beam.sdk.transforms.ParDoTest$TimerTests.testTwoTimersSettingEachOtherWithCreateAsInputUnbounded',
+
+    // Missing output due to timer skew.
+    'org.apache.beam.sdk.transforms.ParDoTest$TimestampTests.testProcessElementSkew',
+
+    // TestStream + BundleFinalization.
+    // Tests seem to assume individual element bundles from test stream, but prism will aggregate them, preventing
+    // a subsequent firing. Tests ultimately hang until timeout.
+    // Either a test problem, or a misunderstanding of how test stream must work problem in prism.
+    // Biased to test problem, due to how they are constructed.
+    'org.apache.beam.sdk.transforms.ParDoTest$BundleFinalizationTests.testBundleFinalization',
+    'org.apache.beam.sdk.transforms.ParDoTest$BundleFinalizationTests.testBundleFinalizationWithSideInputs',
+
+    // Filtered by PortableRunner tests.
+    // Teardown not called in exceptions
+    // https://github.com/apache/beam/issues/20372
+    'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInFinishBundle',
+    'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInFinishBundleStateful',
+    'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInProcessElement',
+    'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInProcessElementStateful',
+    'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInSetup',
+    'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInSetupStateful',
+    'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInStartBundle',
+    'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInStartBundleStateful',
+]
+
+/**
+ * Runs Java ValidatesRunner tests against the Prism Runner
+ * with the specified environment type.
+ */
+def createPrismValidatesRunnerTask = { name, environmentType ->
+  Task vrTask = tasks.create(name: name, type: Test, group: "Verification") {
+    description "PrismRunner Java $environmentType ValidatesRunner suite"
+    classpath = configurations.validatesRunner
+
+    var prismBuildTask = dependsOn(':runners:prism:build')
+    systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
+        "--runner=TestPrismRunner",
+        "--experiments=beam_fn_api",
+        "--defaultEnvironmentType=${environmentType}",
+        "--prismLogLevel=warn",
+        "--prismLocation=${prismBuildTask.project.property('buildTarget').toString()}",
+        "--enableWebUI=false",
+    ])
+    testClassesDirs = files(project(":sdks:java:core").sourceSets.test.output.classesDirs)
+    useJUnit {
+      includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner'
+      // Should be run only in a properly configured SDK harness environment
+      excludeCategories 'org.apache.beam.sdk.testing.UsesExternalService'
+      excludeCategories 'org.apache.beam.sdk.testing.UsesSdkHarnessEnvironment'
+
+      // Not yet implemented in Prism
+      // https://github.com/apache/beam/issues/32211
+      excludeCategories 'org.apache.beam.sdk.testing.UsesOnWindowExpiration'
+      // https://github.com/apache/beam/issues/32929
+      excludeCategories 'org.apache.beam.sdk.testing.UsesOrderedListState'
+
+       // Not supported in Portable Java SDK yet.
+       // https://github.com/apache/beam/issues?q=is%3Aissue+is%3Aopen+MultimapState
+       excludeCategories 'org.apache.beam.sdk.testing.UsesMultimapState'
+    }
+    filter {
+      // Hangs forever with prism. Put here instead of sickbay to allow sickbay runs to terminate.
+      // https://github.com/apache/beam/issues/32222
+      excludeTestsMatching 'org.apache.beam.sdk.transforms.ParDoTest$TimerTests.testEventTimeTimerOrderingWithCreate'
+
+      for (String test : sickbayTests) {
+          excludeTestsMatching test
+      }
+    }
+  }
+  return vrTask
+}
+
+tasks.register("validatesRunnerSickbay", Test) {
+    group = "Verification"
+    description "Validates Prism local runner (Sickbay Tests)"
+
+    var prismBuildTask = dependsOn(':runners:prism:build')
+    systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
+        "--runner=TestPrismRunner",
+        "--experiments=beam_fn_api",
+        "--enableWebUI=false",
+        "--prismLogLevel=warn",
+        "--prismLocation=${prismBuildTask.project.property('buildTarget').toString()}"
+    ])
+
+  classpath = configurations.validatesRunner
+  testClassesDirs = files(project(":sdks:java:core").sourceSets.test.output.classesDirs)
+
+  filter {
+    for (String test : sickbayTests) {
+      includeTestsMatching test
+    }
+  }
+}
+
+task prismDockerValidatesRunner {
+  Task vrTask = createPrismValidatesRunnerTask("prismDockerValidatesRunnerTests", "DOCKER")
+  vrTask.dependsOn ":sdks:java:container:java8:docker"
+}
+
+task prismLoopbackValidatesRunner {
+  dependsOn createPrismValidatesRunnerTask("prismLoopbackValidatesRunnerTests", "LOOPBACK")
+}
diff --git a/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismExecutor.java b/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismExecutor.java
index 0f9816337f9..111d937fcbf 100644
--- a/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismExecutor.java
+++ b/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismExecutor.java
@@ -50,6 +50,7 @@ abstract class PrismExecutor {
   static final String IDLE_SHUTDOWN_TIMEOUT = "-idle_shutdown_timeout=%s";
   static final String JOB_PORT_FLAG_TEMPLATE = "-job_port=%s";
   static final String SERVE_HTTP_FLAG_TEMPLATE = "-serve_http=%s";
+  static final String LOG_LEVEL_FLAG_TEMPLATE = "-log_level=%s";
 
   protected @MonotonicNonNull Process process;
   protected ExecutorService executorService = Executors.newSingleThreadExecutor();
diff --git a/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismLocator.java b/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismLocator.java
index 27aea3f64df..b32f03e78e6 100644
--- a/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismLocator.java
+++ b/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismLocator.java
@@ -110,6 +110,12 @@ class PrismLocator {
   String resolve() throws IOException {
     String from = resolveSource();
 
+    // If the location is set, and it's not an http request or a zip,
+    // use the binary directly.
+    if (!from.startsWith("http") && !from.endsWith("zip") && Files.exists(Paths.get(from))) {
+      return from;
+    }
+
     String fromFileName = getNameWithoutExtension(from);
     Path to = Paths.get(userHome(), PRISM_BIN_PATH, fromFileName);
 
diff --git a/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismPipelineOptions.java b/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismPipelineOptions.java
index 9b280d0a70d..ceec1ad8268 100644
--- a/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismPipelineOptions.java
+++ b/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismPipelineOptions.java
@@ -59,4 +59,10 @@ public interface PrismPipelineOptions extends PortablePipelineOptions {
   String getIdleShutdownTimeout();
 
   void setIdleShutdownTimeout(String idleShutdownTimeout);
+
+  @Description("Sets the log level for Prism. Can be set to 'debug', 'info', 'warn', or 'error'.")
+  @Default.String("warn")
+  String getPrismLogLevel();
+
+  void setPrismLogLevel(String prismLogLevel);
 }
diff --git a/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismRunner.java b/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismRunner.java
index 6099db4b63e..ac1e68237fa 100644
--- a/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismRunner.java
+++ b/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismRunner.java
@@ -101,13 +101,17 @@ public class PrismRunner extends PipelineRunner<PipelineResult> {
     String idleShutdownTimeoutFlag =
         String.format(
             PrismExecutor.IDLE_SHUTDOWN_TIMEOUT, prismPipelineOptions.getIdleShutdownTimeout());
+    String logLevelFlag =
+        String.format(
+            PrismExecutor.LOG_LEVEL_FLAG_TEMPLATE, prismPipelineOptions.getPrismLogLevel());
     String endpoint = "localhost:" + port;
     prismPipelineOptions.setJobEndpoint(endpoint);
     String command = locator.resolve();
     PrismExecutor executor =
         PrismExecutor.builder()
             .setCommand(command)
-            .setArguments(Arrays.asList(portFlag, serveHttpFlag, idleShutdownTimeoutFlag))
+            .setArguments(
+                Arrays.asList(portFlag, serveHttpFlag, idleShutdownTimeoutFlag, logLevelFlag))
             .build();
     executor.execute();
     checkState(executor.isAlive());
diff --git a/runners/prism/java/src/test/java/org/apache/beam/runners/prism/PrismLocatorTest.java b/runners/prism/java/src/test/java/org/apache/beam/runners/prism/PrismLocatorTest.java
index 095d3c9bde6..fa5ba6d3720 100644
--- a/runners/prism/java/src/test/java/org/apache/beam/runners/prism/PrismLocatorTest.java
+++ b/runners/prism/java/src/test/java/org/apache/beam/runners/prism/PrismLocatorTest.java
@@ -134,7 +134,10 @@ public class PrismLocatorTest {
     PrismLocator underTest = new PrismLocator(options);
     String got = underTest.resolve();
 
-    assertThat(got).contains(DESTINATION_DIRECTORY.toString());
+    // Local file overrides should use the local binary in place, not copy
+    // to the cache. Doing so prevents using a locally built version.
+    assertThat(got).doesNotContain(DESTINATION_DIRECTORY.toString());
+    assertThat(got).contains(options.getPrismLocation());
     Path gotPath = Paths.get(got);
     assertThat(Files.exists(gotPath)).isTrue();
   }
diff --git a/sdks/go/cmd/prism/prism.go b/sdks/go/cmd/prism/prism.go
index 070d2f023b7..5e3f42a9e5a 100644
--- a/sdks/go/cmd/prism/prism.go
+++ b/sdks/go/cmd/prism/prism.go
@@ -44,10 +44,10 @@ var (
 
 // Logging flags
 var (
-	debug = flag.Bool("debug", false,
-		"Enables full verbosity debug logging from the runner by default. Used to build SDKs or debug Prism itself.")
 	logKind = flag.String("log_kind", "dev",
 		"Determines the format of prism's logging to std err: valid values are `dev', 'json', or 'text'. Default is `dev`.")
+	logLevelFlag = flag.String("log_level", "info",
+		"Sets the minimum log level of Prism. Valid options are 'debug', 'info','warn', and 'error'. Default is 'info'. Debug adds prism source lines.")
 )
 
 var logLevel = new(slog.LevelVar)
@@ -59,13 +59,20 @@ func main() {
 	var logHandler slog.Handler
 	loggerOutput := os.Stderr
 	handlerOpts := &slog.HandlerOptions{
-		Level:     logLevel,
-		AddSource: *debug,
+		Level: logLevel,
 	}
-	if *debug {
+	switch strings.ToLower(*logLevelFlag) {
+	case "debug":
 		logLevel.Set(slog.LevelDebug)
-		// Print the Prism source line for a log in debug mode.
 		handlerOpts.AddSource = true
+	case "info":
+		logLevel.Set(slog.LevelInfo)
+	case "warn":
+		logLevel.Set(slog.LevelWarn)
+	case "error":
+		logLevel.Set(slog.LevelError)
+	default:
+		log.Fatalf("Invalid value for log_level: %v, must be 'debug', 'info', 'warn', or 'error'", *logKind)
 	}
 	switch strings.ToLower(*logKind) {
 	case "dev":