You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2023/08/03 04:56:24 UTC
[beam] branch nightly-refs/heads/master updated (e0b6d835cc0 -> 66dd8b3caca)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from e0b6d835cc0 Update website with DicomIO (#27791)
add 2f55b2598b5 Track pipeline options revision
add f3bdc8d9594 Safe initialization of file systems using pipeline options revision.
add 663bd529a47 Merge pull request #26694: [Java] Track pipeline options revision for idempotent initialization of file systems
add 507611f5df1 remove action (#27782)
add 22269e172fe Bump github.com/aws/aws-sdk-go-v2/service/s3 in /sdks (#27797)
add 59880fe430a Bump github.com/fsouza/fake-gcs-server from 1.47.0 to 1.47.2 in /sdks (#27794)
add f9449b4365f Log dataflow messages at appropriate levels (#27788)
add b502b3750cc Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#27796)
add 929a26d7a5e Update playground to require Go 1.20 to match SDK. (#27757)
add 9c80b490f61 [#24789] Make Prism the default Go SDK runner. (#27703)
add 1b76101c0b2 [Python] Hugging Face pipeline support (#27399)
add e03a14c7d52 Update stale_dataflow_prebuilt_image_cleaner.sh to clean containers under pushed by GH tests every 6 weeks. (#27807)
add 8ef0cc19f74 Use correct concurrency condition to avoid incorrect interrupts (#27815)
add afa4371b1dd Fix bug causing elapsed time to be negative (#27812)
add 4d4952935e4 Fix gradle command not found error for Python ValidatesContainer Dataflow ARM (#27810)
add 427e0a1c852 [#24789] Mention prism is the new default Go SDK runner in CHANGES.md (#27816)
add 13dd74cf9b7 Update REVIEWERS.yml (#27820)
add 55f4133a7aa Add Neo4j Resource manager. (#27787)
add 553ac1629d9 Add flags to build multiarch Java SDK containers and test (#27760)
add 9c2ce6c1bf2 Add Splunk Resource manager to manage Splunk resources in tests. (#27777)
add 324366f19e3 refactor dicomio_test.py (#27819)
add 4199f44064a Github Workflow Replacement for Jenkins Job, beam_PreCommit_GoPortable (Cron, Commit, Phrase) (#27482)
add 66dd8b3caca Update `it` modules. (#27817)
No new revisions were added by this update.
Summary of changes:
.github/REVIEWERS.yml | 1 -
.github/workflows/README.md | 4 +-
.../workflows/beam_PostCommit_Go_Dataflow_ARM.yml | 2 +-
.../beam_PostCommit_Java_Examples_Dataflow_ARM.yml | 143 +++++++
...aflow_ARM.yml => beam_PreCommit_GoPortable.yml} | 73 ++--
.../workflows/beam_PreCommit_Website_Stage_GCS.yml | 21 +-
...beam_Python_ValidatesContainer_Dataflow_ARM.yml | 22 +-
.../tools/stale_dataflow_prebuilt_image_cleaner.sh | 2 +-
CHANGES.md | 20 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 1 +
.../apache/beam/it/common/PipelineOperator.java | 23 +-
.../beam/it/common/logging/package-info.java | 4 +-
.../org/apache/beam/it/common/package-info.java | 4 +-
.../beam/it/common/utils/ExceptionUtils.java | 20 +
.../apache/beam/it/common/utils/package-info.java | 4 +-
.../beam/it/common/utils/ExceptionUtilsTest.java | 41 ++
.../apache/beam/it/elasticsearch/package-info.java | 22 +-
it/google-cloud-platform/build.gradle | 2 +
.../org/apache/beam/it/gcp/IOLoadTestBase.java | 2 +-
.../java/org/apache/beam/it/gcp/LoadTestBase.java | 47 ++-
.../it/gcp/artifacts/matchers/ArtifactAsserts.java | 3 +-
.../it/gcp/artifacts/matchers/package-info.java | 4 +-
.../apache/beam/it/gcp/artifacts/package-info.java | 4 +-
.../beam/it/gcp/artifacts/utils/package-info.java | 4 +-
.../it/gcp/bigquery/BigQueryResourceManager.java | 4 +-
.../gcp/bigquery/conditions/BigQueryRowsCheck.java | 6 +-
.../it/gcp/bigquery/conditions/package-info.java | 5 +-
.../it/gcp/bigquery/matchers/package-info.java | 4 +-
.../apache/beam/it/gcp/bigquery/package-info.java | 4 +-
.../beam/it/gcp/bigquery/utils/package-info.java | 4 +-
.../it/gcp/bigtable/BigtableResourceManager.java | 108 ++++-
.../it/gcp/dataflow/AbstractPipelineLauncher.java | 2 +
.../beam/it/gcp/dataflow/DirectRunnerClient.java | 2 +-
.../SecretManagerResourceManager.java | 296 ++++++++++++++
.../SecretManagerResourceManagerException.java} | 15 +-
.../package-info.java} | 10 +-
.../it/gcp/spanner/SpannerResourceManager.java | 91 ++++-
.../spanner/SpannerResourceManagerException.java | 4 +
.../spanner/utils/SpannerResourceManagerUtils.java | 28 +-
.../beam/it/gcp/storage/GcsResourceManager.java | 11 +-
.../java/org/apache/beam/it/gcp/WordCountIT.java | 2 +-
.../apache/beam/it/gcp/bigquery/BigQueryIOLT.java | 14 +-
.../apache/beam/it/gcp/bigtable/BigTableIOLT.java | 16 +-
.../SecretManagerResourceManagerTest.java | 127 ++++++
.../it/gcp/spanner/SpannerResourceManagerTest.java | 37 +-
.../utils/SpannerResourceManagerUtilsTest.java | 40 +-
.../apache/beam/it/gcp/storage/FileBasedIOLT.java | 8 +-
.../java/org/apache/beam/it/jdbc/package-info.java | 22 +-
it/kafka/build.gradle | 6 +
.../org/apache/beam/it/kafka/package-info.java | 22 +-
.../java/org/apache/beam/it/kafka/KafkaIOLT.java | 249 ++++++++++++
.../beam/it/mongodb/conditions/package-info.java | 22 +-
.../beam/it/mongodb/matchers/package-info.java | 22 +-
.../org/apache/beam/it/mongodb/package-info.java | 22 +-
it/{kafka => neo4j}/build.gradle | 13 +-
.../apache/beam/it/neo4j/Neo4jResourceManager.java | 242 ++++++++++++
.../it/neo4j/Neo4jResourceManagerException.java} | 12 +-
.../beam/it/neo4j/Neo4jResourceManagerUtils.java | 53 +++
.../beam/it/neo4j/conditions/Neo4jQueryCheck.java | 85 ++++
.../beam/it/neo4j/conditions}/package-info.java | 8 +-
.../org/apache/beam/it/neo4j}/package-info.java | 8 +-
.../beam/it/neo4j/Neo4jResourceManagerIT.java | 68 ++++
.../beam/it/neo4j/Neo4jResourceManagerTest.java | 161 ++++++++
.../it/neo4j/Neo4jResourceManagerUtilsTest.java | 52 +++
it/{kafka => splunk}/build.gradle | 26 +-
.../apache/beam/it/splunk/SplunkClientFactory.java | 47 +++
.../org/apache/beam/it/splunk/SplunkContainer.java | 123 ++++++
.../beam/it/splunk/SplunkResourceManager.java | 438 +++++++++++++++++++++
.../it/splunk/SplunkResourceManagerException.java} | 12 +-
.../beam/it/splunk/SplunkResourceManagerUtils.java | 84 ++++
.../it/splunk/conditions/SplunkEventsCheck.java | 100 +++++
.../beam/it/splunk/conditions}/package-info.java | 4 +-
.../beam/it/splunk/matchers/SplunkAsserts.java | 64 +++
.../beam/it/splunk/matchers}/package-info.java | 4 +-
.../org/apache/beam/it/splunk}/package-info.java | 4 +-
.../beam/it/splunk/SplunkResourceManagerIT.java | 99 +++++
.../beam/it/splunk/SplunkResourceManagerTest.java | 248 ++++++++++++
.../it/splunk/SplunkResourceManagerUtilsTest.java | 112 ++++++
.../beam/it/truthmatchers/RecordsSubject.java | 2 +-
playground/README.md | 2 +-
playground/backend/containers/go/build.gradle | 2 +-
playground/backend/containers/java/Dockerfile | 2 +-
playground/backend/containers/python/Dockerfile | 2 +-
playground/backend/containers/python/build.gradle | 2 +-
playground/backend/containers/router/Dockerfile | 2 +-
playground/backend/containers/router/build.gradle | 2 +-
playground/backend/containers/scio/Dockerfile | 2 +-
playground/backend/go.mod | 2 +-
.../tools/extract_symbols_go/go.mod | 2 +-
.../cloudbuild/playground_ci_examples.sh | 2 +-
.../google-cloud-dataflow-java/arm/build.gradle | 155 ++++++++
.../dataflow/worker/StreamingDataflowWorker.java | 2 +-
sdks/go.mod | 16 +-
sdks/go.sum | 32 +-
.../go/examples/large_wordcount/large_wordcount.go | 1 +
.../minimal_wordcount/minimal_wordcount.go | 6 +-
sdks/go/examples/snippets/10metrics.go | 2 +-
sdks/go/pkg/beam/runner.go | 4 -
sdks/go/pkg/beam/runners/direct/direct.go | 3 +
sdks/go/pkg/beam/runners/prism/README.md | 20 +-
sdks/go/pkg/beam/testing/ptest/ptest.go | 9 +-
sdks/go/pkg/beam/testing/ptest/ptest_test.go | 4 +
sdks/go/pkg/beam/x/beamx/run.go | 5 +-
sdks/go/test/integration/integration.go | 36 +-
sdks/java/container/common.gradle | 7 +-
.../java/org/apache/beam/sdk/io/FileSystems.java | 30 +-
.../apache/beam/sdk/options/PipelineOptions.java | 6 +
.../beam/sdk/options/PipelineOptionsFactory.java | 1 +
.../beam/sdk/options/ProxyInvocationHandler.java | 27 +-
.../sdk/options/PipelineOptionsFactoryTest.java | 12 +
.../inference/huggingface_question_answering.py | 164 ++++++++
.../apache_beam/io/gcp/healthcare/dicomio_test.py | 102 ++---
.../ml/inference/huggingface_inference.py | 200 +++++++++-
.../ml/inference/huggingface_inference_it_test.py | 35 +-
.../inference/huggingface_tests_requirements.txt | 2 +-
.../runners/dataflow/dataflow_runner.py | 14 +-
sdks/python/container/run_validatescontainer.sh | 7 +-
settings.gradle.kts | 2 +
118 files changed, 4198 insertions(+), 473 deletions(-)
create mode 100644 .github/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml
copy .github/workflows/{beam_PostCommit_Go_Dataflow_ARM.yml => beam_PreCommit_GoPortable.yml} (52%)
create mode 100644 it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/secretmanager/SecretManagerResourceManager.java
copy it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/{spanner/SpannerResourceManagerException.java => secretmanager/SecretManagerResourceManagerException.java} (65%)
copy it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/{spanner/SpannerResourceManagerException.java => secretmanager/package-info.java} (72%)
create mode 100644 it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/secretmanager/SecretManagerResourceManagerTest.java
create mode 100644 it/kafka/src/test/java/org/apache/beam/it/kafka/KafkaIOLT.java
copy it/{kafka => neo4j}/build.gradle (78%)
create mode 100644 it/neo4j/src/main/java/org/apache/beam/it/neo4j/Neo4jResourceManager.java
copy it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/SpannerResourceManagerException.java => neo4j/src/main/java/org/apache/beam/it/neo4j/Neo4jResourceManagerException.java} (70%)
create mode 100644 it/neo4j/src/main/java/org/apache/beam/it/neo4j/Neo4jResourceManagerUtils.java
create mode 100644 it/neo4j/src/main/java/org/apache/beam/it/neo4j/conditions/Neo4jQueryCheck.java
copy it/{common/src/main/java/org/apache/beam/it/common/logging => neo4j/src/main/java/org/apache/beam/it/neo4j/conditions}/package-info.java (72%)
copy it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/utils => neo4j/src/main/java/org/apache/beam/it/neo4j}/package-info.java (74%)
create mode 100644 it/neo4j/src/test/java/org/apache/beam/it/neo4j/Neo4jResourceManagerIT.java
create mode 100644 it/neo4j/src/test/java/org/apache/beam/it/neo4j/Neo4jResourceManagerTest.java
create mode 100644 it/neo4j/src/test/java/org/apache/beam/it/neo4j/Neo4jResourceManagerUtilsTest.java
copy it/{kafka => splunk}/build.gradle (64%)
create mode 100644 it/splunk/src/main/java/org/apache/beam/it/splunk/SplunkClientFactory.java
create mode 100644 it/splunk/src/main/java/org/apache/beam/it/splunk/SplunkContainer.java
create mode 100644 it/splunk/src/main/java/org/apache/beam/it/splunk/SplunkResourceManager.java
copy it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp/spanner/SpannerResourceManagerException.java => splunk/src/main/java/org/apache/beam/it/splunk/SplunkResourceManagerException.java} (70%)
create mode 100644 it/splunk/src/main/java/org/apache/beam/it/splunk/SplunkResourceManagerUtils.java
create mode 100644 it/splunk/src/main/java/org/apache/beam/it/splunk/conditions/SplunkEventsCheck.java
copy it/{jdbc/src/main/java/org/apache/beam/it/jdbc => splunk/src/main/java/org/apache/beam/it/splunk/conditions}/package-info.java (85%)
create mode 100644 it/splunk/src/main/java/org/apache/beam/it/splunk/matchers/SplunkAsserts.java
copy it/{jdbc/src/main/java/org/apache/beam/it/jdbc => splunk/src/main/java/org/apache/beam/it/splunk/matchers}/package-info.java (82%)
copy it/{jdbc/src/main/java/org/apache/beam/it/jdbc => splunk/src/main/java/org/apache/beam/it/splunk}/package-info.java (84%)
create mode 100644 it/splunk/src/test/java/org/apache/beam/it/splunk/SplunkResourceManagerIT.java
create mode 100644 it/splunk/src/test/java/org/apache/beam/it/splunk/SplunkResourceManagerTest.java
create mode 100644 it/splunk/src/test/java/org/apache/beam/it/splunk/SplunkResourceManagerUtilsTest.java
mode change 100644 => 100755 playground/infrastructure/cloudbuild/playground_ci_examples.sh
create mode 100644 runners/google-cloud-dataflow-java/arm/build.gradle
create mode 100644 sdks/python/apache_beam/examples/inference/huggingface_question_answering.py