You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by da...@apache.org on 2024/03/08 20:32:56 UTC
(beam) branch users/damccorm/yaml_pipeline updated (8d71007d7c8 -> 6a32514fc51)
This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a change to branch users/damccorm/yaml_pipeline
in repository https://gitbox.apache.org/repos/asf/beam.git
from 8d71007d7c8 Prefer yaml_pipeline in error messages
add 1c7fb4bc8a0 Fix website precommit (#30540)
add 2a10971fb6b Revert "Add test code to overwrite SQL in Beam Python JDBC (#30417)" (#30535)
add f0f1cd37dd1 Bigtable: use InstanceBuilder to dynamic load override class (#30542)
add 8d7adba289b Fix hdfs integration test (#30458)
add 314bd722eea [Python] Vertex AI Feature Store enrichment handler (#30388)
add 1a05f39883f fix playground backend container builds (#30497)
add 363731ebcb8 Add KafkaIO Stress test (#30467)
add ae9cac3c867 Reduce nullness checks in flink adapters.
add d827f7713e2 spotless again
add 77f9e3a85ee Better preconditions.
add 45d5feaad11 Merge pull request #30488 Reduce nullness checks in flink adapters.
add e9bcc37556d Moving to 2.56.0-SNAPSHOT on master branch.
add 827e96d671a Add vertex AI dependency (#30553)
add 1c55117fc33 [BEAM-30531] Automatically execute unbounded pipelines in streaming mode. (#30533)
add 96eabb0a5f5 Add local Run Inference with Gemma notebook (#30532)
add 32f38b92c36 Fixes an upgrade imcompatiblity of BQ read/write transforms
add 2f3fac6e140 Merge pull request #30551: Fixes an upgrade imcompatiblity of BQ read/write transforms
add 9849fcc42c8 Fix internal test failure from PR-30455 (#30546)
add d75210eabe0 Bump InfluxDB PostCommit timeout (#30561)
add 7497495bcf7 [Python] Log dependencies installed in submission environment (#28564)
add 9b71fadd57b Update image name to the one used by the test. (#30566)
add a18c6088457 Do not pre-install beam in tensorrt container (#30552)
add 5342a960af2 Attempt to install staged Beam SDK if one is available, before checking if Beam is installed. (#30565)
add f7fdfa7b348 Duet AI prompts: containers, hints, external calls (no links) (#30435)
add 3f8af4b61b2 Resolve Any environment types before optimization. (#30568)
add afae0215f8f Try fix Dataflow PreCommit on release branch (#30574)
add a391198b5a6 [RRIO]: Add RequestResponseIO examples and documentation on website for the Java SDK (#30430)
add 6a32514fc51 Merge branch 'master' of https://github.com/apache/beam into users/damccorm/yaml_pipeline
No new revisions were added by this update.
Summary of changes:
.asf.yaml | 1 +
.github/trigger_files/beam_PostCommit_Python.json | 0
.../beam_PostCommit_Java_InfluxDbIO_IT.yml | 2 +-
CHANGES.md | 2 +
build.gradle.kts | 3 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 1 +
examples/java/webapis/build.gradle | 51 +
.../beam/examples/webapis/AdditionalSnippets.java | 50 +
.../beam/examples/webapis/GeminiAIClient.java | 126 +++
.../beam/examples/webapis/GeminiAIExample.java | 183 ++++
.../beam/examples/webapis/GeminiAIOptions.java | 34 +
.../webapis/GenerateContentRequestCoder.java | 48 +
.../webapis/GenerateContentResponseCoder.java | 44 +
.../beam/examples/webapis/HttpImageClient.java | 111 ++
.../apache/beam/examples/webapis/ImageRequest.java | 78 ++
.../beam/examples/webapis/ImageRequestCoder.java | 48 +
.../beam/examples/webapis/ImageResponse.java | 53 +
.../beam/examples/webapis/ImageResponseCoder.java | 56 +
.../org/apache/beam/examples/webapis/Images.java | 96 ++
.../java/org/apache/beam/examples/webapis/Log.java | 56 +
.../examples/webapis/UsingHttpClientExample.java | 68 ++
.../beam/examples/webapis/GeminiAIExampleTest.java | 52 +
.../webapis/UsingHttpClientExampleTest.java | 46 +
.../notebooks/beam-ml/run_inference_gemma.ipynb | 1132 ++++++++++++++++++++
gradle.properties | 4 +-
it/google-cloud-platform/build.gradle | 3 +-
it/kafka/build.gradle | 6 +
.../java/org/apache/beam/it/kafka/KafkaIOST.java | 480 +++++++++
learning/prompts/code-explanation/01_io_kafka.md | 9 +-
learning/prompts/code-explanation/02_io_pubsub.md | 9 +-
.../prompts/code-explanation/03_io_bigquery.md | 9 +-
.../prompts/code-explanation/04_io_bigtable.md | 8 +-
learning/prompts/code-explanation/05_io_spanner.md | 12 +-
.../prompts/code-explanation/06_io_tfrecord.md | 8 +-
learning/prompts/code-explanation/07_io_json.md | 6 +-
learning/prompts/code-explanation/08_io_csv.md | 5 +-
learning/prompts/code-explanation/09_io_avro.md | 7 +-
learning/prompts/code-explanation/10_io_parquet.md | 7 +-
learning/prompts/code-explanation/11_io_jdbc.md | 6 +-
.../47_batching_for_external_calls.md | 71 ++
.../48_resource_hints.md | 65 ++
.../49_container_environments.md | 121 +++
playground/backend/containers/java/Dockerfile | 2 +-
playground/backend/containers/python/Dockerfile | 4 +-
.../runners/flink/adapter/BeamAdapterUtils.java | 6 +-
.../flink/adapter/BeamFlinkDataSetAdapter.java | 47 +-
.../beam/runners/flink/adapter/FlinkInput.java | 2 +-
.../beam/runners/flink/adapter/FlinkOutput.java | 2 +-
.../examples/build.gradle | 6 +-
sdks/go/pkg/beam/core/core.go | 2 +-
.../resources/beam/checkstyle/suppressions.xml | 1 +
.../sdk/util/construction/TransformUpgrader.java | 58 +
.../util/construction/TransformUpgraderTest.java | 13 +
.../sdk/io/gcp/bigquery/BigQueryIOTranslation.java | 69 +-
.../io/gcp/bigtable/BigtableConfigTranslator.java | 32 +-
.../gcp/bigtable/BigtableConfigTranslatorTest.java | 2 +-
.../io/external/xlang_jdbcio_it_test.py | 24 +-
.../apache_beam/io/gcp/bigquery_tools_test.py | 57 +-
.../io/hdfs_integration_test/docker-compose.yml | 4 +-
.../runners/dataflow/dataflow_runner.py | 20 +
.../runners/dataflow/dataflow_runner_test.py | 84 +-
.../runners/portability/expansion_service.py | 7 +
.../runners/portability/fn_api_runner/fn_runner.py | 15 +
.../runners/portability/portable_runner_test.py | 41 +-
.../apache_beam/runners/portability/stager.py | 54 +-
.../apache_beam/runners/portability/stager_test.py | 91 +-
sdks/python/apache_beam/transforms/enrichment.py | 6 +-
.../transforms/enrichment_handlers/bigtable.py | 20 +-
.../enrichment_handlers/utils.py} | 22 +-
.../enrichment_handlers/vertex_ai_feature_store.py | 306 ++++++
.../vertex_ai_feature_store_it_test.py | 287 +++++
.../vertex_ai_feature_store_test.py | 58 +
.../vertex_ai_feature_store_tests_requirement.txt} | 6 +-
.../apache_beam/transforms/environments_test.py | 11 +
sdks/python/apache_beam/version.py | 2 +-
sdks/python/container/boot.go | 32 +-
.../containers/tensorrt_runinference/README.md | 2 +-
.../tensorrt_runinference/tensor_rt.dockerfile | 3 +-
sdks/python/test-suites/direct/common.gradle | 11 +-
sdks/typescript/package.json | 2 +-
settings.gradle.kts | 1 +
.../en/documentation/io/built-in/webapis.md | 536 +++++++++
.../site/content/en/documentation/io/connectors.md | 13 +
.../www/site/content/en/documentation/sdks/yaml.md | 7 +-
.../partials/section-menu/en/documentation.html | 1 +
85 files changed, 4903 insertions(+), 243 deletions(-)
delete mode 100644 .github/trigger_files/beam_PostCommit_Python.json
create mode 100644 examples/java/webapis/build.gradle
create mode 100644 examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/AdditionalSnippets.java
create mode 100644 examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/GeminiAIClient.java
create mode 100644 examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/GeminiAIExample.java
create mode 100644 examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/GeminiAIOptions.java
create mode 100644 examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/GenerateContentRequestCoder.java
create mode 100644 examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/GenerateContentResponseCoder.java
create mode 100644 examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/HttpImageClient.java
create mode 100644 examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/ImageRequest.java
create mode 100644 examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/ImageRequestCoder.java
create mode 100644 examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/ImageResponse.java
create mode 100644 examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/ImageResponseCoder.java
create mode 100644 examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/Images.java
create mode 100644 examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/Log.java
create mode 100644 examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/UsingHttpClientExample.java
create mode 100644 examples/java/webapis/src/test/java/org/apache/beam/examples/webapis/GeminiAIExampleTest.java
create mode 100644 examples/java/webapis/src/test/java/org/apache/beam/examples/webapis/UsingHttpClientExampleTest.java
create mode 100644 examples/notebooks/beam-ml/run_inference_gemma.ipynb
create mode 100644 it/kafka/src/test/java/org/apache/beam/it/kafka/KafkaIOST.java
create mode 100644 learning/prompts/documentation-lookup-nolinks/47_batching_for_external_calls.md
create mode 100644 learning/prompts/documentation-lookup-nolinks/48_resource_hints.md
create mode 100644 learning/prompts/documentation-lookup-nolinks/49_container_environments.md
copy sdks/python/apache_beam/{version.py => transforms/enrichment_handlers/utils.py} (61%)
create mode 100644 sdks/python/apache_beam/transforms/enrichment_handlers/vertex_ai_feature_store.py
create mode 100644 sdks/python/apache_beam/transforms/enrichment_handlers/vertex_ai_feature_store_it_test.py
create mode 100644 sdks/python/apache_beam/transforms/enrichment_handlers/vertex_ai_feature_store_test.py
copy sdks/python/apache_beam/{version.py => transforms/enrichment_handlers/vertex_ai_feature_store_tests_requirement.txt} (90%)
create mode 100644 website/www/site/content/en/documentation/io/built-in/webapis.md